From 9fae4eef8452c219c2b8574867091030a93cda87 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 5 Nov 2018 22:28:59 +0000 Subject: [PATCH 001/526] Add the TLSv1_3 setting to disable TLSv1_3 --- doc/admin-guide/files/records.config.en.rst | 8 ++++++++ iocore/net/SSLConfig.cc | 11 +++++++++++ mgmt/RecordsConfig.cc | 4 ++++ tests/gold_tests/tls_hooks/tls_hooks.test.py | 4 +++- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index a5075e06e00..f7d2f64f920 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3245,6 +3245,10 @@ SSL Termination Enables (``1``) or disables (``0``) TLS v1.2. If not specified, enabled by default. [Requires OpenSSL v1.0.1 and higher] +.. ts:cv:: CONFIG proxy.config.ssl.TLSv1_3 INT 1 + + Enables (``1``) or disables (``0``) TLS v1.3. If not specified, enabled by default. [Requires OpenSSL v1.1.1 and higher] + .. ts:cv:: CONFIG proxy.config.ssl.client.certification_level INT 0 Sets the client certification level: @@ -3571,6 +3575,10 @@ Client-Related Configuration Enables (``1``) or disables (``0``) TLSv1_2 in the ATS client context. If not specified, enabled by default +.. ts:cv:: CONFIG proxy.config.ssl.client.TLSv1_3 INT 1 + + Enables (``1``) or disables (``0``) TLSv1_3 in the ATS client context. If not specified, enabled by default + .. ts:cv:: CONFIG proxy.config.ssl.async.handshake.enabled INT 0 Enables the use of openssl async job during the TLS handshake. Traffic diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 018cfcf051d..4948a2d2822 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -236,6 +236,17 @@ SSLConfigParams::initialize() ssl_client_ctx_options |= SSL_OP_NO_TLSv1_2; } #endif +#ifdef SSL_OP_NO_TLSv1_3 + REC_ReadConfigInteger(options, "proxy.config.ssl.TLSv1_3"); + if (!options) { + ssl_ctx_options |= SSL_OP_NO_TLSv1_3; + } + + REC_ReadConfigInteger(client_ssl_options, "proxy.config.ssl.client.TLSv1_3"); + if (!client_ssl_options) { + ssl_client_ctx_options |= SSL_OP_NO_TLSv1_3; + } +#endif #ifdef SSL_OP_CIPHER_SERVER_PREFERENCE REC_ReadConfigInteger(options, "proxy.config.ssl.server.honor_cipher_order"); diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index b28e33090b1..dff7e283a16 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1096,6 +1096,8 @@ static const RecordElement RecordsConfig[] = // Disable this when using some versions of OpenSSL that causes crashes. See TS-2355. {RECT_CONFIG, "proxy.config.ssl.TLSv1_2", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.TLSv1_3", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , // Client SSL protocols #if TS_USE_SSLV3_CLIENT @@ -1108,6 +1110,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.client.TLSv1_2", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.client.TLSv1_3", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , {RECT_CONFIG, "proxy.config.ssl.server.cipher_suite", RECD_STRING, "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.client.cipher_suite", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} diff --git a/tests/gold_tests/tls_hooks/tls_hooks.test.py b/tests/gold_tests/tls_hooks/tls_hooks.test.py index deb644858fe..7b4f006d41b 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks.test.py @@ -45,6 +45,7 @@ # enable ssl port 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.TLSv1_3': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -63,9 +64,10 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'curl -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.Command = 'curl -v -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/preaccept-1.gold" +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("TLSv1.3 (IN), TLS handshake, Finished (20):", "Should not negotiate a TLSv1.3 connection") ts.Streams.stderr = "gold/ts-preaccept-1.gold" From 63f24f438bfac80357024e8e75caa8c12b5ce551 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 5 Nov 2018 17:04:54 +0000 Subject: [PATCH 002/526] Make autests friendlier with openssl 1.1.1 --- tests/gold_tests/headers/forwarded.gold | 6 +++--- tests/gold_tests/headers/via.gold | 6 +++--- tests/gold_tests/redirect/gold/redirect.gold | 1 + tests/gold_tests/tls/ssl-post.c | 2 ++ tests/gold_tests/tls/tls.test.py | 5 +++-- tests/gold_tests/tls/tls_ticket.test.py | 4 ++-- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/gold_tests/headers/forwarded.gold b/tests/gold_tests/headers/forwarded.gold index 45451d62a54..939825a1372 100644 --- a/tests/gold_tests/headers/forwarded.gold +++ b/tests/gold_tests/headers/forwarded.gold @@ -31,11 +31,11 @@ for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=http for=0.6.6.6 for=_argh, for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=http;host=www.no-oride.com;connection=http;connection=http/1.0;connection=http/1.0-tcp-ipv4 - -for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=https;host=www.no-oride.com;connection=https;connection=https/2;connection=http/1.1-h2-tls/1.2-tcp-ipv4 +for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=https;host=www.no-oride.com;connection=https;connection=https/2;connection=http/1.1-h2-tls/1.{}-tcp-ipv4 - -for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=https;host=www.no-oride.com;connection=https;connection=https/1.1;connection=http/1.1-tls/1.2-tcp-ipv4 +for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=https;host=www.no-oride.com;connection=https;connection=https/1.1;connection=http/1.1-tls/1.{}-tcp-ipv4 - for="[::1]";by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by="[::1]";proto=http;host=www.no-oride.com;connection=http;connection=http/1.1;connection=http/1.1-tcp-ipv6 - -for="[::1]";by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by="[::1]";proto=https;host=www.no-oride.com;connection=https;connection=https/1.1;connection=http/1.1-tls/1.2-tcp-ipv6 +for="[::1]";by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by="[::1]";proto=https;host=www.no-oride.com;connection=https;connection=https/1.1;connection=http/1.1-tls/1.{}-tcp-ipv6 - diff --git a/tests/gold_tests/headers/via.gold b/tests/gold_tests/headers/via.gold index 598ef376e75..01165cfe7e5 100644 --- a/tests/gold_tests/headers/via.gold +++ b/tests/gold_tests/headers/via.gold @@ -1,6 +1,6 @@ Via: http/1.1 = http/1.1 tcp ipv4 Via: http/1.0 = http/1.0 tcp ipv4 -Via: https/2 = http/1.1 h2 tls/1.2 tcp ipv4 -Via: https/1.1 = http/1.1 tls/1.2 tcp ipv4 +Via: https/2 = http/1.1 h2 tls/1.{} tcp ipv4 +Via: https/1.1 = http/1.1 tls/1.{} tcp ipv4 Via: http/1.1 = http/1.1 tcp ipv6 -Via: https/1.1 = http/1.1 tls/1.2 tcp ipv6 +Via: https/1.1 = http/1.1 tls/1.{} tcp ipv6 diff --git a/tests/gold_tests/redirect/gold/redirect.gold b/tests/gold_tests/redirect/gold/redirect.gold index d6616ba55f9..9500bf976a0 100644 --- a/tests/gold_tests/redirect/gold/redirect.gold +++ b/tests/gold_tests/redirect/gold/redirect.gold @@ -1,4 +1,5 @@ HTTP/1.1 204 No Content +`` Age: `` Connection: keep-alive diff --git a/tests/gold_tests/tls/ssl-post.c b/tests/gold_tests/tls/ssl-post.c index cc8ea132aac..00dc3a07a3f 100644 --- a/tests/gold_tests/tls/ssl-post.c +++ b/tests/gold_tests/tls/ssl-post.c @@ -89,6 +89,7 @@ spawn_same_session_send(void *arg) SSL_CTX *client_ctx = SSL_CTX_new(SSLv23_client_method()); SSL *ssl = SSL_new(client_ctx); + SSL_set_max_proto_version(ssl, TLS1_2_VERSION); SSL_set_session(ssl, tinfo->session); SSL_set_fd(ssl, sfd); @@ -282,6 +283,7 @@ main(int argc, char *argv[]) SSL_CTX *client_ctx = SSL_CTX_new(SSLv23_client_method()); SSL *ssl = SSL_new(client_ctx); + SSL_set_max_proto_version(ssl, TLS1_2_VERSION); SSL_set_fd(ssl, sfd); int ret = SSL_connect(ssl); diff --git a/tests/gold_tests/tls/tls.test.py b/tests/gold_tests/tls/tls.test.py index f1ef8cdc7ed..f23232a0398 100644 --- a/tests/gold_tests/tls/tls.test.py +++ b/tests/gold_tests/tls/tls.test.py @@ -40,10 +40,11 @@ testName = "" header_count = 378 +#header_count = 78 header_string = "POST /post HTTP/1.1\r\nHost: www.example.com\r\nContent-Length:1000\r\n" -for i in range(0, 378): +for i in range(0, header_count): header_string = "{1}header{0}:{0}\r\n".format(i, header_string) header_string = "{0}\r\n".format(header_string) @@ -78,7 +79,7 @@ }) tr = Test.AddTestRun("Run-Test") -tr.Command = './ssl-post 127.0.0.1 40 378 4443' +tr.Command = './ssl-post 127.0.0.1 40 {0} 4443'.format(header_count) tr.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/tls/tls_ticket.test.py b/tests/gold_tests/tls/tls_ticket.test.py index 9a2b43b1252..c9953cc313e 100644 --- a/tests/gold_tests/tls/tls_ticket.test.py +++ b/tests/gold_tests/tls/tls_ticket.test.py @@ -82,7 +82,7 @@ tr = Test.AddTestRun("Create ticket") tr.Setup.Copy('file.ticket') -tr.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -connect 127.0.0.1:{0} -sess_out ticket.out'.format(ts.Variables.ssl_port) +tr.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_out ticket.out'.format(ts.Variables.ssl_port) tr.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server) @@ -120,7 +120,7 @@ def checkSession(ev) : tr2 = Test.AddTestRun("Test ticket") tr2.Setup.Copy('file.ticket') -tr2.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -connect 127.0.0.1:{0} -sess_in ticket.out'.format(ts2.Variables.ssl_port) +tr2.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_in ticket.out'.format(ts2.Variables.ssl_port) tr2.Processes.Default.StartBefore(Test.Processes.ts2, ready=When.PortOpen(ts2.Variables.ssl_port)) tr2.ReturnCode = 0 path2 = tr2.Processes.Default.Streams.stdout.AbsPath From bf1cfb928e42361185012256fd8051e9256c53c2 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 6 Nov 2018 17:43:12 -0600 Subject: [PATCH 003/526] Cleanup: Update Version support to be actual C++ code. --- include/tscore/I_Version.h | 107 ++++++++++++------ iocore/aio/AIO.cc | 4 +- iocore/aio/I_AIO.h | 6 +- iocore/aio/P_AIO.h | 3 +- iocore/aio/test_AIO.cc | 4 +- iocore/cache/Cache.cc | 20 ++-- iocore/cache/CacheRead.cc | 2 +- iocore/cache/I_Cache.h | 10 +- iocore/cache/I_CacheDefs.h | 2 +- iocore/cache/P_CacheVol.h | 2 +- iocore/dns/DNS.cc | 4 +- iocore/dns/I_DNSProcessor.h | 2 +- iocore/dns/I_SplitDNS.h | 6 +- iocore/dns/P_SplitDNS.h | 4 +- iocore/dns/P_SplitDNSProcessor.h | 2 +- iocore/dns/SplitDNS.cc | 4 +- iocore/dns/test_I_DNS.cc | 2 +- iocore/eventsystem/EventSystem.cc | 4 +- iocore/eventsystem/I_EventSystem.h | 7 +- iocore/eventsystem/P_EventSystem.h | 8 +- iocore/eventsystem/test_Buffer.cc | 2 +- iocore/eventsystem/test_Event.cc | 2 +- .../unit_tests/test_MIOBufferWriter.cc | 2 +- iocore/hostdb/HostDB.cc | 4 +- iocore/hostdb/I_HostDB.h | 5 +- iocore/hostdb/I_HostDBProcessor.h | 6 +- iocore/hostdb/P_HostDB.h | 4 +- iocore/hostdb/P_RefCountCache.h | 16 +-- iocore/hostdb/RefCountCache.cc | 12 +- iocore/hostdb/test_I_HostDB.cc | 6 +- iocore/hostdb/test_RefCountCache.cc | 2 +- iocore/net/I_Net.h | 9 +- iocore/net/Net.cc | 5 +- iocore/net/P_Net.h | 4 +- iocore/net/test_I_Net.cc | 4 +- iocore/net/test_I_UDPNet.cc | 2 +- iocore/net/test_P_Net.cc | 2 +- src/traffic_cache_tool/CacheDefs.cc | 10 +- src/traffic_cache_tool/CacheTool.cc | 5 +- src/traffic_server/traffic_server.cc | 22 ++-- 40 files changed, 173 insertions(+), 154 deletions(-) diff --git a/include/tscore/I_Version.h b/include/tscore/I_Version.h index 7aa616d4b3b..7f89f9cfc9a 100644 --- a/include/tscore/I_Version.h +++ b/include/tscore/I_Version.h @@ -30,18 +30,40 @@ #pragma once +namespace ts +{ +/** Container for standard two part version number. + */ struct VersionNumber { - short int ink_major; // incompatible change - short int ink_minor; // minor change, not incompatible + /// Construct invalid (0.0) version. + constexpr VersionNumber() = default; - VersionNumber() : ink_major(0), ink_minor(0) {} - VersionNumber(short int major, short int minor) : ink_major(major), ink_minor(minor) {} + /// Construct explicit version. + constexpr explicit VersionNumber(unsigned short major, unsigned short minor = 0); + + // Can't use unadorned "major" because that's a macro. + unsigned short _major = 0; ///< Major version. + unsigned short _minor = 0; ///< Minor version. }; +inline constexpr VersionNumber::VersionNumber(unsigned short major, unsigned short minor) : _major(major), _minor(minor) {} + inline bool operator<(VersionNumber const &lhs, VersionNumber const &rhs) { - return lhs.ink_major < rhs.ink_major || (lhs.ink_major == rhs.ink_major && lhs.ink_minor < rhs.ink_minor); + return lhs._major < rhs._major || (lhs._major == rhs._major && lhs._minor < rhs._minor); +} + +inline bool +operator==(VersionNumber const &lhs, VersionNumber const &rhs) +{ + return lhs._major == rhs._major && lhs._minor == rhs._minor; +} + +inline bool +operator!=(VersionNumber const &lhs, VersionNumber const &rhs) +{ + return !(lhs == rhs); } inline bool @@ -51,9 +73,15 @@ operator>(VersionNumber const &lhs, VersionNumber const &rhs) } inline bool -operator==(VersionNumber const &lhs, VersionNumber const &rhs) +operator<=(VersionNumber const &lhs, VersionNumber const &rhs) +{ + return !(lhs > rhs); +} + +inline bool +operator>=(VersionNumber const &lhs, VersionNumber const &rhs) { - return lhs.ink_major == rhs.ink_major && lhs.ink_minor == rhs.ink_minor; + return !(rhs > lhs); } struct Version { @@ -61,39 +89,50 @@ struct Version { VersionNumber cacheDir; }; -enum ModuleVersion { - MODULE_VERSION_MIN = 0, - MODULE_VERSION_MAX = 2147483647, -}; -enum ModuleHeaderType { - PUBLIC_MODULE_HEADER, - PRIVATE_MODULE_HEADER, -}; +/// Container for a module version. +struct ModuleVersion { + /// Type of module. + enum Type : unsigned char { PUBLIC, PRIVATE }; + + constexpr ModuleVersion(unsigned char major, unsigned char minor, Type type = PUBLIC); + constexpr ModuleVersion(ModuleVersion const &base, Type type); -#define makeModuleVersion(_major_version, _minor_version, _module_type) \ - ((ModuleVersion)((((int)_module_type) << 24) + (((int)_major_version) << 16) + (((int)_minor_version) << 8))) + /** Check if @a that is a version compatible with @a this. + * + * @param that Version to check against. + * @return @a true if @a that is compatible with @a this, @c false otherwise. + */ + bool check(ModuleVersion const &that); -#define majorModuleVersion(_v) ((((int)_v) >> 16) & 255) -#define minorModuleVersion(_v) ((((int)_v) >> 8) & 255) -#define moduleVersionType(_v) ((((int)_v) >> 24) & 127) + Type _type = PUBLIC; ///< The numeric value of the module version. + unsigned char _major = 0; ///< Major version. + unsigned char _minor = 0; +}; -static inline int -checkModuleVersion(ModuleVersion userVersion, ModuleVersion libVersion) +inline constexpr ModuleVersion::ModuleVersion(unsigned char major, unsigned char minor, ts::ModuleVersion::Type type) + : _type(type), _major(major), _minor(minor) { - if (moduleVersionType(userVersion) == PUBLIC_MODULE_HEADER) { - if ((majorModuleVersion(userVersion) != majorModuleVersion(libVersion)) || - (minorModuleVersion(userVersion) > minorModuleVersion(libVersion))) - return -1; - return 0; - } else if (moduleVersionType(userVersion) == PRIVATE_MODULE_HEADER) { - if ((majorModuleVersion(userVersion) != majorModuleVersion(libVersion)) || - (minorModuleVersion(userVersion) != minorModuleVersion(libVersion))) - return -1; - return 0; - } else - return -1; } +inline constexpr ModuleVersion::ModuleVersion(ModuleVersion const &base, ts::ModuleVersion::Type type) + : _type(type), _major(base._major), _minor(base._minor) +{ +} + +inline bool +ModuleVersion::check(ModuleVersion const &that) +{ + switch (_type) { + case PUBLIC: + return _major == that._major && _minor <= that._minor; + case PRIVATE: + return _major == that._major && _minor == that._minor; + } + return false; +}; + +} // namespace ts + class AppVersionInfo { public: diff --git a/iocore/aio/AIO.cc b/iocore/aio/AIO.cc index 0e7f15add5f..ca7860cd72b 100644 --- a/iocore/aio/AIO.cc +++ b/iocore/aio/AIO.cc @@ -143,9 +143,9 @@ ink_aio_set_callback(Continuation *callback) } void -ink_aio_init(ModuleVersion v) +ink_aio_init(ts::ModuleVersion v) { - ink_release_assert(!checkModuleVersion(v, AIO_MODULE_VERSION)); + ink_release_assert(v.check(AIO_MODULE_INTERNAL_VERSION)); aio_rsb = RecAllocateRawStatBlock((int)AIO_STAT_COUNT); RecRegisterRawStat(aio_rsb, RECT_PROCESS, "proxy.process.cache.read_per_sec", RECD_FLOAT, RECP_PERSISTENT, diff --git a/iocore/aio/I_AIO.h b/iocore/aio/I_AIO.h index 8222ba1a59c..298ca54b0a5 100644 --- a/iocore/aio/I_AIO.h +++ b/iocore/aio/I_AIO.h @@ -34,9 +34,7 @@ #include "I_EventSystem.h" #include "records/I_RecProcess.h" -#define AIO_MODULE_MAJOR_VERSION 1 -#define AIO_MODULE_MINOR_VERSION 0 -#define AIO_MODULE_VERSION makeModuleVersion(AIO_MODULE_MAJOR_VERSION, AIO_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion AIO_MODULE_PUBLIC_VERSION(1, 0, ts::ModuleVersion::PUBLIC); #define AIO_EVENT_DONE (AIO_EVENT_EVENTS_START + 0) @@ -141,7 +139,7 @@ struct DiskHandler : public Continuation { }; #endif -void ink_aio_init(ModuleVersion version); +void ink_aio_init(ts::ModuleVersion version); int ink_aio_start(); void ink_aio_set_callback(Continuation *error_callback); diff --git a/iocore/aio/P_AIO.h b/iocore/aio/P_AIO.h index d4d21e5992a..06eb15b8fb1 100644 --- a/iocore/aio/P_AIO.h +++ b/iocore/aio/P_AIO.h @@ -36,8 +36,7 @@ // for debugging // #define AIO_STATS 1 -#undef AIO_MODULE_VERSION -#define AIO_MODULE_VERSION makeModuleVersion(AIO_MODULE_MAJOR_VERSION, AIO_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) +static constexpr ts::ModuleVersion AIO_MODULE_INTERNAL_VERSION{AIO_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE}; TS_INLINE int AIOCallback::ok() diff --git a/iocore/aio/test_AIO.cc b/iocore/aio/test_AIO.cc index 33a972179b8..4292e6de55a 100644 --- a/iocore/aio/test_AIO.cc +++ b/iocore/aio/test_AIO.cc @@ -414,7 +414,7 @@ main(int /* argc ATS_UNUSED */, char *argv[]) Layout::create(); init_diags("", nullptr); RecProcessInit(RECM_STAND_ALONE); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(ink_number_of_processors()); Thread *main_thread = new EThread; @@ -431,7 +431,7 @@ main(int /* argc ATS_UNUSED */, char *argv[]) #endif RecProcessStart(); - ink_aio_init(AIO_MODULE_VERSION); + ink_aio_init(AIO_MODULE_PUBLIC_VERSION); srand48(time(nullptr)); printf("input file %s\n", argv[1]); if (!read_config(argv[1])) { diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 2bac39c9ad0..28bace51a27 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -39,7 +39,7 @@ #include -const VersionNumber CACHE_DB_VERSION(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION); +constexpr ts::VersionNumber CACHE_DB_VERSION(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION); // Compilation Options #define USELESS_REENABLES // allow them for now @@ -1192,9 +1192,9 @@ vol_clear_init(Vol *d) size_t dir_len = d->dirlen(); memset(d->raw_dir, 0, dir_len); vol_init_dir(d); - d->header->magic = VOL_MAGIC; - d->header->version.ink_major = CACHE_DB_MAJOR_VERSION; - d->header->version.ink_minor = CACHE_DB_MINOR_VERSION; + d->header->magic = VOL_MAGIC; + d->header->version._major = CACHE_DB_MAJOR_VERSION; + d->header->version._minor = CACHE_DB_MINOR_VERSION; d->scan_pos = d->header->agg_pos = d->header->write_pos = d->start; d->header->last_write_pos = d->header->write_pos; d->header->phase = 0; @@ -1363,12 +1363,12 @@ Vol::handle_dir_read(int event, void *data) } } - if (!(header->magic == VOL_MAGIC && footer->magic == VOL_MAGIC && - CACHE_DB_MAJOR_VERSION_COMPATIBLE <= header->version.ink_major && header->version.ink_major <= CACHE_DB_MAJOR_VERSION)) { + if (!(header->magic == VOL_MAGIC && footer->magic == VOL_MAGIC && CACHE_DB_MAJOR_VERSION_COMPATIBLE <= header->version._major && + header->version._major <= CACHE_DB_MAJOR_VERSION)) { Warning("bad footer in cache directory for '%s', clearing", hash_text.get()); Note("VOL_MAGIC %d\n header magic: %d\n footer_magic %d\n CACHE_DB_MAJOR_VERSION_COMPATIBLE %d\n major version %d\n" "CACHE_DB_MAJOR_VERSION %d\n", - VOL_MAGIC, header->magic, footer->magic, CACHE_DB_MAJOR_VERSION_COMPATIBLE, header->version.ink_major, + VOL_MAGIC, header->magic, footer->magic, CACHE_DB_MAJOR_VERSION_COMPATIBLE, header->version._major, CACHE_DB_MAJOR_VERSION); Note("clearing cache directory '%s'", hash_text.get()); clear_dir(); @@ -2295,7 +2295,7 @@ CacheVC::handleReadDone(int event, Event *e) goto Ldone; } } else if (doc->doc_type == CACHE_FRAG_TYPE_HTTP) { // handle any version updates based on the object version - if (VersionNumber(doc->v_major, doc->v_minor) > CACHE_DB_VERSION) { + if (ts::VersionNumber(doc->v_major, doc->v_minor) > CACHE_DB_VERSION) { // future version, count as corrupted doc->magic = DOC_CORRUPT; Debug("cache_bc", "Object is future version %d:%d - disk %s - doc id = %" PRIx64 ":%" PRIx64 "", doc->v_major, doc->v_minor, @@ -3153,9 +3153,9 @@ register_cache_stats(RecRawStatBlock *rsb, const char *prefix) } void -ink_cache_init(ModuleVersion v) +ink_cache_init(ts::ModuleVersion v) { - ink_release_assert(!checkModuleVersion(v, CACHE_MODULE_VERSION)); + ink_release_assert(v.check(CACHE_MODULE_VERSION)); cache_rsb = RecAllocateRawStatBlock((int)cache_stat_count); diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index 662a1027695..92179469b73 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -172,7 +172,7 @@ CacheVC::load_http_info(CacheHTTPInfoVector *info, Doc *doc, RefCountObj *block_ if (!this->f.doc_from_ram_cache && // ram cache is always already fixed up. // If this is an old object, the object version will be old or 0, in either case this is // correct. Forget the 4.2 compatibility, always update older versioned objects. - VersionNumber(doc->v_major, doc->v_minor) < CACHE_DB_VERSION) { + ts::VersionNumber(doc->v_major, doc->v_minor) < CACHE_DB_VERSION) { for (int i = info->xcount - 1; i >= 0; --i) { info->data(i).alternate.m_alt->m_response_hdr.m_mime->recompute_accelerators_and_presence_bits(); info->data(i).alternate.m_alt->m_request_hdr.m_mime->recompute_accelerators_and_presence_bits(); diff --git a/iocore/cache/I_Cache.h b/iocore/cache/I_Cache.h index 14e19d90241..57f574a84dd 100644 --- a/iocore/cache/I_Cache.h +++ b/iocore/cache/I_Cache.h @@ -29,9 +29,7 @@ #include "I_CacheDefs.h" #include "I_Store.h" -#define CACHE_MODULE_MAJOR_VERSION 1 -#define CACHE_MODULE_MINOR_VERSION 0 -#define CACHE_MODULE_VERSION makeModuleVersion(CACHE_MODULE_MAJOR_VERSION, CACHE_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion CACHE_MODULE_VERSION(1, 0); #define CACHE_WRITE_OPT_OVERWRITE 0x0001 #define CACHE_WRITE_OPT_CLOSE_COMPLETE 0x0002 @@ -158,8 +156,8 @@ struct CacheProcessor : public Processor { static int start_internal_flags; static int auto_clear_flag; - VersionNumber min_stripe_version; - VersionNumber max_stripe_version; + ts::VersionNumber min_stripe_version; + ts::VersionNumber max_stripe_version; CALLBACK_FUNC cb_after_init; int wait_for_cache; @@ -218,6 +216,6 @@ struct CacheVConnection : public VConnection { CacheVConnection(); }; -void ink_cache_init(ModuleVersion version); +void ink_cache_init(ts::ModuleVersion version); extern inkcoreapi CacheProcessor cacheProcessor; extern Continuation *cacheRegexDeleteCont; diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h index c4529745fd3..619d20c158f 100644 --- a/iocore/cache/I_CacheDefs.h +++ b/iocore/cache/I_CacheDefs.h @@ -37,7 +37,7 @@ static const uint8_t CACHE_DB_MINOR_VERSION = 1; // This is used in various comparisons because otherwise if the minor version is 0, // the compile fails because the condition is always true or false. Running it through // VersionNumber prevents that. -extern const VersionNumber CACHE_DB_VERSION; +extern const ts::VersionNumber CACHE_DB_VERSION; static const uint8_t CACHE_DIR_MAJOR_VERSION = 18; static const uint8_t CACHE_DIR_MINOR_VERSION = 0; diff --git a/iocore/cache/P_CacheVol.h b/iocore/cache/P_CacheVol.h index a59df2c848b..a927eb46113 100644 --- a/iocore/cache/P_CacheVol.h +++ b/iocore/cache/P_CacheVol.h @@ -76,7 +76,7 @@ struct CacheVol; struct VolHeaderFooter { unsigned int magic; - VersionNumber version; + ts::VersionNumber version; time_t create_time; off_t write_pos; off_t last_write_pos; diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 8868f88e257..5db7f67d0d8 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -1771,13 +1771,13 @@ Lerror:; RecRawStatBlock *dns_rsb; void -ink_dns_init(ModuleVersion v) +ink_dns_init(ts::ModuleVersion v) { static int init_called = 0; Debug("dns", "ink_dns_init: called with init_called = %d", init_called); - ink_release_assert(!checkModuleVersion(v, HOSTDB_MODULE_VERSION)); + ink_release_assert(v.check(HOSTDB_MODULE_PUBLIC_VERSION)); if (init_called) { return; } diff --git a/iocore/dns/I_DNSProcessor.h b/iocore/dns/I_DNSProcessor.h index 02aa4cace12..15af75049ce 100644 --- a/iocore/dns/I_DNSProcessor.h +++ b/iocore/dns/I_DNSProcessor.h @@ -202,4 +202,4 @@ DNSProcessor::Options::reset() return *this; } -void ink_dns_init(ModuleVersion version); +void ink_dns_init(ts::ModuleVersion version); diff --git a/iocore/dns/I_SplitDNS.h b/iocore/dns/I_SplitDNS.h index c038ea24bd7..e277bce0664 100644 --- a/iocore/dns/I_SplitDNS.h +++ b/iocore/dns/I_SplitDNS.h @@ -31,8 +31,6 @@ #pragma once #include "I_SplitDNSProcessor.h" +#include "tscore/I_Version.h" -#define SPLITDNS_MODULE_MAJOR_VERSION 1 -#define SPLITDNS_MODULE_MINOR_VERSION 0 -#define SPLITDNS_MODULE_VERSION \ - makeModuleVersion(SPLITDNS_MODULE_MAJOR_VERSION, SPLITDNS_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion SPLITDNS_MODULE_PUBLIC_VERSION(1, 0, ts::ModuleVersion::PUBLIC); diff --git a/iocore/dns/P_SplitDNS.h b/iocore/dns/P_SplitDNS.h index 6cf92f16c2f..19df3076e0b 100644 --- a/iocore/dns/P_SplitDNS.h +++ b/iocore/dns/P_SplitDNS.h @@ -37,6 +37,4 @@ #include "ControlMatcher.h" #include "P_SplitDNSProcessor.h" -#undef SPLITDNS_MODULE_VERSION -#define SPLITDNS_MODULE_VERSION \ - makeModuleVersion(SPLITDNS_MODULE_MAJOR_VERSION, SPLITDNS_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) +static constexpr ts::ModuleVersion SPLITDNS_MODULE_INTERNAL_VERSION{SPLITDNS_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE}; diff --git a/iocore/dns/P_SplitDNSProcessor.h b/iocore/dns/P_SplitDNSProcessor.h index 33de46eabf1..4e206e21103 100644 --- a/iocore/dns/P_SplitDNSProcessor.h +++ b/iocore/dns/P_SplitDNSProcessor.h @@ -42,7 +42,7 @@ /* --------------------------- forward declarations ... --------------------------- */ -void ink_split_dns_init(ModuleVersion version); +void ink_split_dns_init(ts::ModuleVersion version); struct RequestData; diff --git a/iocore/dns/SplitDNS.cc b/iocore/dns/SplitDNS.cc index a7cbcd2be2c..e435ba05296 100644 --- a/iocore/dns/SplitDNS.cc +++ b/iocore/dns/SplitDNS.cc @@ -544,11 +544,11 @@ SplitDNSRecord::Print() } void -ink_split_dns_init(ModuleVersion v) +ink_split_dns_init(ts::ModuleVersion v) { static int init_called = 0; - ink_release_assert(!checkModuleVersion(v, SPLITDNS_MODULE_VERSION)); + ink_release_assert(v.check(SPLITDNS_MODULE_INTERNAL_VERSION)); if (init_called) { return; } diff --git a/iocore/dns/test_I_DNS.cc b/iocore/dns/test_I_DNS.cc index 9ba3d863eae..87fd8f99b5c 100644 --- a/iocore/dns/test_I_DNS.cc +++ b/iocore/dns/test_I_DNS.cc @@ -29,7 +29,7 @@ main() { init_diags("net_test", nullptr); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); signal(SIGPIPE, SIG_IGN); eventProcessor.start(2); diff --git a/iocore/eventsystem/EventSystem.cc b/iocore/eventsystem/EventSystem.cc index 1cb361397c9..c7f1e99c509 100644 --- a/iocore/eventsystem/EventSystem.cc +++ b/iocore/eventsystem/EventSystem.cc @@ -31,9 +31,9 @@ #include "P_EventSystem.h" void -ink_event_system_init(ModuleVersion v) +ink_event_system_init(ts::ModuleVersion v) { - ink_release_assert(!checkModuleVersion(v, EVENT_SYSTEM_MODULE_VERSION)); + ink_release_assert(v.check(EVENT_SYSTEM_MODULE_INTERNAL_VERSION)); int config_max_iobuffer_size = DEFAULT_MAX_BUFFER_SIZE; int iobuffer_advice = 0; diff --git a/iocore/eventsystem/I_EventSystem.h b/iocore/eventsystem/I_EventSystem.h index 8d51a93bb9b..4675a5e8c16 100644 --- a/iocore/eventsystem/I_EventSystem.h +++ b/iocore/eventsystem/I_EventSystem.h @@ -44,9 +44,6 @@ #include "records/I_RecProcess.h" #include "I_SocketManager.h" -#define EVENT_SYSTEM_MODULE_MAJOR_VERSION 1 -#define EVENT_SYSTEM_MODULE_MINOR_VERSION 0 -#define EVENT_SYSTEM_MODULE_VERSION \ - makeModuleVersion(EVENT_SYSTEM_MODULE_MAJOR_VERSION, EVENT_SYSTEM_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion EVENT_SYSTEM_MODULE_PUBLIC_VERSION(1, 0, ts::ModuleVersion::PUBLIC); -void ink_event_system_init(ModuleVersion version); +void ink_event_system_init(ts::ModuleVersion version); diff --git a/iocore/eventsystem/P_EventSystem.h b/iocore/eventsystem/P_EventSystem.h index 006fa3d95c8..a6222ba49d3 100644 --- a/iocore/eventsystem/P_EventSystem.h +++ b/iocore/eventsystem/P_EventSystem.h @@ -29,7 +29,7 @@ **************************************************************************/ #pragma once -#define _P_EventSystem_h +#define _P_EventSystem_H #include "tscore/ink_platform.h" @@ -45,6 +45,6 @@ #include "P_ProtectedQueue.h" #include "P_UnixEventProcessor.h" #include "P_UnixSocketManager.h" -#undef EVENT_SYSTEM_MODULE_VERSION -#define EVENT_SYSTEM_MODULE_VERSION \ - makeModuleVersion(EVENT_SYSTEM_MODULE_MAJOR_VERSION, EVENT_SYSTEM_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) + +static constexpr ts::ModuleVersion EVENT_SYSTEM_MODULE_INTERNAL_VERSION{EVENT_SYSTEM_MODULE_PUBLIC_VERSION, + ts::ModuleVersion::PRIVATE}; diff --git a/iocore/eventsystem/test_Buffer.cc b/iocore/eventsystem/test_Buffer.cc index 3e31a496fd3..ccef1036864 100644 --- a/iocore/eventsystem/test_Buffer.cc +++ b/iocore/eventsystem/test_Buffer.cc @@ -39,7 +39,7 @@ main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) init_diags("", nullptr); RecProcessInit(mode_type); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(TEST_THREADS); Thread *main_thread = new EThread; diff --git a/iocore/eventsystem/test_Event.cc b/iocore/eventsystem/test_Event.cc index 6f52da1c47a..714aff534b9 100644 --- a/iocore/eventsystem/test_Event.cc +++ b/iocore/eventsystem/test_Event.cc @@ -68,7 +68,7 @@ main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) init_diags("", nullptr); RecProcessInit(mode_type); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(TEST_THREADS, 1048576); // Hardcoded stacksize at 1MB alarm_printer *alrm = new alarm_printer(new_ProxyMutex()); diff --git a/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc b/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc index fd78899e702..759a3e9b22b 100644 --- a/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc +++ b/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc @@ -42,7 +42,7 @@ main(int argc, char *argv[]) init_diags("", nullptr); RecProcessInit(RECM_STAND_ALONE); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(2); Thread *main_thread = new EThread; diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index eb4cc5b2e33..5f76b1c96bf 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -2182,11 +2182,11 @@ REGRESSION_TEST(HostDBTests)(RegressionTest *t, int atype, int *pstatus) RecRawStatBlock *hostdb_rsb; void -ink_hostdb_init(ModuleVersion v) +ink_hostdb_init(ts::ModuleVersion v) { static int init_called = 0; - ink_release_assert(!checkModuleVersion(v, HOSTDB_MODULE_VERSION)); + ink_release_assert(v.check(HOSTDB_MODULE_INTERNAL_VERSION)); if (init_called) { return; } diff --git a/iocore/hostdb/I_HostDB.h b/iocore/hostdb/I_HostDB.h index f21e870c6a0..d9aecd8d8c3 100644 --- a/iocore/hostdb/I_HostDB.h +++ b/iocore/hostdb/I_HostDB.h @@ -39,7 +39,4 @@ // TS-1925: switch from MMH to MD5 hash; bumped to version 2 // switch from MD5 to SHA256 hash; bumped to version 3 // 2.1: Switched to mark RR elements. -#define HOSTDB_MODULE_MAJOR_VERSION 3 -#define HOSTDB_MODULE_MINOR_VERSION 1 - -#define HOSTDB_MODULE_VERSION makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion HOSTDB_MODULE_PUBLIC_VERSION(3, 1); diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h index 3c4c64263cd..a9f5d0cf096 100644 --- a/iocore/hostdb/I_HostDBProcessor.h +++ b/iocore/hostdb/I_HostDBProcessor.h @@ -163,10 +163,10 @@ struct HostDBInfo : public RefCountObj { } // return a version number-- so we can manage compatibility with the marshal/unmarshal - static VersionNumber + static ts::VersionNumber version() { - return VersionNumber(1, 0); + return ts::VersionNumber(1, 0); } static HostDBInfo * @@ -522,4 +522,4 @@ void run_HostDBTest(); extern inkcoreapi HostDBProcessor hostDBProcessor; -void ink_hostdb_init(ModuleVersion version); +void ink_hostdb_init(ts::ModuleVersion version); diff --git a/iocore/hostdb/P_HostDB.h b/iocore/hostdb/P_HostDB.h index 0e7e281e3fa..c29d3c613e7 100644 --- a/iocore/hostdb/P_HostDB.h +++ b/iocore/hostdb/P_HostDB.h @@ -45,8 +45,8 @@ #include "P_RefCountCache.h" #include "P_HostDBProcessor.h" -#undef HOSTDB_MODULE_VERSION -#define HOSTDB_MODULE_VERSION makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) +static constexpr ts::ModuleVersion HOSTDB_MODULE_INTERNAL_VERSION{HOSTDB_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE}; + Ptr probe(ProxyMutex *mutex, CryptoHash const &hash, bool ignore_timeout); void make_crypto_hash(CryptoHash &hash, const char *hostname, int len, int port, const char *pDNSServers, HostDBMark mark); diff --git a/iocore/hostdb/P_RefCountCache.h b/iocore/hostdb/P_RefCountCache.h index f876897c033..5e1b7d6675f 100644 --- a/iocore/hostdb/P_RefCountCache.h +++ b/iocore/hostdb/P_RefCountCache.h @@ -37,8 +37,10 @@ #define REFCOUNT_CACHE_EVENT_SYNC REFCOUNT_CACHE_EVENT_EVENTS_START #define REFCOUNTCACHE_MAGIC_NUMBER 0x0BAD2D9 -#define REFCOUNTCACHE_MAJOR_VERSION 1 -#define REFCOUNTCACHE_MINOR_VERSION 0 + +static constexpr unsigned char REFCOUNTCACHE_MAJOR_VERSION = 1; +static constexpr unsigned char REFCOUNTCACHE_MINOR_VERSION = 0; +static constexpr ts::VersionNumber REFCOUNTCACHE_VERSION(1, 0); // Stats enum RefCountCache_Stats { @@ -367,10 +369,10 @@ class RefCountCacheHeader { public: unsigned int magic; - VersionNumber version; - VersionNumber object_version; // version passed in of whatever it is we are caching + ts::VersionNumber version{REFCOUNTCACHE_VERSION}; + ts::VersionNumber object_version; // version passed in of whatever it is we are caching - RefCountCacheHeader(VersionNumber object_version = VersionNumber()); + RefCountCacheHeader(ts::VersionNumber object_version = ts::VersionNumber()); bool operator==(const RefCountCacheHeader other) const; bool compatible(RefCountCacheHeader *other) const; }; @@ -393,7 +395,7 @@ template class RefCountCache { public: // Constructor - RefCountCache(unsigned int num_partitions, int size = -1, int items = -1, VersionNumber object_version = VersionNumber(), + RefCountCache(unsigned int num_partitions, int size = -1, int items = -1, ts::VersionNumber object_version = ts::VersionNumber(), std::string metrics_prefix = ""); // Destructor ~RefCountCache(); @@ -424,7 +426,7 @@ template class RefCountCache }; template -RefCountCache::RefCountCache(unsigned int num_partitions, int size, int items, VersionNumber object_version, +RefCountCache::RefCountCache(unsigned int num_partitions, int size, int items, ts::VersionNumber object_version, std::string metrics_prefix) : header(RefCountCacheHeader(object_version)), rsb(nullptr) { diff --git a/iocore/hostdb/RefCountCache.cc b/iocore/hostdb/RefCountCache.cc index 26abc75ea71..832a880c648 100644 --- a/iocore/hostdb/RefCountCache.cc +++ b/iocore/hostdb/RefCountCache.cc @@ -38,12 +38,8 @@ RefCountCacheHashEntry::dealloc(RefCountCacheHashEntry *e) return refCountCacheHashingValueAllocator.free(e); } -RefCountCacheHeader::RefCountCacheHeader(VersionNumber object_version) - : magic(REFCOUNTCACHE_MAGIC_NUMBER), object_version(object_version) -{ - this->version.ink_major = REFCOUNTCACHE_MAJOR_VERSION; - this->version.ink_minor = REFCOUNTCACHE_MINOR_VERSION; -}; +RefCountCacheHeader::RefCountCacheHeader(ts::VersionNumber object_version) + : magic(REFCOUNTCACHE_MAGIC_NUMBER), object_version(object_version){}; bool RefCountCacheHeader::operator==(const RefCountCacheHeader other) const @@ -54,6 +50,6 @@ RefCountCacheHeader::operator==(const RefCountCacheHeader other) const bool RefCountCacheHeader::compatible(RefCountCacheHeader *other) const { - return (this->magic == other->magic && this->version.ink_major == other->version.ink_major && - this->object_version.ink_major == other->version.ink_major); + return (this->magic == other->magic && this->version._major == other->version._major && + this->object_version._major == other->version._major); }; diff --git a/iocore/hostdb/test_I_HostDB.cc b/iocore/hostdb/test_I_HostDB.cc index a6d077eb4b1..fe507c73b1a 100644 --- a/iocore/hostdb/test_I_HostDB.cc +++ b/iocore/hostdb/test_I_HostDB.cc @@ -33,9 +33,9 @@ class HostDBTest : Continuation main() { init_diags("net_test", nullptr); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); - ink_net_init(NET_SYSTEM_MODULE_VERSION); - ink_hostdb_init(HOSTDB_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + ink_net_init(NET_SYSTEM_MODULE_PUBLIC_VERSION); + ink_hostdb_init(HOSTDB_MODULE_PUBLIC_VERSION); signal(SIGPIPE, SIG_IGN); eventProcessor.start(2); diff --git a/iocore/hostdb/test_RefCountCache.cc b/iocore/hostdb/test_RefCountCache.cc index b02ac604822..04df6c148ca 100644 --- a/iocore/hostdb/test_RefCountCache.cc +++ b/iocore/hostdb/test_RefCountCache.cc @@ -211,7 +211,7 @@ test() Layout::create(); init_diags("", nullptr); RecProcessInit(mode_type); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); int ret = 0; diff --git a/iocore/net/I_Net.h b/iocore/net/I_Net.h index 29cf0cb8a74..e4e82a64592 100644 --- a/iocore/net/I_Net.h +++ b/iocore/net/I_Net.h @@ -50,12 +50,9 @@ #define NET_MAX_IOV UIO_MAXIOV #endif -#define NET_SYSTEM_MODULE_MAJOR_VERSION 1 -#define NET_SYSTEM_MODULE_MINOR_VERSION 0 -#define NET_SYSTEM_MODULE_VERSION \ - makeModuleVersion(NET_SYSTEM_MODULE_MAJOR_VERSION, NET_SYSTEM_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion NET_SYSTEM_MODULE_PUBLIC_VERSION(1, 0, ts::ModuleVersion::PUBLIC); -static int const NO_FD = -1; +static constexpr int NO_FD = -1; // All in milli-seconds extern int net_config_poll_timeout; @@ -93,4 +90,4 @@ extern int net_throttle_delay; #include "I_NetProcessor.h" #include "I_SessionAccept.h" -void ink_net_init(ModuleVersion version); +void ink_net_init(ts::ModuleVersion version); diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc index ff6b0cb75f0..af1d1f75bcd 100644 --- a/iocore/net/Net.cc +++ b/iocore/net/Net.cc @@ -120,11 +120,12 @@ register_net_stats() } void -ink_net_init(ModuleVersion version) +ink_net_init(ts::ModuleVersion version) { static int init_called = 0; - ink_release_assert(!checkModuleVersion(version, NET_SYSTEM_MODULE_VERSION)); + ink_release_assert(version.check(NET_SYSTEM_MODULE_INTERNAL_VERSION)); + if (!init_called) { // do one time stuff // create a stat block for NetStats diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h index c59250be3b9..d611903b469 100644 --- a/iocore/net/P_Net.h +++ b/iocore/net/P_Net.h @@ -110,9 +110,7 @@ extern RecRawStatBlock *net_rsb; #include "P_SSLNetAccept.h" #include "P_SSLCertLookup.h" -#undef NET_SYSTEM_MODULE_VERSION -#define NET_SYSTEM_MODULE_VERSION \ - makeModuleVersion(NET_SYSTEM_MODULE_MAJOR_VERSION, NET_SYSTEM_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) +static constexpr ts::ModuleVersion NET_SYSTEM_MODULE_INTERNAL_VERSION(NET_SYSTEM_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE); // For very verbose iocore debugging. #ifndef DEBUG diff --git a/iocore/net/test_I_Net.cc b/iocore/net/test_I_Net.cc index d980292e5d4..9b6a79e18f9 100644 --- a/iocore/net/test_I_Net.cc +++ b/iocore/net/test_I_Net.cc @@ -43,8 +43,8 @@ main() init_diags("net_test", nullptr); RecProcessInit(mode_type); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); - ink_net_init(NET_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + ink_net_init(NET_SYSTEM_MODULE_PUBLIC_VERSION); /* * ignore broken pipe diff --git a/iocore/net/test_I_UDPNet.cc b/iocore/net/test_I_UDPNet.cc index 355a917b276..5ee79f0c4cc 100644 --- a/iocore/net/test_I_UDPNet.cc +++ b/iocore/net/test_I_UDPNet.cc @@ -135,7 +135,7 @@ udp_echo_server() net_config_poll_timeout = 10; init_diags("udp-.*", nullptr); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(2); udpNet.start(1, 1048576); diff --git a/iocore/net/test_P_Net.cc b/iocore/net/test_P_Net.cc index 248b8bdeebc..f56dd19d2c5 100644 --- a/iocore/net/test_P_Net.cc +++ b/iocore/net/test_P_Net.cc @@ -89,7 +89,7 @@ struct NetTesterAccept : public Continuation { int main() { - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); MIOBuffer *mbuf = new_MIOBuffer(5); eventProcessor.start(1); netProcessor.start(); diff --git a/src/traffic_cache_tool/CacheDefs.cc b/src/traffic_cache_tool/CacheDefs.cc index 820c5d10b89..a87e3c5aad3 100644 --- a/src/traffic_cache_tool/CacheDefs.cc +++ b/src/traffic_cache_tool/CacheDefs.cc @@ -237,9 +237,9 @@ Stripe::InitializeMeta() // memset(this->raw_dir, 0, dir_len); for (auto &i : _meta) { for (auto &j : i) { - j.magic = StripeMeta::MAGIC; - j.version.ink_major = ts::CACHE_DB_MAJOR_VERSION; - j.version.ink_minor = ts::CACHE_DB_MINOR_VERSION; + j.magic = StripeMeta::MAGIC; + j.version._major = ts::CACHE_DB_MAJOR_VERSION; + j.version._minor = ts::CACHE_DB_MINOR_VERSION; j.agg_pos = j.last_write_pos = j.write_pos = this->_content; j.phase = j.cycle = j.sync_serial = j.write_serial = j.dirty = 0; j.create_time = time(nullptr); @@ -264,8 +264,8 @@ bool Stripe::validateMeta(StripeMeta const *meta) { // Need to be bit more robust at some point. - return StripeMeta::MAGIC == meta->magic && meta->version.ink_major <= ts::CACHE_DB_MAJOR_VERSION && - meta->version.ink_minor <= 2 // This may have always been zero, actually. + return StripeMeta::MAGIC == meta->magic && meta->version._major <= ts::CACHE_DB_MAJOR_VERSION && + meta->version._minor <= 2 // This may have always been zero, actually. ; } diff --git a/src/traffic_cache_tool/CacheTool.cc b/src/traffic_cache_tool/CacheTool.cc index f16a1e86a6e..e20a76f4783 100644 --- a/src/traffic_cache_tool/CacheTool.cc +++ b/src/traffic_cache_tool/CacheTool.cc @@ -618,9 +618,8 @@ Cache::dumpSpans(SpanDumpDepth depth) for (int i = 0; i < 2; i++) { for (int j = 0; j < 2; j++) { std::cout << "\n" << MetaCopy[i] << ":" << MetaType[j] << "\n" << std::endl; - std::cout << " Magic:" << stripe->_meta[i][j].magic - << "\n version: ink_major: " << stripe->_meta[i][j].version.ink_major - << "\n version: ink_minor: " << stripe->_meta[i][j].version.ink_minor + std::cout << " Magic:" << stripe->_meta[i][j].magic << "\n version: major: " << stripe->_meta[i][j].version._major + << "\n version: minor: " << stripe->_meta[i][j].version._minor << "\n create_time: " << stripe->_meta[i][j].create_time << "\n write_pos: " << stripe->_meta[i][j].write_pos << "\n last_write_pos: " << stripe->_meta[i][j].last_write_pos diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 0eb340299b4..3080045a7a0 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -686,11 +686,11 @@ CB_After_Cache_Init() #if TS_ENABLE_FIPS == 0 // Check for cache BC after the cache is initialized and before listen, if possible. - if (cacheProcessor.min_stripe_version.ink_major < CACHE_DB_MAJOR_VERSION) { + if (cacheProcessor.min_stripe_version._major < CACHE_DB_MAJOR_VERSION) { // Versions before 23 need the MMH hash. - if (cacheProcessor.min_stripe_version.ink_major < 23) { + if (cacheProcessor.min_stripe_version._major < 23) { Debug("cache_bc", "Pre 4.0 stripe (cache version %d.%d) found, forcing MMH hash for cache URLs", - cacheProcessor.min_stripe_version.ink_major, cacheProcessor.min_stripe_version.ink_minor); + cacheProcessor.min_stripe_version._major, cacheProcessor.min_stripe_version._minor); URLHashContext::Setting = URLHashContext::MMH; } } @@ -1794,13 +1794,15 @@ main(int /* argc ATS_UNUSED */, const char **argv) REC_ReadConfigInteger(thread_max_heartbeat_mseconds, "proxy.config.thread.max_heartbeat_mseconds"); - ink_event_system_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); - ink_net_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); - ink_aio_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); - ink_cache_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); - ink_hostdb_init(makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER)); - ink_dns_init(makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER)); - ink_split_dns_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); + ink_event_system_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_net_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_aio_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_cache_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_hostdb_init( + ts::ModuleVersion(HOSTDB_MODULE_INTERNAL_VERSION._major, HOSTDB_MODULE_INTERNAL_VERSION._minor, ts::ModuleVersion::PRIVATE)); + ink_dns_init( + ts::ModuleVersion(HOSTDB_MODULE_INTERNAL_VERSION._major, HOSTDB_MODULE_INTERNAL_VERSION._minor, ts::ModuleVersion::PRIVATE)); + ink_split_dns_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); naVecMutex = new_ProxyMutex(); From c696743c1854ce3734ce88f855292e212b57795a Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Sun, 4 Nov 2018 22:26:54 +0000 Subject: [PATCH 004/526] Fix TLS hooks diagram and other docs cleanup --- .../hooks-and-transactions/ssl-hooks.en.rst | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst b/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst index 40f484d2792..902be513e59 100644 --- a/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst +++ b/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst @@ -123,19 +123,17 @@ TS_VCONN_OUTBOUND_CLOSE_HOOK This hook is invoked after the SSL handshake is done and right before the outbound connection closes. A callback at this point must reenable. -TLS Hook State Diagram ----------------------- +TLS Inbound Hook State Diagram +------------------------------ .. graphviz:: :alt: TLS Inbound Hook State Diagram digraph tls_hook_state_diagram{ HANDSHAKE_HOOKS_PRE -> TS_VCONN_START_HOOK; - HANDSHAKE_HOOKS_PRE -> TS_SSL_VERIFY_CLIENT_HOOK; HANDSHAKE_HOOKS_PRE -> TS_SSL_CERT_HOOK; HANDSHAKE_HOOKS_PRE -> TS_SSL_SERVERNAME_HOOK; HANDSHAKE_HOOKS_PRE -> HANDSHAKE_HOOKS_DONE; - TS_SSL_VERIFY_CLIENT_HOOK -> HANDSHAKE_HOOKS_PRE; TS_VCONN_START_HOOK -> HANDSHAKE_HOOKS_PRE_INVOKE; HANDSHAKE_HOOKS_PRE_INVOKE -> TSVConnReenable; TSVConnReenable -> HANDSHAKE_HOOKS_PRE; @@ -147,27 +145,46 @@ TLS Hook State Diagram TS_SSL_CERT_HOOK -> HANDSHAKE_HOOKS_CERT_INVOKE; HANDSHAKE_HOOKS_CERT_INVOKE -> TSVConnReenable2; TSVConnReenable2 -> HANDSHAKE_HOOKS_CERT; + + HANDSHAKE_HOOKS_CERT -> TS_SSL_VERIFY_CLIENT_HOOK; + HANDSHAKE_HOOKS_SNI -> TS_SSL_VERIFY_CLIENT_HOOK; + HANDSHAKE_HOOKS_PRE -> TS_SSL_VERIFY_CLIENT_HOOK; + TS_SSL_VERIFY_CLIENT_HOOK -> HANDSHAKE_HOOKS_VERIFY; + HANDSHAKE_HOOKS_VERIFY -> TS_SSL_VERIFY_CLIENT_HOOK; + HANDSHAKE_HOOKS_VERIFY -> HANDSHAKE_HOOKS_DONE; + HANDSHAKE_HOOKS_CERT -> HANDSHAKE_HOOKS_DONE; HANDSHAKE_HOOKS_DONE -> TS_VCONN_CLOSE_HOOK; + TS_VCONN_CLOSE_HOOK -> HANDSHAKE_HOOKS_DONE; HANDSHAKE_HOOKS_PRE [shape=box]; - TS_VCONN_START_HOOK [shape=box]; - TS_SSL_VERIFY_CLIENT_HOOK [shape=box]; HANDSHAKE_HOOKS_PRE_INVOKE [shape=box]; HANDSHAKE_HOOKS_SNI [shape=box]; + HANDSHAKE_HOOKS_VERIFY [shape=box]; HANDSHAKE_HOOKS_CERT [shape=box]; HANDSHAKE_HOOKS_CERT_INVOKE [shape=box]; HANDSHAKE_HOOKS_DONE [shape=box]; } +TLS Outbound Hook State Diagram +------------------------------- + .. graphviz:: :alt: TLS Outbound Hook State Diagram digraph tls_hook_state_diagram{ - HANDSHAKE_HOOKS_OUTBOUND_PRE -> HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE; - HANDSHAKE_HOOKS_PRE_INVOKE -> TSVConnReenable; + HANDSHAKE_HOOKS_OUTBOUND_PRE -> TS_VCONN_OUTBOUND_START_HOOK; + TS_VCONN_OUTBOUND_START_HOOK -> HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE; + HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE-> TSVConnReenable; TSVConnReenable -> HANDSHAKE_HOOKS_OUTBOUND_PRE; - HANDSHAKE_HOOKS_OUTBOUND_PRE -> HANDSHAKE_HOOKS_DONE; - HANDSHAKE_HOOKS_DONE -> HANDSHAKE_HOOKS_OUTBOUND_CLOSE; + HANDSHAKE_HOOKS_OUTBOUND_PRE -> TS_SSL_VERIFY_SERVER_HOOK; + TS_SSL_VERIFY_SERVER_HOOK -> HANDSHAKE_HOOKS_OUTBOUND_PRE; + HANDSHAKE_HOOKS_OUTBOUND_PRE -> TS_VCONN_OUTBOUND_CLOSE; + TS_VCONN_OUTBOUND_CLOSE -> HANDSHAKE_HOOKS_OUTBOUND_PRE; + TS_VCONN_OUTBOUND_CLOSE -> HANDSHAKE_HOOKS_DONE; + + HANDSHAKE_HOOKS_OUTBOUND_PRE [shape=box]; + HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE [shape=box]; + HANDSHAKE_HOOKS_DONE [shape=box]; } From 0bba409e3c88979b3273696fdf9ceb6f6a1e3a51 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 7 Nov 2018 12:31:31 -0600 Subject: [PATCH 005/526] MIME: Add string_view based overload for value_get. --- proxy/hdrs/MIME.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/proxy/hdrs/MIME.h b/proxy/hdrs/MIME.h index d95f147d9f8..59431a67394 100644 --- a/proxy/hdrs/MIME.h +++ b/proxy/hdrs/MIME.h @@ -24,6 +24,7 @@ #pragma once #include +#include #include "tscore/ink_assert.h" #include "tscore/ink_apidefs.h" @@ -952,6 +953,7 @@ class MIMEHdr : public HdrHeapSDKHandle int value_get_index(const char *name, int name_length, const char *value, int value_length) const; const char *value_get(const char *name, int name_length, int *value_length) const; + std::string_view value_get(std::string_view const &name) const; // Convenience overload. int32_t value_get_int(const char *name, int name_length) const; uint32_t value_get_uint(const char *name, int name_length) const; int64_t value_get_int64(const char *name, int name_length) const; @@ -1260,6 +1262,15 @@ MIMEHdr::value_get(const char *name, int name_length, int *value_length_return) return (nullptr); } +inline std::string_view +MIMEHdr::value_get(std::string_view const &name) const +{ + if (MIMEField const *field = field_find(name.data(), name.size()); field) { + return {field->m_ptr_value, field->m_len_value}; + } + return {}; +} + inline int32_t MIMEHdr::value_get_int(const char *name, int name_length) const { From 3a5763b1c0f649ec9fb2fe90c8c740b28ce0874e Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 5 Nov 2018 16:52:28 -0800 Subject: [PATCH 006/526] Adjusts the format for header_rewrite string concatenation Drop the "+" from the syntax. This syntax was never released and therefore doesn't have backwards compatibilty requirements. eg add-header X-Party "let's party like it's %{NOW:YEAR}!" (quotes optional) --- doc/admin-guide/plugins/header_rewrite.en.rst | 7 +- plugins/header_rewrite/header_rewrite.cc | 4 +- plugins/header_rewrite/header_rewrite_test.cc | 91 ++++++++++++++++++- plugins/header_rewrite/parser.cc | 71 ++++++++++++--- plugins/header_rewrite/parser.h | 16 +++- plugins/header_rewrite/regex_helper.cc | 12 --- plugins/header_rewrite/regex_helper.h | 2 - plugins/header_rewrite/value.cc | 30 +++--- 8 files changed, 182 insertions(+), 51 deletions(-) diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index 0ee9d049950..9b38438b804 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -828,14 +828,13 @@ Variable New condition variable to use The % tags can now be replaced with the %{INBOUND:...} equivalent. -You can concatenate values using strings, condition values and variable expansions via the ``+`` operator. -Variables (eg %) in the concatenation must be enclosed in double quotes ``"``:: +You can concatenate values using strings, condition values and variable expansions on the same line. - add-header CustomHeader "Hello from " + %{IP:SERVER} + ":" + "%" + add-header CustomHeader "Hello from %{IP:SERVER}:%" However, the above example is somewhat contrived to show the old tags, it should instead be written as - add-header CustomHeader "Hello from " + %{IP:SERVER} + ":" + %{INBOUND:LOCAL-PORT} + add-header CustomHeader "Hello from %{IP:SERVER}:%{INBOUND:LOCAL-PORT}" Concatenation is not supported in condition testing. diff --git a/plugins/header_rewrite/header_rewrite.cc b/plugins/header_rewrite/header_rewrite.cc index af19305e48f..60caecda9bb 100644 --- a/plugins/header_rewrite/header_rewrite.cc +++ b/plugins/header_rewrite/header_rewrite.cc @@ -184,7 +184,7 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook) continue; } - Parser p(line, true); // Tokenize and parse this line preserving quotes from input + Parser p(line); // Tokenize and parse this line if (p.empty()) { continue; } @@ -470,6 +470,6 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) rule = rule->next; } - TSDebug(PLUGIN_NAME_DBG, "Returing from TSRemapDoRemap with status: %d", rval); + TSDebug(PLUGIN_NAME_DBG, "Returning from TSRemapDoRemap with status: %d", rval); return rval; } diff --git a/plugins/header_rewrite/header_rewrite_test.cc b/plugins/header_rewrite/header_rewrite_test.cc index c81113a1e61..6a4e18172f2 100644 --- a/plugins/header_rewrite/header_rewrite_test.cc +++ b/plugins/header_rewrite/header_rewrite_test.cc @@ -33,6 +33,12 @@ const char PLUGIN_NAME_DBG[] = "TEST_dbg_header_rewrite"; extern "C" void TSError(const char *fmt, ...) { + va_list args; + + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); } class ParserTest : public Parser @@ -40,7 +46,7 @@ class ParserTest : public Parser public: ParserTest(const std::string &line) : Parser(line), res(true) { std::cout << "Finished parser test: " << line << std::endl; } std::vector - getTokens() + getTokens() const { return _tokens; } @@ -58,17 +64,38 @@ class ParserTest : public Parser bool res; }; +class SimpleTokenizerTest : public SimpleTokenizer +{ +public: + SimpleTokenizerTest(const std::string &line) : SimpleTokenizer(line), res(true) + { + std::cout << "Finished tokenizer test: " << line << std::endl; + } + + template + void + do_parser_check(T x, U y, int line = 0) + { + if (x != y) { + std::cerr << "CHECK FAILED on line " << line << ": |" << x << "| != |" << y << "|" << std::endl; + res = false; + } + } + + bool res; +}; + #define CHECK_EQ(x, y) \ do { \ p.do_parser_check((x), (y), __LINE__); \ - } while (false); + } while (false) #define END_TEST(s) \ do { \ if (!p.res) { \ ++errors; \ } \ - } while (false); + } while (false) int test_parsing() @@ -336,6 +363,17 @@ test_parsing() END_TEST(); } + { + ParserTest p(R"(set-header Alt-Svc "quic=\":443\"; v=\"35\"")"); + + CHECK_EQ(p.getTokens().size(), 3UL); + CHECK_EQ(p.getTokens()[0], "set-header"); + CHECK_EQ(p.getTokens()[1], "Alt-Svc"); + CHECK_EQ(p.getTokens()[2], R"(quic=":443"; v="35")"); + + END_TEST(); + } + /* * test some failure scenarios */ @@ -425,10 +463,55 @@ test_processing() return errors; } +int +test_tokenizer() +{ + int errors = 0; + + { + SimpleTokenizerTest p("a simple test"); + CHECK_EQ(p.get_tokens().size(), 1UL); + CHECK_EQ(p.get_tokens()[0], "a simple test"); + } + + { + SimpleTokenizerTest p(R"(quic=":443"; v="35")"); + CHECK_EQ(p.get_tokens().size(), 1UL); + CHECK_EQ(p.get_tokens()[0], R"(quic=":443"; v="35")"); + } + + { + SimpleTokenizerTest p(R"(let's party like it's %{NOW:YEAR})"); + CHECK_EQ(p.get_tokens().size(), 2UL); + CHECK_EQ(p.get_tokens()[0], "let's party like it's "); + CHECK_EQ(p.get_tokens()[1], "%{NOW:YEAR}"); + } + { + SimpleTokenizerTest p("A racoon's favorite tag is % in %{NOW:YEAR}!"); + CHECK_EQ(p.get_tokens().size(), 5UL); + CHECK_EQ(p.get_tokens()[0], "A racoon's favorite tag is "); + CHECK_EQ(p.get_tokens()[1], "%"); + CHECK_EQ(p.get_tokens()[2], " in "); + CHECK_EQ(p.get_tokens()[3], "%{NOW:YEAR}"); + CHECK_EQ(p.get_tokens()[4], "!"); + } + + { + SimpleTokenizerTest p(R"(Hello from %{IP:SERVER}:%{INBOUND:LOCAL-PORT})"); + CHECK_EQ(p.get_tokens().size(), 4UL); + CHECK_EQ(p.get_tokens()[0], "Hello from "); + CHECK_EQ(p.get_tokens()[1], "%{IP:SERVER}"); + CHECK_EQ(p.get_tokens()[2], ":"); + CHECK_EQ(p.get_tokens()[3], "%{INBOUND:LOCAL-PORT}"); + } + + return errors; +} + int main() { - if (test_parsing() || test_processing()) { + if (test_parsing() || test_processing() || test_tokenizer()) { return 1; } diff --git a/plugins/header_rewrite/parser.cc b/plugins/header_rewrite/parser.cc index fc821d3483c..4fc1bf40671 100644 --- a/plugins/header_rewrite/parser.cc +++ b/plugins/header_rewrite/parser.cc @@ -28,9 +28,9 @@ #include "parser.h" -enum ParserState { PARSER_DEFAULT, PARSER_IN_QUOTE, PARSER_IN_REGEX }; +enum ParserState { PARSER_DEFAULT, PARSER_IN_QUOTE, PARSER_IN_REGEX, PARSER_IN_EXPANSION }; -Parser::Parser(const std::string &original_line, bool preserve_quotes) : _cond(false), _empty(false) +Parser::Parser(const std::string &original_line) : _cond(false), _empty(false) { std::string line = original_line; ParserState state = PARSER_DEFAULT; @@ -39,8 +39,7 @@ Parser::Parser(const std::string &original_line, bool preserve_quotes) : _cond(f size_t cur_token_length = 0; for (size_t i = 0; i < line.size(); ++i) { - if ((state == PARSER_DEFAULT) && - (std::isspace(line[i]) || ((line[i] == '=') || (line[i] == '>') || (line[i] == '<') || (line[i] == '+')))) { + if ((state == PARSER_DEFAULT) && (std::isspace(line[i]) || ((line[i] == '=')))) { if (extracting_token) { cur_token_length = i - cur_token_start; if (cur_token_length > 0) { @@ -78,11 +77,7 @@ Parser::Parser(const std::string &original_line, bool preserve_quotes) : _cond(f cur_token_start = i + 1; // Eat the leading quote } else if ((state == PARSER_IN_QUOTE) && extracting_token) { cur_token_length = i - cur_token_start; - if (preserve_quotes) { - _tokens.push_back(line.substr(cur_token_start - 1, i - cur_token_start + 2)); - } else { - _tokens.push_back(line.substr(cur_token_start, cur_token_length)); - } + _tokens.push_back(line.substr(cur_token_start, cur_token_length)); state = PARSER_DEFAULT; extracting_token = false; } else { @@ -99,7 +94,7 @@ Parser::Parser(const std::string &original_line, bool preserve_quotes) : _cond(f break; } - if ((line[i] == '=') || (line[i] == '>') || (line[i] == '<') || (line[i] == '+')) { + if ((line[i] == '=') || (line[i] == '+')) { // These are always a seperate token _tokens.push_back(std::string(1, line[i])); continue; @@ -202,7 +197,7 @@ Parser::preprocess(std::vector tokens) } } else { // Syntax error - TSError("[%s] mods have to be embraced in []", PLUGIN_NAME); + TSError("[%s] mods have to be enclosed in []", PLUGIN_NAME); return; } } @@ -251,3 +246,57 @@ Parser::get_tokens() const { return _tokens; } + +SimpleTokenizer::SimpleTokenizer(const std::string &original_line) +{ + std::string line = original_line; + ParserState state = PARSER_DEFAULT; + bool extracting_token = false; + off_t cur_token_start = 0; + size_t cur_token_length = 0; + + for (size_t i = 0; i < line.size(); ++i) { + extracting_token = true; + switch (state) { + case PARSER_DEFAULT: + if ((line[i] == '{') || (line[i] == '<')) { + if (line[i - 1] == '%') { + // pickup what we currently have + cur_token_length = i - cur_token_start - 1; + if (cur_token_length > 0) { + _tokens.push_back(line.substr(cur_token_start, cur_token_length)); + } + + cur_token_start = i - 1; + state = PARSER_IN_EXPANSION; + extracting_token = false; + } + } + break; + case PARSER_IN_EXPANSION: + if ((line[i] == '}') || (line[i] == '>')) { + cur_token_length = i - cur_token_start + 1; + if (cur_token_length > 0) { + _tokens.push_back(line.substr(cur_token_start, cur_token_length)); + } + cur_token_start = i + 1; + state = PARSER_DEFAULT; + extracting_token = false; + } + break; + default: + break; + } + } + + // take what was left behind + if (extracting_token) { + _tokens.push_back(line.substr(cur_token_start)); + } +} + +const std::vector & +SimpleTokenizer::get_tokens() const +{ + return _tokens; +} diff --git a/plugins/header_rewrite/parser.h b/plugins/header_rewrite/parser.h index d82203d8b7b..d6f4db09d06 100644 --- a/plugins/header_rewrite/parser.h +++ b/plugins/header_rewrite/parser.h @@ -33,7 +33,7 @@ class Parser { public: - explicit Parser(const std::string &line, bool preserve_quotes = false); + explicit Parser(const std::string &line); bool empty() const @@ -88,3 +88,17 @@ class Parser protected: std::vector _tokens; }; + +class SimpleTokenizer +{ +public: + explicit SimpleTokenizer(const std::string &line); + + const std::vector &get_tokens() const; + +private: + DISALLOW_COPY_AND_ASSIGN(SimpleTokenizer); + +protected: + std::vector _tokens; +}; diff --git a/plugins/header_rewrite/regex_helper.cc b/plugins/header_rewrite/regex_helper.cc index c07d117081b..8b30361123e 100644 --- a/plugins/header_rewrite/regex_helper.cc +++ b/plugins/header_rewrite/regex_helper.cc @@ -40,18 +40,6 @@ regexHelper::setRegexMatch(const std::string &s) return true; } -const std::string & -regexHelper::getRegexString() const -{ - return regexString; -} - -int -regexHelper::getRegexCcount() const -{ - return regexCcount; -} - int regexHelper::regexMatch(const char *str, int len, int ovector[]) const { diff --git a/plugins/header_rewrite/regex_helper.h b/plugins/header_rewrite/regex_helper.h index 8f812181805..d156fee730a 100644 --- a/plugins/header_rewrite/regex_helper.h +++ b/plugins/header_rewrite/regex_helper.h @@ -39,8 +39,6 @@ class regexHelper } bool setRegexMatch(const std::string &s); - const std::string &getRegexString() const; - int getRegexCcount() const; int regexMatch(const char *, int, int ovector[]) const; private: diff --git a/plugins/header_rewrite/value.cc b/plugins/header_rewrite/value.cc index ef15e43ffc6..bd664c63cee 100644 --- a/plugins/header_rewrite/value.cc +++ b/plugins/header_rewrite/value.cc @@ -43,26 +43,26 @@ Value::set_value(const std::string &val) { _value = val; - if (_value.find("%{") != std::string::npos || _value.find("%<") != std::string::npos || _value.find("\"") != std::string::npos) { - Parser parser(_value); - auto tokens = parser.get_tokens(); + if (_value.find("%{") != std::string::npos || _value.find("%<") != std::string::npos) { + SimpleTokenizer tokenizer(_value); + + auto tokens = tokenizer.get_tokens(); for (auto it = tokens.begin(); it != tokens.end(); it++) { - Parser tparser(*it); + std::string token = *it; Condition *tcond_val = nullptr; - if ((*it).substr(0, 2) == "%<") { + if (token.substr(0, 2) == "%<") { tcond_val = new ConditionExpandableString(*it); - } else if ((*it) == "+") { - // Skip concat token - continue; - } else { - tcond_val = condition_factory(tparser.get_op()); + } else if (token.substr(0, 2) == "%{") { + std::string cond_token = token.substr(2, token.size() - 3); + tcond_val = condition_factory(cond_token); + } - if (tcond_val) { - tcond_val->initialize(tparser); - } else { - tcond_val = new ConditionStringLiteral(*it); - } + if (tcond_val) { + Parser parser(_value); + tcond_val->initialize(parser); + } else { + tcond_val = new ConditionStringLiteral(token); } _cond_vals.push_back(tcond_val); } From fa3ae700721912128b68019b06fd1d5dc9d9ab63 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Fri, 2 Nov 2018 22:25:38 +0000 Subject: [PATCH 007/526] Fix tunnel_route action and fedora/openssl timing changes --- iocore/net/P_SSLUtils.h | 4 +- iocore/net/SSLNetVConnection.cc | 14 ++- iocore/net/SSLUtils.cc | 10 ++ tests/bootstrap.py | 2 +- tests/gold_tests/autest-site/init.cli.ext | 2 +- tests/gold_tests/cache/cache-control.test.py | 1 + .../cache/gold/cache_and_req_body-hit.gold | 1 - .../cache/gold/cache_and_req_body-miss.gold | 1 - .../cache/gold/cache_hit_stale.gold | 1 - tests/gold_tests/cache/gold/cache_no_cc.gold | 1 - tests/gold_tests/tls/tls_client_cert.test.py | 4 +- tests/gold_tests/tls/tls_hooks_verify.test.py | 4 +- tests/gold_tests/tls/tls_tunnel.test.py | 95 +++++++++++++++++++ tests/gold_tests/tls/tls_verify.test.py | 8 +- tests/gold_tests/tls/tls_verify2.test.py | 8 +- 15 files changed, 127 insertions(+), 29 deletions(-) create mode 100644 tests/gold_tests/tls/tls_tunnel.test.py diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index b165fd6b82f..f70072b8047 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -248,7 +248,7 @@ class TunnelHashMap struct HostStruct { std::string hostname; int port; - HostStruct(const std::string &name, int port_) : hostname(name), port(port_) {} + HostStruct(const std::string_view &name, int port_) : hostname(name), port(port_) {} }; using Tunnel_hashMap = std::unordered_map; Tunnel_hashMap TunnelhMap; @@ -258,7 +258,7 @@ class TunnelHashMap { std::string_view addr, port; if (ats_ip_parse(std::string_view(hostname), &addr, &port) == 0) { - TunnelhMap.emplace(key, HostStruct(addr.data(), atoi(port.data()))); + TunnelhMap.emplace(key, HostStruct(addr, atoi(port.data()))); } } diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 49732633012..8790883bf58 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -361,6 +361,8 @@ SSLNetVConnection::read_raw_data() NET_INCREMENT_DYN_STAT(net_calls_to_read_stat); total_read += rattempted; + Debug("ssl", "read_raw_data r=%" PRId64 " rattempted=%" PRId64 " total_read=%" PRId64 " fd=%d", r, rattempted, total_read, + con.fd); // last read failed or was incomplete if (r != rattempted || !b) { break; @@ -368,7 +370,6 @@ SSLNetVConnection::read_raw_data() rattempted = b->write_avail(); } - // If we have already moved some bytes successfully, adjust total_read to reflect reality // If any read succeeded, we should return success if (r != rattempted) { @@ -567,6 +568,7 @@ SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) nh->write_ready_list.remove(this); writeReschedule(nh); } else if (ret == EVENT_DONE) { + Debug("ssl", "ssl handshake EVENT_DONE ntodo=%" PRId64, ntodo); // If this was driven by a zero length read, signal complete when // the handshake is complete. Otherwise set up for continuing read // operations. @@ -1423,18 +1425,17 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) unsigned long e = ERR_peek_last_error(); ERR_error_string_n(e, buf, sizeof(buf)); // FIXME -- This triggers a retry on cases of cert validation errors.... - Debug("ssl", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL"); SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL"); TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, e, buf, errno); if (e) { if (this->options.sni_servername) { - Error("SSL connection failed for '%s': %s", this->options.sni_servername.get(), buf); + Debug("ssl.error", "SSL connection failed for '%s': %s", this->options.sni_servername.get(), buf); } else { char buff[INET6_ADDRSTRLEN]; ats_ip_ntop(this->get_remote_addr(), buff, INET6_ADDRSTRLEN); - Error("SSL connection failed for '%s': %s", buff, buf); + Debug("ssl.error", "SSL connection failed for '%s': %s", buff, buf); } } return EVENT_ERROR; @@ -1614,7 +1615,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) // Only dealing with the SNI/CERT hook so far. ink_assert(eventId == TS_EVENT_SSL_CERT || eventId == TS_EVENT_SSL_SERVERNAME || eventId == TS_EVENT_SSL_VERIFY_SERVER || eventId == TS_EVENT_SSL_VERIFY_CLIENT || eventId == TS_EVENT_VCONN_CLOSE || eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE); - Debug("ssl", "callHooks sslHandshakeHookState=%d", this->sslHandshakeHookState); + Debug("ssl", "callHooks sslHandshakeHookState=%d eventID=%d", this->sslHandshakeHookState, eventId); // Move state if it is appropriate switch (this->sslHandshakeHookState) { @@ -1631,6 +1632,9 @@ SSLNetVConnection::callHooks(TSEvent eventId) case HANDSHAKE_HOOKS_SNI: if (eventId == TS_EVENT_SSL_CERT) { this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; + } else if (eventId == TS_EVENT_VCONN_CLOSE) { + // Jump to the end + this->sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; } break; default: diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 9a528381653..7a0b579d220 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -420,6 +420,11 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) bool reenabled; int retval = 1; + // If we are in tunnel mode, don't select a cert. Pause! + if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == netvc->attributes) { + return -1; // Pause + } + // Do the common certificate lookup only once. If we pause // and restart processing, do not execute the common logic again if (!netvc->calledHooks(TS_EVENT_SSL_CERT)) { @@ -470,6 +475,11 @@ ssl_servername_and_cert_callback(SSL *ssl, int * /* ad */, void * /*arg*/) bool reenabled; int retval = 1; + // If we are in tunnel mode, don't select a cert. Pause! + if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == netvc->attributes) { + return -1; // Pause + } + // Do the common certificate lookup only once. If we pause // and restart processing, do not execute the common logic again if (!netvc->calledHooks(TS_EVENT_SSL_CERT)) { diff --git a/tests/bootstrap.py b/tests/bootstrap.py index 4438b287603..de9b04e3e61 100755 --- a/tests/bootstrap.py +++ b/tests/bootstrap.py @@ -26,7 +26,7 @@ import sys pip_packages = [ - "autest==1.7.0", + "autest==1.7.2", "hyper", "requests", "dnslib", diff --git a/tests/gold_tests/autest-site/init.cli.ext b/tests/gold_tests/autest-site/init.cli.ext index 28ed5748e30..6d7e89b13e2 100644 --- a/tests/gold_tests/autest-site/init.cli.ext +++ b/tests/gold_tests/autest-site/init.cli.ext @@ -23,7 +23,7 @@ if sys.version_info < (3, 5, 0): host.WriteError( "You need python 3.5 or later to run these tests\n", show_stack=False) -autest_version ="1.7.0" +autest_version ="1.7.2" if AuTestVersion() < autest_version: host.WriteError( "Tests need AuTest version {ver} or better\n Please update AuTest:\n pip install --upgrade autest\n".format(ver=autest_version), show_stack=False) diff --git a/tests/gold_tests/cache/cache-control.test.py b/tests/gold_tests/cache/cache-control.test.py index ed0f24ecbae..e2c9fa4f534 100644 --- a/tests/gold_tests/cache/cache-control.test.py +++ b/tests/gold_tests/cache/cache-control.test.py @@ -53,6 +53,7 @@ 'proxy.config.http.response_via_str': 3, 'proxy.config.http.cache.http': 1, 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, }) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/cache/gold/cache_and_req_body-hit.gold b/tests/gold_tests/cache/gold/cache_and_req_body-hit.gold index c10c0ba0705..34bb179c50b 100644 --- a/tests/gold_tests/cache/gold/cache_and_req_body-hit.gold +++ b/tests/gold_tests/cache/gold/cache_and_req_body-hit.gold @@ -2,7 +2,6 @@ HTTP/1.1 200 OK Cache-Control: max-age=10,public Content-Length: 11 Date: `` -Age: `` Connection: keep-alive Via: `` Server: `` diff --git a/tests/gold_tests/cache/gold/cache_and_req_body-miss.gold b/tests/gold_tests/cache/gold/cache_and_req_body-miss.gold index 0f8938e1c50..33efb4bac07 100644 --- a/tests/gold_tests/cache/gold/cache_and_req_body-miss.gold +++ b/tests/gold_tests/cache/gold/cache_and_req_body-miss.gold @@ -2,7 +2,6 @@ HTTP/1.1 200 OK Cache-Control: max-age=10,public Content-Length: 11 Date: `` -Age: `` Connection: keep-alive Via: `` Server: `` diff --git a/tests/gold_tests/cache/gold/cache_hit_stale.gold b/tests/gold_tests/cache/gold/cache_hit_stale.gold index 094162645e5..25a3d6670ca 100644 --- a/tests/gold_tests/cache/gold/cache_hit_stale.gold +++ b/tests/gold_tests/cache/gold/cache_hit_stale.gold @@ -2,7 +2,6 @@ HTTP/1.1 200 OK Cache-Control: max-age=10,public Content-Length: 11 Date: `` -Age: 0 Connection: keep-alive Via: `` Server: `` diff --git a/tests/gold_tests/cache/gold/cache_no_cc.gold b/tests/gold_tests/cache/gold/cache_no_cc.gold index 9bca02d5f45..7f50b005e62 100644 --- a/tests/gold_tests/cache/gold/cache_no_cc.gold +++ b/tests/gold_tests/cache/gold/cache_no_cc.gold @@ -1,7 +1,6 @@ HTTP/1.1 200 OK Content-Length: 14 Date: `` -Age: 0 Connection: keep-alive Via: `` Server: `` diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index 1973031224d..e48bbe2621a 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -65,7 +65,7 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl|http', + 'proxy.config.diags.debug.tags': 'ssl_verify_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), @@ -310,5 +310,3 @@ tr4fail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") tr4fail.TimeOut = 5 -# Allow for error messages in diags -ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", "Some connections should fail") diff --git a/tests/gold_tests/tls/tls_hooks_verify.test.py b/tests/gold_tests/tls/tls_hooks_verify.test.py index 88fd03737a9..c3d694d221d 100644 --- a/tests/gold_tests/tls/tls_hooks_verify.test.py +++ b/tests/gold_tests/tls/tls_hooks_verify.test.py @@ -95,10 +95,8 @@ tr3.Processes.Default.ReturnCode = 0 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have failed") -# Over riding the built in ERROR check since we expect tr2 to fail -ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Terminate SNI=random.com", "random.com should fail") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Terminate SNI=random.com", "random.com should fail") ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Continue SNI=bar.com", "bar.com should fail but continue") -ts.Disk.diags_log.Content += Testers.ContainsExpression("ERROR: SSL connection failed for 'random.com': .+?:certificate verify failed", "random.com should really fail") ts.Disk.diags_log.Content += Testers.ExcludesExpression("SNI=foo.com", "foo.com should not fail in any way") ts.Streams.All += Testers.ContainsExpression("Server verify callback 0 [\da-fx]+? - event is good SNI=foo.com good HS", "verify callback happens 2 times") diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py new file mode 100644 index 00000000000..390a11e6bb9 --- /dev/null +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -0,0 +1,95 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tunneling based on SNI +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True) +server_bar = Test.MakeOriginServer("server_bar", ssl=False) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_foo.Variables.Port,server_bar.Variables.Port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# foo.com should not terminate. Just tunnel to server_foo +# bar.com should terminate. Forward its tcp stream to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: foo.com', + " tunnel_route: localhost:{0}".format(server_foo.Variables.Port) ]) + +tr = Test.AddTestRun("Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") + + diff --git a/tests/gold_tests/tls/tls_verify.test.py b/tests/gold_tests/tls/tls_verify.test.py index 3a33e9eeb96..d95ec160143 100644 --- a/tests/gold_tests/tls/tls_verify.test.py +++ b/tests/gold_tests/tls/tls_verify.test.py @@ -71,7 +71,7 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.enabled': 0, 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), @@ -104,7 +104,7 @@ tr.Setup.Copy("ssl/signed-foo.pem") tr.Setup.Copy("ssl/signed-bar.key") tr.Setup.Copy("ssl/signed-bar.pem") -tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v -k -H \"host: foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server_foo) @@ -118,7 +118,7 @@ tr.TimeOut = 5 tr2 = Test.AddTestRun("Override-enforcing-Test") -tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.Processes.Default.TimeOut = 5 @@ -127,7 +127,7 @@ tr2.TimeOut = 5 tr3 = Test.AddTestRun("Override-enforcing-Test-fail-name-check") -tr3.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr3.Processes.Default.Command = "curl -v -k -H \"host: bad_bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr3.ReturnCode = 0 tr3.StillRunningAfter = server diff --git a/tests/gold_tests/tls/tls_verify2.test.py b/tests/gold_tests/tls/tls_verify2.test.py index 5958c76c7ba..5e4f2b5e21c 100644 --- a/tests/gold_tests/tls/tls_verify2.test.py +++ b/tests/gold_tests/tls/tls_verify2.test.py @@ -71,7 +71,7 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.enabled': 0, 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), @@ -162,14 +162,10 @@ tr6.Processes.Default.TimeOut = 5 -# Over riding the built in ERROR check since we expect tr4 and tr7 to fail # No name checking for the sig-only permissive override for bad_bar -ts.Disk.diags_log.Content = Testers.ExcludesExpression("WARNING: SNI \(bad_bar.com\) not in certificate", "bad_bar name checked should be skipped.") +ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI \(bad_bar.com\) not in certificate", "bad_bar name checked should be skipped.") ts.Disk.diags_log.Content = Testers.ExcludesExpression("WARNING: SNI \(foo.com\) not in certificate", "foo name checked should be skipped.") # No checking for the self-signed on random.com. No messages ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: Core server certificate verification failed for \(random.com\)", "signature check for random.com should be skipped") -ts.Disk.diags_log.Content += Testers.ExcludesExpression("ERROR: SSL connection failed for 'random.com'", "random.com should not fail") -ts.Disk.diags_log.Content += Testers.ContainsExpression("ERROR: SSL connection failed for 'random2.com'", "random2.com should fail") ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random2.com\)", "signature check for random.com should fail'") ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bad_foo.com\) not in certificate", "bad_foo name checked should be checked.") -ts.Disk.diags_log.Content += Testers.ContainsExpression("ERROR: SSL connection failed for 'bad_foo.com'", "bad_foo.com should fail") From 458fdb634211801a59de27d284c729c2f84a6e50 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 7 Nov 2018 14:26:32 -0800 Subject: [PATCH 008/526] Allows the use of certs with no commonName, but with subjectAltNames --- iocore/net/SSLUtils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 7a0b579d220..bceae8b1cfb 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -1496,7 +1496,7 @@ ssl_index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cer if (name->type == GEN_DNS) { ats_scoped_str dns(asn1_strdup(name->d.dNSName)); // only try to insert if the alternate name is not the main name - if (strcmp(dns, subj_name) != 0) { + if (subj_name == nullptr || strcmp(dns, subj_name) != 0) { Debug("ssl", "mapping '%s' to certificates %s", (const char *)dns, certname); if (lookup->insert(dns, cc) >= 0) { inserted = true; From f6e3db952fb1e4dde49fb68d6a8bc8e36e04611f Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Wed, 7 Nov 2018 20:45:48 -0800 Subject: [PATCH 009/526] Doc: change code snippet highlighting scheme The lime green background for code snippets doesn't go with the light blue background. The 'default' code highlighting scheme uses a moderate gray background, but similar font colors for syntax highlighting. --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 11884b6f4f1..54e2ceec6a8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -161,7 +161,7 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'default' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] From b7b87850d9e39572741487dcdd3963c54bed45a9 Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Wed, 7 Nov 2018 14:14:54 -0800 Subject: [PATCH 010/526] access_control: reduced some log errs to debug Reduced some errors to debug if they can happen during normal operation and added some extra debugging info for the unexpected ones. --- plugins/experimental/access_control/plugin.cc | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/plugins/experimental/access_control/plugin.cc b/plugins/experimental/access_control/plugin.cc index bcc68f9c763..5f003e39d41 100644 --- a/plugins/experimental/access_control/plugin.cc +++ b/plugins/experimental/access_control/plugin.cc @@ -32,6 +32,8 @@ #include "utils.h" /* cryptoBase64Decode.* functions */ #include "headers.h" /* getHeader, setHeader, removeHeader */ +static const std::string_view UNKNOWN{"unknown"}; + static const char * getEventName(TSEvent event) { @@ -415,12 +417,18 @@ contHandleAccessControl(const TSCont contp, TSEvent event, void *edata) TSHandleMLocRelease(serverRespBufp, TS_NULL_MLOC, serverRespHdrLoc); } else { - AccessControlError("failed to retrieve server response header"); + int len; + char *url = TSHttpTxnEffectiveUrlStringGet(txnp, &len); + AccessControlError("failed to retrieve server response header for request url:%.*s", + (len ? len : static_cast(UNKNOWN.size())), (url ? url : UNKNOWN.data())); } TSHandleMLocRelease(clientRespBufp, TS_NULL_MLOC, clientRespHdrLoc); } else { - AccessControlError("failed to retrieve client response header"); + int len; + char *url = TSHttpTxnEffectiveUrlStringGet(txnp, &len); + AccessControlError("failed to retrieve client response header for request url:%.*s", + (len ? len : static_cast(UNKNOWN.size())), (url ? url : UNKNOWN.data())); } } } break; @@ -583,7 +591,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txnp, TSRemapRequestInfo *rri) String pattern; if (config->_uriPathScope.empty()) { /* Scope match enforce access control */ - AccessControlError("no plugin scope specified, enforcing access control"); + AccessControlDebug("no plugin scope specified, enforcing access control"); remapStatus = enforceAccessControl(txnp, rri, config); } else { if (true == config->_uriPathScope.matchAll(reqPath, filename, pattern)) { @@ -592,13 +600,13 @@ TSRemapDoRemap(void *instance, TSHttpTxn txnp, TSRemapRequestInfo *rri) /* Scope match enforce access control */ remapStatus = enforceAccessControl(txnp, rri, config); } else { - AccessControlError("not matching plugin scope (file: %s, pattern %s), skipping access control for path '%s'", + AccessControlDebug("not matching plugin scope (file: %s, pattern %s), skipping access control for path '%s'", filename.c_str(), pattern.c_str(), reqPath.c_str()); } } } else { TSHttpTxnStatusSet(txnp, config->_invalidRequest); - AccessControlError("https is the only allowed scheme (plugin should be used only with TLS)"); + AccessControlDebug("https is the only allowed scheme (plugin should be used only with TLS)"); remapStatus = TSREMAP_DID_REMAP; } } else { From 245c0113ee6c8c806a4127d358fef016380f230a Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Thu, 8 Nov 2018 20:41:00 -0800 Subject: [PATCH 011/526] Doc: Add missing overrides and sort #4202 Added TS_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_CHARSET_MISMATCH TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_ENCODING_MISMATCH TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_LANGUAGE_MISMATCH TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_IMAGES TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_OTHER TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_TEXT TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED TS_CONFIG_SSL_CLIENT_VERIFY_SERVER and sorted the list alphabetically --- .../api/functions/TSHttpOverridableConfig.en.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index 125e1111706..c99f3836b8d 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -78,6 +78,7 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_BACKGROUND_FILL_COMPLETED_THRESHOLD` :ts:cv:`proxy.config.http.background_fill_completed_threshold` :c:macro:`TS_CONFIG_HTTP_CACHE_CACHE_RESPONSES_TO_COOKIES` :ts:cv:`proxy.config.http.cache.cache_responses_to_cookies` :c:macro:`TS_CONFIG_HTTP_CACHE_CACHE_URLS_THAT_LOOK_DYNAMIC` :ts:cv:`proxy.config.http.cache.cache_urls_that_look_dynamic` +:c:macro:`TS_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER` :ts:cv:`proxy.config.http.cache.enable_default_vary_headers` :c:macro:`TS_CONFIG_HTTP_CACHE_GENERATION` :ts:cv:`proxy.config.http.cache.generation` :c:macro:`TS_CONFIG_HTTP_CACHE_GUARANTEED_MAX_LIFETIME` :ts:cv:`proxy.config.http.cache.guaranteed_max_lifetime` :c:macro:`TS_CONFIG_HTTP_CACHE_GUARANTEED_MIN_LIFETIME` :ts:cv:`proxy.config.http.cache.guaranteed_min_lifetime` @@ -85,6 +86,10 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_CACHE_HEURISTIC_MAX_LIFETIME` :ts:cv:`proxy.config.http.cache.heuristic_max_lifetime` :c:macro:`TS_CONFIG_HTTP_CACHE_HEURISTIC_MIN_LIFETIME` :ts:cv:`proxy.config.http.cache.heuristic_min_lifetime` :c:macro:`TS_CONFIG_HTTP_CACHE_HTTP` :ts:cv:`proxy.config.http.cache.http` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_mismatch` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_CHARSET_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_charset_mismatch` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_ENCODING_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_encoding_mismatch` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_LANGUAGE_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_language_mismatch` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_AUTHENTICATION` :ts:cv:`proxy.config.http.cache.ignore_authentication` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE` :ts:cv:`proxy.config.http.cache.ignore_client_cc_max_age` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE` :ts:cv:`proxy.config.http.cache.ignore_client_no_cache` @@ -98,6 +103,9 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_CACHE_RANGE_LOOKUP` :ts:cv:`proxy.config.http.cache.range.lookup` :c:macro:`TS_CONFIG_HTTP_CACHE_RANGE_WRITE` :ts:cv:`proxy.config.http.cache.range.write` :c:macro:`TS_CONFIG_HTTP_CACHE_REQUIRED_HEADERS` :ts:cv:`proxy.config.http.cache.required_headers` +:c:macro:`TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_IMAGES` :ts:cv:`proxy.config.http.cache.vary_default_images` +:c:macro:`TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_OTHER` :ts:cv:`proxy.config.http.cache.vary_default_other` +:c:macro:`TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_TEXT` :ts:cv:`proxy.config.http.cache.vary_default_text` :c:macro:`TS_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE` :ts:cv:`proxy.config.http.cache.when_to_revalidate` :c:macro:`TS_CONFIG_HTTP_CHUNKING_ENABLED` :ts:cv:`proxy.config.http.chunking_enabled` :c:macro:`TS_CONFIG_HTTP_CHUNKING_SIZE` :ts:cv:`proxy.config.http.chunking.size` @@ -136,6 +144,7 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED` :ts:cv:`proxy.config.http.post.check.content_length.enabled` :c:macro:`TS_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT` :ts:cv:`proxy.config.http.post_connect_attempts_timeout` :c:macro:`TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY` :ts:cv:`proxy.config.http.redirect_use_orig_cache_key` +:c:macro:`TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED` :ts:cv:`proxy.config.http.request_buffer_enabled` :c:macro:`TS_CONFIG_HTTP_REQUEST_HEADER_MAX_SIZE` :ts:cv:`proxy.config.http.request_header_max_size` :c:macro:`TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE` :ts:cv:`proxy.config.http.response_header_max_size` :c:macro:`TS_CONFIG_HTTP_RESPONSE_SERVER_ENABLED` :ts:cv:`proxy.config.http.response_server_enabled` @@ -162,6 +171,7 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT` :ts:cv:`proxy.config.websocket.no_activity_timeout` :c:macro:`TS_CONFIG_SSL_CERT_FILEPATH` :ts:cv:`proxy.config.ssl.client.cert.path` :c:macro:`TS_CONFIG_SSL_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER` :ts:cv:`proxy.config.ssl.client.verify.server` :c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD` :ts:cv:`proxy.config.http.parent_proxy.fail_threshold` :c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME` :ts:cv:`proxy.config.http.parent_proxy.retry_time` :c:macro:`TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.per_parent_connect_attempts` From 09e3ba2c1e202d1eca854aaece2c77fa30220e90 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 7 Nov 2018 14:18:42 -0800 Subject: [PATCH 012/526] Fixes another memory leak related to OCSP also fixes a thinko I made in a previous commit (d83798a5) --- iocore/net/OCSPStapling.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index 90133e42cb2..ee0534c5869 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -60,6 +60,9 @@ certinfo_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/ if (cinf->certname) { ats_free(cinf->certname); } + if (cinf->cid) { + OCSP_CERTID_free(cinf->cid); + } ink_mutex_destroy(&cinf->stapling_mutex); OPENSSL_free(cinf); } @@ -193,7 +196,7 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) return true; err: - if (cinf->uri) { + if (cinf->cid) { OCSP_CERTID_free(cinf->cid); } From eb0a5ca74c0587e42063613516c9e423a704402e Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Thu, 1 Nov 2018 16:12:17 -0500 Subject: [PATCH 013/526] Cleanup: Replace Dynarray by std::vector --- include/tscore/DynArray.h | 191 -------------------------------- proxy/ControlMatcher.h | 1 - proxy/RegressionSM.cc | 61 +++++----- proxy/RegressionSM.h | 23 ++-- proxy/hdrs/HdrHeap.cc | 3 +- proxy/hdrs/HdrHeap.h | 1 + proxy/http/HttpPages.h | 1 - src/traffic_server/CoreUtils.cc | 72 ++++-------- src/traffic_server/CoreUtils.h | 1 - src/tscore/Makefile.am | 1 - 10 files changed, 63 insertions(+), 292 deletions(-) delete mode 100644 include/tscore/DynArray.h diff --git a/include/tscore/DynArray.h b/include/tscore/DynArray.h deleted file mode 100644 index 49c0886e1b7..00000000000 --- a/include/tscore/DynArray.h +++ /dev/null @@ -1,191 +0,0 @@ -/** @file - - Dynamic Array Implementation used by Regex.cc - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -template class DynArray -{ -public: - DynArray(const T *val = 0, intptr_t initial_size = 0); - ~DynArray(); - -#ifndef __GNUC__ - operator const T *() const; -#endif - operator T *(); - T &operator[](intptr_t idx); - T &operator()(intptr_t idx); - T *detach(); - T defvalue() const; - intptr_t length(); - void clear(); - void set_length(intptr_t i); - -private: - void resize(intptr_t new_size); - -private: - T *data; - const T *default_val; - int size; - int pos; -}; - -template -inline DynArray::DynArray(const T *val, intptr_t initial_size) : data(nullptr), default_val(val), size(0), pos(-1) -{ - if (initial_size > 0) { - int i = 1; - - while (i < initial_size) { - i <<= 1; - } - - resize(i); - } -} - -template inline DynArray::~DynArray() -{ - if (data) { - delete[] data; - } -} - -#ifndef __GNUC__ -template inline DynArray::operator const T *() const -{ - return data; -} -#endif - -template inline DynArray::operator T *() -{ - return data; -} - -template inline T &DynArray::operator[](intptr_t idx) -{ - return data[idx]; -} - -template -inline T & -DynArray::operator()(intptr_t idx) -{ - if (idx >= size) { - intptr_t new_size; - - if (size == 0) { - new_size = 64; - } else { - new_size = size * 2; - } - - if (idx >= new_size) { - new_size = idx + 1; - } - - resize(new_size); - } - - if (idx > pos) { - pos = idx; - } - - return data[idx]; -} - -template -inline T * -DynArray::detach() -{ - T *d; - - d = data; - data = nullptr; - - return d; -} - -template -inline T -DynArray::defvalue() const -{ - return *default_val; -} - -template -inline intptr_t -DynArray::length() -{ - return pos + 1; -} - -template -inline void -DynArray::clear() -{ - if (data) { - delete[] data; - data = nullptr; - } - - size = 0; - pos = -1; -} - -template -inline void -DynArray::set_length(intptr_t i) -{ - pos = i - 1; -} - -template -inline void -DynArray::resize(intptr_t new_size) -{ - if (new_size > size) { - T *new_data; - intptr_t i; - - new_data = new T[new_size]; - - for (i = 0; i < size; i++) { - new_data[i] = data[i]; - } - - for (; i < new_size; i++) { - if (default_val) { - new_data[i] = (T)*default_val; - } - } - - if (data) { - delete[] data; - } - data = new_data; - size = new_size; - } -} diff --git a/proxy/ControlMatcher.h b/proxy/ControlMatcher.h index 16dc8ecb600..3c7c725a3a5 100644 --- a/proxy/ControlMatcher.h +++ b/proxy/ControlMatcher.h @@ -86,7 +86,6 @@ #pragma once -#include "tscore/DynArray.h" #include "tscore/IpMap.h" #include "tscore/Result.h" #include "tscore/MatcherUtils.h" diff --git a/proxy/RegressionSM.cc b/proxy/RegressionSM.cc index b31368e7867..ac83a697184 100644 --- a/proxy/RegressionSM.cc +++ b/proxy/RegressionSM.cc @@ -120,16 +120,15 @@ r_sequential(RegressionTest *t, RegressionSM *sm, ...) RegressionSM *new_sm = new RegressionSM(t); va_list ap; va_start(ap, sm); - new_sm->parallel = false; - new_sm->repeat = false; - new_sm->ichild = 0; - new_sm->nchildren = 0; - new_sm->nwaiting = 0; - while (nullptr != sm) { - new_sm->children(new_sm->nchildren++) = sm; - sm = va_arg(ap, RegressionSM *); + new_sm->parallel = false; + new_sm->repeat = false; + new_sm->ichild = 0; + new_sm->nwaiting = 0; + while (sm) { + new_sm->children.push_back(sm); + sm = va_arg(ap, RegressionSM *); } - new_sm->n = new_sm->nchildren; + new_sm->n = new_sm->children.size(); va_end(ap); return new_sm; } @@ -141,10 +140,9 @@ r_sequential(RegressionTest *t, int an, RegressionSM *sm) new_sm->parallel = false; new_sm->repeat = true; new_sm->ichild = 0; - new_sm->nchildren = 1; - new_sm->children(0) = sm; - new_sm->nwaiting = 0; - new_sm->n = an; + new_sm->children.push_back(sm); + new_sm->nwaiting = 0; + new_sm->n = an; return new_sm; } @@ -154,16 +152,15 @@ r_parallel(RegressionTest *t, RegressionSM *sm, ...) RegressionSM *new_sm = new RegressionSM(t); va_list ap; va_start(ap, sm); - new_sm->parallel = true; - new_sm->repeat = false; - new_sm->ichild = 0; - new_sm->nchildren = 0; - new_sm->nwaiting = 0; + new_sm->parallel = true; + new_sm->repeat = false; + new_sm->ichild = 0; + new_sm->nwaiting = 0; while (sm) { - new_sm->children(new_sm->nchildren++) = sm; - sm = va_arg(ap, RegressionSM *); + new_sm->children.push_back(sm); + sm = va_arg(ap, RegressionSM *); } - new_sm->n = new_sm->nchildren; + new_sm->n = new_sm->children.size(); va_end(ap); return new_sm; } @@ -175,10 +172,9 @@ r_parallel(RegressionTest *t, int an, RegressionSM *sm) new_sm->parallel = true; new_sm->repeat = true; new_sm->ichild = 0; - new_sm->nchildren = 1; - new_sm->children(0) = sm; - new_sm->nwaiting = 0; - new_sm->n = an; + new_sm->children.push_back(sm); + new_sm->nwaiting = 0; + new_sm->n = an; return new_sm; } @@ -227,15 +223,14 @@ RegressionSM::RegressionSM(const RegressionSM &ao) : Continuation(ao) { RegressionSM &o = *(RegressionSM *)&ao; - t = o.t; - status = o.status; - pstatus = o.pstatus; - parent = &o; - nwaiting = o.nwaiting; - nchildren = o.nchildren; + t = o.t; + status = o.status; + pstatus = o.pstatus; + parent = &o; + nwaiting = o.nwaiting; - for (intptr_t i = 0; i < nchildren; i++) { - children(i) = o.children[i]->clone(); + for (unsigned i = 0; i < o.children.size(); i++) { + children.push_back(o.children[i]->clone()); } n = o.n; diff --git a/proxy/RegressionSM.h b/proxy/RegressionSM.h index d643de37a8a..31cabf807bd 100644 --- a/proxy/RegressionSM.h +++ b/proxy/RegressionSM.h @@ -25,7 +25,6 @@ #include "I_EventSystem.h" #include "tscore/Regression.h" -#include "tscore/DynArray.h" /* Regression Test Composition State Machine @@ -50,17 +49,17 @@ struct RegressionSM : public Continuation { void run_in(int *pstatus, ink_hrtime t); // internal - int status = REGRESSION_TEST_INPROGRESS; - int *pstatus = nullptr; - RegressionSM *parent = nullptr; - int nwaiting = 0; - int nchildren = 0; - DynArray children = nullptr; - intptr_t n = 0; - intptr_t ichild = 0; - bool parallel = false; - bool repeat = false; - Action *pending_action = nullptr; + int status = REGRESSION_TEST_INPROGRESS; + int *pstatus = nullptr; + RegressionSM *parent = nullptr; + int nwaiting = 0; + int nchildren = 0; + std::vector children; + intptr_t n = 0; + intptr_t ichild = 0; + bool parallel = false; + bool repeat = false; + Action *pending_action = nullptr; int regression_sm_start(int event, void *data); int regression_sm_waiting(int event, void *data); diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index 0de45b212c9..38fe92fe955 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -616,6 +616,8 @@ HdrHeap::marshal(char *buf, int len) int ptr_xl_size = 2; MarshalXlate static_table[2]; MarshalXlate *ptr_xlation = static_table; + // need to initialize it here because of those gotos + MarshalXlate str_xlation[HDR_BUF_RONLY_HEAPS + 1]; // Let's start by skipping over the header block // and copying the pointer blocks to marshalled @@ -690,7 +692,6 @@ HdrHeap::marshal(char *buf, int len) // is too big and only copy over live strings if it is. May // not be too much of a problem since I've prevented too much // lost string space both in string alloc and inherit - MarshalXlate str_xlation[HDR_BUF_RONLY_HEAPS + 1]; if (m_read_write_heap) { char *copy_start = ((char *)m_read_write_heap.get()) + sizeof(HdrStrHeap); diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index faa8c488910..c1fe0b2bbe6 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -320,6 +320,7 @@ struct MarshalXlate { char *start; char *end; char *offset; + MarshalXlate() : start(nullptr), end(nullptr), offset(nullptr) {} }; struct HeapCheck { diff --git a/proxy/http/HttpPages.h b/proxy/http/HttpPages.h index b51e45ed9d8..7b9fea2337f 100644 --- a/proxy/http/HttpPages.h +++ b/proxy/http/HttpPages.h @@ -35,7 +35,6 @@ #include "tscore/ink_platform.h" #include "P_EventSystem.h" -#include "tscore/DynArray.h" #include "HTTP.h" #include "StatPages.h" #include "HttpSM.h" diff --git a/src/traffic_server/CoreUtils.cc b/src/traffic_server/CoreUtils.cc index ac628b5e68d..eac83b0d1a3 100644 --- a/src/traffic_server/CoreUtils.cc +++ b/src/traffic_server/CoreUtils.cc @@ -116,7 +116,7 @@ int program_counter = 0; bool inTable; FILE *fp; memTable default_memTable = {0, 0, 0}; -DynArray arrayMem(&default_memTable, 0); +std::vector arrayMem(0, default_memTable); HTTPHdrImpl *global_http; HttpSM *last_seen_http_sm = nullptr; @@ -172,39 +172,11 @@ CoreUtils::insert_table(intptr_t vaddr1, intptr_t offset1, intptr_t fsize1) m.fsize = fsize1; #endif - if (arrayMem.length() == 0) { - arrayMem(0); - arrayMem[(intptr_t)0].vaddr = vaddr1; - arrayMem[(intptr_t)0].offset = offset1; - arrayMem[(intptr_t)0].fsize = fsize1; + if (arrayMem.empty()) { + arrayMem.push_back({vaddr1, offset1, fsize1}); } else { - intptr_t index = find_vaddr(vaddr1, arrayMem.length(), 0); - if (index == arrayMem.length()) { - arrayMem(index); - arrayMem[index].vaddr = vaddr1; - arrayMem[index].offset = offset1; - arrayMem[index].fsize = fsize1; - } else if (index == 0) { - arrayMem(arrayMem.length()); - for (intptr_t i = 0; i < arrayMem.length(); i++) { - arrayMem[arrayMem.length() - i - 1].vaddr = arrayMem[arrayMem.length() - i - 2].vaddr; - arrayMem[arrayMem.length() - i - 1].offset = arrayMem[arrayMem.length() - i - 2].offset; - arrayMem[arrayMem.length() - i - 1].fsize = arrayMem[arrayMem.length() - i - 2].fsize; - } - arrayMem[(intptr_t)0].vaddr = vaddr1; - arrayMem[(intptr_t)0].offset = offset1; - arrayMem[(intptr_t)0].fsize = fsize1; - } else { - arrayMem(arrayMem.length()); - for (intptr_t i = 1; i < arrayMem.length() - index; i++) { - arrayMem[arrayMem.length() - i].vaddr = arrayMem[arrayMem.length() - i - 1].vaddr; - arrayMem[arrayMem.length() - i].offset = arrayMem[arrayMem.length() - i - 1].offset; - arrayMem[arrayMem.length() - i].fsize = arrayMem[arrayMem.length() - i - 1].fsize; - } - arrayMem[index].vaddr = vaddr1; - arrayMem[index].offset = offset1; - arrayMem[index].fsize = fsize1; - } + unsigned index = find_vaddr(vaddr1, arrayMem.size(), 0); + arrayMem.insert(arrayMem.begin() + index, {vaddr1, offset1, fsize1}); } } @@ -213,7 +185,7 @@ CoreUtils::insert_table(intptr_t vaddr1, intptr_t offset1, intptr_t fsize1) intptr_t CoreUtils::read_from_core(intptr_t vaddr, intptr_t bytes, char *buf) { - intptr_t index = find_vaddr(vaddr, arrayMem.length(), 0); + intptr_t index = find_vaddr(vaddr, arrayMem.size(), 0); intptr_t vadd = arrayMem[index - 1].vaddr; intptr_t offset = arrayMem[index - 1].offset; intptr_t size = arrayMem[index - 1].fsize; @@ -253,7 +225,7 @@ void CoreUtils::get_base_frame(intptr_t framep, core_stack_state *coress) { // finds vaddress less than framep - intptr_t index = find_vaddr(framep, arrayMem.length(), 0); + intptr_t index = find_vaddr(framep, arrayMem.size(), 0); intptr_t vadd = arrayMem[index - 1].vaddr; intptr_t off = arrayMem[index - 1].offset; intptr_t off2 = std::abs(vadd - framep); @@ -294,7 +266,7 @@ CoreUtils::get_next_frame(core_stack_state *coress) intptr_t i = 0; intptr_t framep = coress->framep; - intptr_t index = find_vaddr(framep, arrayMem.length(), 0); + intptr_t index = find_vaddr(framep, arrayMem.size(), 0); // finds vaddress less than framep intptr_t vadd = arrayMem[index - 1].vaddr; @@ -490,16 +462,14 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) { char buf[sizeof(char) * sizeof(HdrHeap)]; // Load HdrHeap chain - HTTPHdr *http_hdr = core_hdr; - HdrHeap *heap = (HdrHeap *)core_hdr->m_heap; - HdrHeap *heap_ptr = (HdrHeap *)http_hdr->m_heap; - intptr_t ptr_heaps = 0; - intptr_t ptr_heap_size = 0; - intptr_t ptr_xl_size = 2; - intptr_t str_size = 0; - intptr_t str_heaps = 0; - MarshalXlate default_MarshalXlate = {nullptr, nullptr, nullptr}; - DynArray ptr_xlation(&default_MarshalXlate, 2); + HTTPHdr *http_hdr = core_hdr; + HdrHeap *heap = (HdrHeap *)core_hdr->m_heap; + HdrHeap *heap_ptr = (HdrHeap *)http_hdr->m_heap; + intptr_t ptr_heaps = 0; + intptr_t ptr_heap_size = 0; + intptr_t str_size = 0; + intptr_t str_heaps = 0; + std::vector ptr_xlation(2); // MarshalXlate static_table[2]; // MarshalXlate* ptr_xlation = static_table; intptr_t used; @@ -537,8 +507,8 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) ::exit(0); } // Expand ptr xlation table if necessary - if (ptr_heaps >= ptr_xl_size) { - ptr_xlation(ptr_heaps); + if (static_cast(ptr_heaps) >= ptr_xlation.size()) { + ptr_xlation.resize(ptr_heaps + 1); } char *data, *free, *off; @@ -647,18 +617,18 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) } break; case HDR_HEAP_OBJ_HTTP_HEADER: - if (((HTTPHdrImpl *)obj)->marshal(ptr_xlation, ptr_heaps, str_xlation, str_heaps) < 0) { + if (((HTTPHdrImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps) < 0) { goto Failed; } live_hdr->m_http = (HTTPHdrImpl *)obj; break; case HDR_HEAP_OBJ_FIELD_BLOCK: - if (((MIMEFieldBlockImpl *)obj)->marshal(ptr_xlation, ptr_heaps, str_xlation, str_heaps) < 0) { + if (((MIMEFieldBlockImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps) < 0) { goto Failed; } break; case HDR_HEAP_OBJ_MIME_HEADER: - if (((MIMEHdrImpl *)obj)->marshal(ptr_xlation, ptr_heaps, str_xlation, str_heaps)) { + if (((MIMEHdrImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps)) { goto Failed; } break; diff --git a/src/traffic_server/CoreUtils.h b/src/traffic_server/CoreUtils.h index f4817290523..e3e77505585 100644 --- a/src/traffic_server/CoreUtils.h +++ b/src/traffic_server/CoreUtils.h @@ -38,7 +38,6 @@ #include #include #include -#include "tscore/DynArray.h" #define SP_REGNUM 15 /* Contains address of top of stack USP */ #define PC_REGNUM 12 /* Contains program counter EIP */ diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index 367cf8e900f..8c3c9f985f8 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -70,7 +70,6 @@ libtscore_la_SOURCES = \ defalloc.h \ Diags.cc \ Diags.h \ - DynArray.h \ EventNotify.cc \ EventNotify.h \ Extendible.h \ From 4d5d377a785bec291a67a52b5186ff4c6d733459 Mon Sep 17 00:00:00 2001 From: shinrich Date: Thu, 8 Nov 2018 14:51:06 -0500 Subject: [PATCH 014/526] Make double* and openclose* reliably wait for stats to be ready --- tests/gold_tests/continuations/double.test.py | 137 ++++++++++-------- .../continuations/double_h2.test.py | 92 ++++++------ .../continuations/openclose.test.py | 73 +++++----- .../continuations/openclose_h2.test.py | 136 ++++++++--------- tests/tools/plugins/continuations_verify.cc | 55 ++++--- tests/tools/plugins/ssntxnorder_verify.cc | 35 ++--- 6 files changed, 282 insertions(+), 246 deletions(-) diff --git a/tests/gold_tests/continuations/double.test.py b/tests/gold_tests/continuations/double.test.py index b3f98d0823a..f6ba7053603 100644 --- a/tests/gold_tests/continuations/double.test.py +++ b/tests/gold_tests/continuations/double.test.py @@ -17,112 +17,125 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' -Test transactions and sessions, making sure two continuations catch the same number of hooks. +Test transactions and sessions for http1, making sure the two continuations catch the same number of hooks. ''' -Test.SkipIf(Condition.true('This test errors frequently, and so it is disabled.')) + Test.SkipUnless( - Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work") + Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), ) Test.ContinueOnFail = True # Define default ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager") - +ts = Test.MakeATSProcess("ts", select_ports=False, command="traffic_manager") server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") Test.testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\nHost: double.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET / HTTP/1.1\r\nHost: double_h2.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} - -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'continuations_verify.cc'), ts) +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # add response to the server dictionary server.addResponse("sessionfile.log", request_header, response_header) + +# add port and remap rule +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'continuations_verify.*', - 'proxy.config.http.cache.http' : 0, #disable cache to simply the test. - 'proxy.config.cache.enable_read_while_writer' : 0 + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.cache.http': 0, # disable cache to simply the test. + 'proxy.config.cache.enable_read_while_writer': 0, + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.http2.max_concurrent_streams_in': 65535 }) -ts.Disk.remap_config.AddLine( - 'map http://double.test:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) -) -cmd = 'curl -vs http://127.0.0.1:{0}'.format(ts.Variables.port) -numberOfRequests = 25 +# add plugin to assist with test metrics +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, + 'plugins', 'continuations_verify.cc'), ts) + +comparator_command = ''' +if test "`traffic_ctl metric get continuations_verify.{0}.close.1 | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get continuations_verify.{0}.close.2 | cut -d ' ' -f 2`" ; then\ + echo yes;\ + else \ + echo no; \ + fi; \ + traffic_ctl metric match continuations_verify + ''' + +cmd = 'curl -vs http://127.0.0.1:{0}/'.format(ts.Variables.port) +numberOfRequests = 55 tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests) +# On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 +# Examinig the packet capture shows that Traffic Server dutifully sends the response +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = Any(0,2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( server, ready=When.PortOpen(server.Variables.Port)) # Adds a delay once the ts port is ready. This is because we cannot test the ts state. -tr.Processes.Default.StartBefore(ts, ready=10) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) ts.StartAfter(*ps) server.StartAfter(*ps) tr.StillRunningAfter = ts -comparator_command = ''' -if test "`traffic_ctl metric get continuations_verify.{0}.close.1 | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get continuations_verify.{0}.close.2 | cut -d ' ' -f 2`" ; then\ - echo yes;\ - else \ - echo no; \ - fi; - ''' +# Signal that all the curl processes have completed +tr = Test.AddTestRun("Curl Done") +tr.Processes.Default.Command = "traffic_ctl plugin msg done done" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Env = ts.Env +tr.StillRunningAfter = ts -records = ts.Disk.File(os.path.join(ts.Variables.RUNTIMEDIR, "records.snap")) +# Parking this as a ready tester on a meaningless process +# To stall the test runs that check for the stats until the +# stats have propagated and are ready to read. +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get continuations_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) + return retval.returncode == 0 + return done_stat_ready + # number of sessions/transactions opened and closed are equal -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated +tr = Test.AddTestRun("Check Ssn") +server2.StartupTimeout = 60 +# Again, here the imporant thing is the ready function not the server2 process +tr.Processes.Default.StartBefore(server2, ready=make_done_stat_ready(ts.Env)) tr.Processes.Default.Command = comparator_command.format('ssn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') +# Session and Txn's should be non-zero +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.ssn.close.1 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.ssn.close.2 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.txn.close.1 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.txn.close.2 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "continuations_verify.txn.close.1 {}".format(numberOfRequests), 'should be the number of transactions we made') +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "continuations_verify.txn.close.2 {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts -# for debugging session number -ssn1 = tr.Processes.Process("session1", 'traffic_ctl metric get continuations_verify.ssn.close.1 > ssn1') -ssn2 = tr.Processes.Process("session2", 'traffic_ctl metric get continuations_verify.ssn.close.2 > ssn2') -ssn1.Env = ts.Env -ssn2.Env = ts.Env -tr.Processes.Default.StartBefore(ssn1) -tr.Processes.Default.StartBefore(ssn2) +tr.StillRunningAfter = server2 -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated +tr = Test.AddTestRun("Check Txn") tr.Processes.Default.Command = comparator_command.format('txn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') tr.StillRunningAfter = ts -# for debugging transaction number -txn1 = tr.Processes.Process("transaction1", 'traffic_ctl metric get continuations_verify.txn.close.1 > txn1') -txn2 = tr.Processes.Process("transaction2", 'traffic_ctl metric get continuations_verify.txn.close.2 > txn2') -txn1.Env = ts.Env -txn2.Env = ts.Env -tr.Processes.Default.StartBefore(txn1) -tr.Processes.Default.StartBefore(txn2) - -# session count is positive, -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated -tr.Processes.Default.Command = "traffic_ctl metric get continuations_verify.ssn.close.1" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression(" 0", 'should be nonzero') -tr.StillRunningAfter = ts +tr.StillRunningAfter = server2 -# and we receive the same number of transactions as we asked it to make -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated -tr.Processes.Default.Command = "traffic_ctl metric get continuations_verify.txn.close.1" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( - "continuations_verify.txn.close.1 {}".format(numberOfRequests), 'should be the number of transactions we made') -tr.StillRunningAfter = ts diff --git a/tests/gold_tests/continuations/double_h2.test.py b/tests/gold_tests/continuations/double_h2.test.py index 61da07d7932..86aace464a1 100644 --- a/tests/gold_tests/continuations/double_h2.test.py +++ b/tests/gold_tests/continuations/double_h2.test.py @@ -17,6 +17,7 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' Test transactions and sessions for http2, making sure the two continuations catch the same number of hooks. ''' @@ -28,11 +29,12 @@ # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=False, command="traffic_manager") server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") Test.testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: double_h2.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # add response to the server dictionary @@ -45,7 +47,7 @@ # add port and remap rule ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map http://double_h2.test:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) ) ts.Disk.ssl_multicert_config.AddLine( @@ -54,7 +56,7 @@ ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'continuations_verify.*', + 'proxy.config.diags.debug.tags': 'continuations_verify', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.cache.http': 0, # disable cache to simply the test. @@ -75,7 +77,8 @@ echo yes;\ else \ echo no; \ - fi; + fi; \ + traffic_ctl metric match continuations_verify ''' # curl with http2 @@ -85,69 +88,66 @@ tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests) +# On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 +# Examinig the packet capture shows that Traffic Server dutifully sends the response +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = Any(0,2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( server, ready=When.PortOpen(server.Variables.Port)) # Adds a delay once the ts port is ready. This is because we cannot test the ts state. -tr.Processes.Default.StartBefore(ts, ready=10) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) ts.StartAfter(*ps) server.StartAfter(*ps) tr.StillRunningAfter = ts -# Watch the records snapshot file. -records = ts.Disk.File(os.path.join(ts.Variables.RUNTIMEDIR, "records.snap")) +# Signal that all the curl processes have completed +tr = Test.AddTestRun("Curl Done") +tr.Processes.Default.Command = "traffic_ctl plugin msg done done" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Env = ts.Env +tr.StillRunningAfter = ts +# Parking this as a ready tester on a meaningless process +# To stall the test runs that check for the stats until the +# stats have propagated and are ready to read. +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get continuations_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + # number of sessions/transactions opened and closed are equal -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated +tr = Test.AddTestRun("Check Ssn") +server2.StartupTimeout = 60 +# Again, here the imporant thing is the ready function not the server2 process +tr.Processes.Default.StartBefore(server2, ready=make_done_stat_ready(ts.Env)) tr.Processes.Default.Command = comparator_command.format('ssn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') +# Session and Txn's should be non-zero +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.ssn.close.1 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.ssn.close.2 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.txn.close.1 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.txn.close.2 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "continuations_verify.txn.close.1 {}".format(numberOfRequests), 'should be the number of transactions we made') +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "continuations_verify.txn.close.2 {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts +tr.StillRunningAfter = server2 -# for debugging session number -ssn1 = tr.Processes.Process("session1", 'traffic_ctl metric get continuations_verify.ssn.close.1 > ssn1') -ssn2 = tr.Processes.Process("session2", 'traffic_ctl metric get continuations_verify.ssn.close.2 > ssn2') -ssn1.Env = ts.Env -ssn2.Env = ts.Env -tr.Processes.Default.StartBefore(ssn1) -tr.Processes.Default.StartBefore(ssn2) - -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated +tr = Test.AddTestRun("Check Txn") tr.Processes.Default.Command = comparator_command.format('txn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') tr.StillRunningAfter = ts +tr.StillRunningAfter = server2 -# for debugging transaction number -txn1 = tr.Processes.Process("transaction1", 'traffic_ctl metric get continuations_verify.txn.close.1 > txn1') -txn2 = tr.Processes.Process("transaction2", 'traffic_ctl metric get continuations_verify.txn.close.2 > txn2') -txn1.Env = ts.Env -txn2.Env = ts.Env -tr.Processes.Default.StartBefore(txn1) -tr.Processes.Default.StartBefore(txn2) - -# session count is positive, -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated -tr.Processes.Default.Command = "traffic_ctl metric get continuations_verify.ssn.close.1" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression(" 0", 'should be nonzero') -tr.StillRunningAfter = ts - -# and we receive the same number of transactions as we asked it to make -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated -tr.Processes.Default.Command = "traffic_ctl metric get continuations_verify.txn.close.1" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( - "continuations_verify.txn.close.1 {}".format(numberOfRequests), 'should be the number of transactions we made') -tr.StillRunningAfter = ts diff --git a/tests/gold_tests/continuations/openclose.test.py b/tests/gold_tests/continuations/openclose.test.py index 32d45bdc8d1..452ea131513 100644 --- a/tests/gold_tests/continuations/openclose.test.py +++ b/tests/gold_tests/continuations/openclose.test.py @@ -17,6 +17,7 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' Test transactions and sessions, making sure they open and close in the proper order. ''' @@ -29,12 +30,13 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager") server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") Test.testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: oc.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", "timestamp": "1469733493.993", "body": ""} Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, @@ -59,8 +61,11 @@ tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests) +# On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 +# Examinig the packet capture shows that Traffic Server dutifully sends the response +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = Any(0,2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( @@ -71,13 +76,30 @@ server.StartAfter(*ps) tr.StillRunningAfter = ts -# Watch the records snapshot file. -records = ts.Disk.File(os.path.join(ts.Variables.RUNTIMEDIR, "records.snap")) +# Signal that all the curl processes have completed +tr = Test.AddTestRun("Curl Done") +tr.Processes.Default.Command = "traffic_ctl plugin msg done done" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Env = ts.Env +tr.StillRunningAfter = ts -# Check our work on traffic_ctl -# no errors happened, -tr = Test.AddTestRun() -tr.DelayStart = 10 +# Parking this as a ready tester on a meaningless process +# To stall the test runs that check for the stats until the +# stats have propagated and are ready to read. +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get ssntxnorder_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + +# number of sessions/transactions opened and closed are equal +tr = Test.AddTestRun("Check Ssn order errors") +server2.StartupTimeout = 60 +# Again, here the imporant thing is the ready function not the server2 process +tr.Processes.Default.StartBefore(server2, ready=make_done_stat_ready(ts.Env)) tr.Processes.Default.Command = 'traffic_ctl metric get ssntxnorder_verify.err' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env @@ -91,48 +113,33 @@ echo yes;\ else \ echo no; \ - fi; + fi; \ + traffic_ctl metric match ssntxnorder_verify ''' # number of sessions/transactions opened and closed are equal -tr = Test.AddTestRun() -tr.DelayStart = 10 +tr = Test.AddTestRun("Check for ssn open/close") tr.Processes.Default.Command = comparator_command.format('ssn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( "yes", 'should verify contents') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "ssntxnorder_verify.ssn.start 0", 'should be nonzero') tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr = Test.AddTestRun() -tr.DelayStart = 10 +tr = Test.AddTestRun("Check for txn/open/close") tr.Processes.Default.Command = comparator_command.format('txn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( "yes", 'should verify contents') -tr.StillRunningAfter = ts -tr.StillRunningAfter = server - -# session count is positive, -tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = "traffic_ctl metric get ssntxnorder_verify.ssn.start" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression( - " 0", 'should be nonzero') -tr.StillRunningAfter = ts -tr.StillRunningAfter = server - +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "ssntxnorder_verify.txn.start 0", 'should be nonzero') # and we receive the same number of transactions as we asked it to make -tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = "traffic_ctl metric get ssntxnorder_verify.txn.start" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( "ssntxnorder_verify.txn.start {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts tr.StillRunningAfter = server + diff --git a/tests/gold_tests/continuations/openclose_h2.test.py b/tests/gold_tests/continuations/openclose_h2.test.py index f14db70c79e..07aec811f7f 100644 --- a/tests/gold_tests/continuations/openclose_h2.test.py +++ b/tests/gold_tests/continuations/openclose_h2.test.py @@ -17,75 +17,67 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' -Test transactions and sessions for http2, making sure they open and close in the proper order. +Test transactions and sessions over http2, making sure they open and close in the proper order. ''' -Test.SkipIf(Condition.true('This test errors frequently, and so it is disabled.')) Test.SkipUnless( Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), Condition.HasCurlFeature('http2') ) -Test.ContinueOnFail = True + # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=False, command="traffic_manager") +ts.Variables.ssl_port = 4443 + server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") Test.testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\nHost: oc_h2.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET / HTTP/1.1\r\nHost: oc.test\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -# add response to the server dictionary -server.addResponse("sessionfile.log", request_header, response_header) - +# add ssl materials like key, certificates for the server ts.addSSLfile("ssl/server.pem") ts.addSSLfile("ssl/server.key") -ts.Variables.ssl_port = 4443 -ts.Disk.remap_config.AddLine( - 'map http://oc_h2.test:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) -) - -ts.Disk.ssl_multicert_config.AddLine( - 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' -) +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, + 'plugins', 'ssntxnorder_verify.cc'), ts) +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssntxnorder_verify.*', - 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.diags.debug.tags': 'ssntxnorder_verify', 'proxy.config.http.cache.http': 0, # disable cache to simply the test. 'proxy.config.cache.enable_read_while_writer': 0, - # enable ssl port + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), - 'proxy.config.ssl.client.verify.server': 0, - 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', - 'proxy.config.http2.max_concurrent_streams_in': 65535 }) -# add plugin to assist with test metrics -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, - 'plugins', 'ssntxnorder_verify.cc'), ts) - -comparator_command = ''' -if test "`traffic_ctl metric get ssntxnorder_verify.{0}.start | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get ssntxnorder_verify.{0}.close | cut -d ' ' -f 2`" ; then\ - echo yes;\ - else \ - echo no; \ - fi; - ''' +ts.Disk.remap_config.AddLine( + 'map http://oc.test:{0} http://127.0.0.1:{1}'.format( + ts.Variables.port, server.Variables.Port) +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) -# curl with http2 -cmd = 'curl --http2 -k -vs https://127.0.0.1:{0}/'.format(ts.Variables.ssl_port) +cmd = 'curl -k --http2 -vs https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) numberOfRequests = 25 tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests) +# On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 +# Examinig the packet capture shows that Traffic Server dutifully sends the response +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = Any(0,2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( @@ -96,13 +88,30 @@ server.StartAfter(*ps) tr.StillRunningAfter = ts -# Watch the records snapshot file. -records = ts.Disk.File(os.path.join(ts.Variables.RUNTIMEDIR, "records.snap")) +# Signal that all the curl processes have completed +tr = Test.AddTestRun("Curl Done") +tr.Processes.Default.Command = "traffic_ctl plugin msg done done" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Env = ts.Env +tr.StillRunningAfter = ts -# Check our work on traffic_ctl -# no errors happened, -tr = Test.AddTestRun() -tr.DelayStart = 10 +# Parking this as a ready tester on a meaningless process +# To stall the test runs that check for the stats until the +# stats have propagated and are ready to read. +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get ssntxnorder_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + +# number of sessions/transactions opened and closed are equal +tr = Test.AddTestRun("Check Ssn order errors") +server2.StartupTimeout = 60 +# Again, here the imporant thing is the ready function not the server2 process +tr.Processes.Default.StartBefore(server2, ready=make_done_stat_ready(ts.Env)) tr.Processes.Default.Command = 'traffic_ctl metric get ssntxnorder_verify.err' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env @@ -111,45 +120,38 @@ tr.StillRunningAfter = ts tr.StillRunningAfter = server +comparator_command = ''' +if test "`traffic_ctl metric get ssntxnorder_verify.{0}.start | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get ssntxnorder_verify.{0}.close | cut -d ' ' -f 2`" ; then\ + echo yes;\ + else \ + echo no; \ + fi; \ + traffic_ctl metric match ssntxnorder_verify + ''' + # number of sessions/transactions opened and closed are equal -tr = Test.AddTestRun() -tr.DelayStart = 10 +tr = Test.AddTestRun("Check for ssn open/close") tr.Processes.Default.Command = comparator_command.format('ssn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( "yes", 'should verify contents') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "ssntxnorder_verify.ssn.start 0", 'should be nonzero') tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr = Test.AddTestRun() -tr.DelayStart = 10 +tr = Test.AddTestRun("Check for txn/open/close") tr.Processes.Default.Command = comparator_command.format('txn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( "yes", 'should verify contents') -tr.StillRunningAfter = ts -tr.StillRunningAfter = server - -# session count is positive, -tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = "traffic_ctl metric get ssntxnorder_verify.ssn.start" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression( - " 0", 'should be nonzero') -tr.StillRunningAfter = ts -tr.StillRunningAfter = server - +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "ssntxnorder_verify.txn.start 0", 'should be nonzero') # and we receive the same number of transactions as we asked it to make -tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = "traffic_ctl metric get ssntxnorder_verify.txn.start" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( "ssntxnorder_verify.txn.start {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts tr.StillRunningAfter = server + diff --git a/tests/tools/plugins/continuations_verify.cc b/tests/tools/plugins/continuations_verify.cc index d87fd51993c..d4529684610 100644 --- a/tests/tools/plugins/continuations_verify.cc +++ b/tests/tools/plugins/continuations_verify.cc @@ -23,35 +23,46 @@ limitations under the License. */ -#define __STDC_FORMAT_MACROS 1 // for inttypes.h -#include // for PRIu64 -#include #include // for abort -#include // for NULL macro #include // for debug -// TODO Is LIFECYCLE_MSG enabled in 6.2.0, or 7.0.0, might require push -// with version rework - // debug messages viewable by setting 'proxy.config.diags.debug.tags' // in 'records.config' // debug messages during one-time initialization static const char DEBUG_TAG_INIT[] = "continuations_verify.init"; +static const char DEBUG_TAG_MSG[] = "continuations_verify.msg"; +static const char DEBUG_TAG_HOOK[] = "continuations_verify.hook"; // plugin registration info static char plugin_name[] = "continuations_verify"; -static char vendor_name[] = "Yahoo! Inc."; -static char support_email[] = "ats-devel@yahoo-inc.com"; - -static TSMutex order_mutex_1; // lock on global data -static TSMutex order_mutex_2; // lock on global data +static char vendor_name[] = "apache"; +static char support_email[] = "shinrich@apache.org"; // Statistics provided by the plugin static int stat_ssn_close_1 = 0; // number of TS_HTTP_SSN_CLOSE hooks caught by the first continuation static int stat_ssn_close_2 = 0; // number of TS_HTTP_SSN_CLOSE hooks caught by the second continuation static int stat_txn_close_1 = 0; // number of TS_HTTP_TXN_CLOSE hooks caught by the first continuation static int stat_txn_close_2 = 0; // number of TS_HTTP_TXN_CLOSE hooks caught by the second continuation +static int stat_test_done = 0; // Incremented when receiving a traffic_ctl message + +static int +handle_msg(TSCont contp, TSEvent event, void *edata) +{ + if (event == TS_EVENT_LIFECYCLE_MSG) { // External trigger, such as traffic_ctl + TSDebug(DEBUG_TAG_MSG, "event TS_EVENT_LIFECYCLE_MSG"); + // Send to a ET net thread just to be sure. + // Turns out the msg is sent to a task thread, but task + // threads do not get their thread local copy of the stats + // merged in. So externally, test.done was stuck at 0 without + // the Schedule to a NET thread + TSContSchedule(contp, 0, TS_THREAD_POOL_NET); + } else { + TSDebug(DEBUG_TAG_MSG, "event %d", event); + TSStatIntIncrement(stat_test_done, 1); + } + return 0; +} /** This function is called on every request and logs session and transaction @@ -64,6 +75,8 @@ handle_order_1(TSCont contp, TSEvent event, void *edata) TSHttpSsn ssnp; // session data TSHttpTxn txnp; // transaction data + TSDebug(DEBUG_TAG_HOOK, "order_1 event %d", event); + // Find the event that happened switch (event) { case TS_EVENT_HTTP_TXN_CLOSE: // End of transaction @@ -92,6 +105,8 @@ handle_order_2(TSCont contp, TSEvent event, void *edata) TSHttpSsn ssnp; // session data TSHttpTxn txnp; // transaction data + TSDebug(DEBUG_TAG_HOOK, "order_2 event %d", event); + // Find the event that happened switch (event) { case TS_EVENT_HTTP_TXN_CLOSE: // End of transaction @@ -142,14 +157,11 @@ TSPluginInit(int argc, const char *argv[]) TSError("[%s] Plugin registration failed. \n", plugin_name); } - order_mutex_1 = TSMutexCreate(); - TSCont contp_1; - order_mutex_2 = TSMutexCreate(); - TSCont contp_2; + TSCont contp_1 = TSContCreate(handle_order_1, TSMutexCreate()); + TSCont contp_2 = TSContCreate(handle_order_2, TSMutexCreate()); + TSCont contp = TSContCreate(handle_msg, TSMutexCreate()); - contp_1 = TSContCreate(handle_order_1, order_mutex_1); - contp_2 = TSContCreate(handle_order_2, order_mutex_2); - if (contp_1 == NULL || contp_2 == NULL) { + if (contp_1 == nullptr || contp_2 == nullptr || contp == nullptr) { // Continuation initialization failed. Unrecoverable, report and exit. TSError("[%s] could not create continuation", plugin_name); abort(); @@ -164,6 +176,8 @@ TSPluginInit(int argc, const char *argv[]) TSStatCreate("continuations_verify.txn.close.2", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); stat_ssn_close_2 = TSStatCreate("continuations_verify.ssn.close.2", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + stat_test_done = + TSStatCreate("continuations_verify.test.done", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); // Add all hooks. TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, contp_1); @@ -171,5 +185,8 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, contp_2); TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, contp_2); + + // Respond to a traffic_ctl message + TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, contp); } } diff --git a/tests/tools/plugins/ssntxnorder_verify.cc b/tests/tools/plugins/ssntxnorder_verify.cc index e2889d8531e..9ff16aedf14 100644 --- a/tests/tools/plugins/ssntxnorder_verify.cc +++ b/tests/tools/plugins/ssntxnorder_verify.cc @@ -22,19 +22,13 @@ See the License for the specific language governing permissions and limitations under the License. */ - -#define __STDC_FORMAT_MACROS 1 // for inttypes.h -#include // for PRIu64 #include -#include #include #include -#include // for abort -#include // for NULL macro -#include // for debug - -// TODO Is LIFECYCLE_MSG enabled in 6.2.0, or 7.0.0, might require push -// with version rework +#include // for abort +#include // for debug +#include // for PRIu64 +#include // debug messages viewable by setting 'proxy.config.diags.debug.tags' // in 'records.config' @@ -47,10 +41,8 @@ static const char DEBUG_TAG_HOOK[] = "ssntxnorder_verify.hook"; // plugin registration info static char plugin_name[] = "ssntxnorder_verify"; -static char vendor_name[] = "Yahoo! Inc."; -static char support_email[] = "ats-devel@yahoo-inc.com"; - -static TSMutex order_mutex; // lock on global data +static char vendor_name[] = "Apache"; +static char support_email[] = "shinrich@apache.org"; // List of started sessions, SSN_START seen, SSN_CLOSE not seen yet. static std::set started_ssns; @@ -84,6 +76,7 @@ static int stat_ssn_start = 0; // number of TS_HTTP_SSN_START hooks caught static int stat_txn_close = 0; // number of TS_HTTP_TXN_CLOSE hooks caught static int stat_txn_start = 0; // number of TS_HTTP_TXN_START hooks caught static int stat_err = 0; // number of inaccuracies encountered +static int stat_test_done = 0; // Set to 1 when the test is done // IPC information static char *ctl_tag = plugin_name; // name is a convenient identifier @@ -255,12 +248,18 @@ handle_order(TSCont contp, TSEvent event, void *edata) // Verify message is with the appropriate tag if (!strcmp(ctl_tag, msgp->tag) && strncmp(ctl_dump, reinterpret_cast(msgp->data), strlen(ctl_dump)) == 0) { dump_tables(); + } else { + TSContSchedule(contp, 0, TS_THREAD_POOL_NET); } break; } #endif + case TS_EVENT_IMMEDIATE: + TSStatIntIncrement(stat_test_done, 1); + break; + // Just release the lock for all other states and do nothing default: break; @@ -297,11 +296,8 @@ TSPluginInit(int argc, const char *argv[]) TSError("[%s] Plugin registration failed. \n", plugin_name); } - order_mutex = TSMutexCreate(); - TSCont contp; - - contp = TSContCreate(handle_order, order_mutex); - if (contp == NULL) { + TSCont contp = TSContCreate(handle_order, TSMutexCreate()); + if (contp == nullptr) { // Continuation initialization failed. Unrecoverable, report and exit. TSError("[%s] could not create continuation", plugin_name); abort(); @@ -313,6 +309,7 @@ TSPluginInit(int argc, const char *argv[]) stat_txn_start = TSStatCreate("ssntxnorder_verify.txn.start", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); stat_txn_close = TSStatCreate("ssntxnorder_verify.txn.close", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); stat_err = TSStatCreate("ssntxnorder_verify.err", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + stat_test_done = TSStatCreate("ssntxnorder_verify.test.done", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); // Add all hooks. TSHttpHookAdd(TS_HTTP_SSN_START_HOOK, contp); From 9db7acfe0996cb86a93d4b32ac2cbc5215b92af4 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 9 Nov 2018 16:05:42 -0700 Subject: [PATCH 015/526] Turn this Debug back into an Error --- plugins/header_rewrite/factory.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/header_rewrite/factory.cc b/plugins/header_rewrite/factory.cc index 41e29fe3cab..028fc6289ce 100644 --- a/plugins/header_rewrite/factory.cc +++ b/plugins/header_rewrite/factory.cc @@ -144,7 +144,7 @@ condition_factory(const std::string &cond) } else if (c_name == "INBOUND") { c = new ConditionInbound(); } else { - TSDebug(PLUGIN_NAME, "Unknown condition: %s", c_name.c_str()); + TSError("[%s] Unknown condition %s", PLUGIN_NAME, c_name.c_str()); return nullptr; } From c042358af14010f3bdcb4d1762c3558422283979 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Sun, 11 Nov 2018 16:15:23 +0900 Subject: [PATCH 016/526] Remove unused code --- proxy/http2/Http2Stream.cc | 18 ------------------ proxy/http2/Http2Stream.h | 9 --------- 2 files changed, 27 deletions(-) diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 1f3cd887dd0..2e1e65e6882 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -757,24 +757,6 @@ Http2Stream::destroy() THREAD_FREE(this, http2StreamAllocator, this_ethread()); } -bool -check_stream_thread(Continuation *cont) -{ - Http2Stream *stream = dynamic_cast(cont); - if (stream) { - return stream->get_thread() == this_ethread(); - } else { - return true; - } -} - -bool -check_continuation(Continuation *cont) -{ - Http2Stream *stream = dynamic_cast(cont); - return stream == nullptr; -} - void Http2Stream::response_initialize_data_handling(bool &is_done) { diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index e9950906536..b46134d8038 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -108,12 +108,6 @@ class Http2Stream : public ProxyClientTransaction bool change_state(uint8_t type, uint8_t flags); - void - set_id(Http2StreamId sid) - { - _id = sid; - } - void update_initial_rwnd(Http2WindowSize new_size) { @@ -298,6 +292,3 @@ class Http2Stream : public ProxyClientTransaction }; extern ClassAllocator http2StreamAllocator; - -extern bool check_continuation(Continuation *cont); -extern bool check_stream_thread(Continuation *cont); From 121948b6320e2d668b9c0dbafdfce321c5062043 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Thu, 8 Nov 2018 15:08:46 -0600 Subject: [PATCH 017/526] ArgParser: Standardize version message and other updates --- .../internal-libraries/ArgParser.en.rst | 8 ++++- include/tscore/ArgParser.h | 1 + src/traffic_cache_tool/Makefile.inc | 5 +++ src/traffic_layout/traffic_layout.cc | 5 +-- src/tscore/ArgParser.cc | 33 +++++++++++++++++-- 5 files changed, 46 insertions(+), 6 deletions(-) diff --git a/doc/developer-guide/internal-libraries/ArgParser.en.rst b/doc/developer-guide/internal-libraries/ArgParser.en.rst index 6e1619771c8..1448a90c4b0 100644 --- a/doc/developer-guide/internal-libraries/ArgParser.en.rst +++ b/doc/developer-guide/internal-libraries/ArgParser.en.rst @@ -144,11 +144,13 @@ from the :class:`Arguments` object returned from the parsing. The function can b args.invoke(); -Help message +Help and Version messages ------------------------- - Help message will be outputted when a wrong usage of the program is detected or `--help` option found. +- Version message is defined unified in :code:`ArgParser::version_message()`. + Classes +++++++ @@ -175,6 +177,10 @@ Classes Output usage to the console. + .. function:: void version_message() const + + Output version string to the console. + .. function:: void add_global_usage(std::string const &usage) Add a global_usage for :code:`help_message()`. Example: `traffic_blabla [--SWITCH [ARG]]`. diff --git a/include/tscore/ArgParser.h b/include/tscore/ArgParser.h index 927882d2c2d..95a5a2d64ee 100644 --- a/include/tscore/ArgParser.h +++ b/include/tscore/ArgParser.h @@ -175,6 +175,7 @@ class ArgParser bool parse(Arguments &ret, AP_StrVec &args); // The help & version messages void help_message(std::string_view err = "") const; + void version_message() const; // Helpr method for parse() void append_option_data(Arguments &ret, AP_StrVec &args, int index); // The command name and help message diff --git a/src/traffic_cache_tool/Makefile.inc b/src/traffic_cache_tool/Makefile.inc index 82063805082..6cb9430f8ce 100644 --- a/src/traffic_cache_tool/Makefile.inc +++ b/src/traffic_cache_tool/Makefile.inc @@ -48,4 +48,9 @@ traffic_cache_tool_traffic_cache_tool_LDADD = \ $(top_builddir)/src/tscore/.libs/Regex.o \ $(top_builddir)/src/tscore/.libs/CryptoHash.o \ $(top_builddir)/src/tscore/.libs/MMH.o \ + $(top_builddir)/src/tscore/.libs/Version.o \ + $(top_builddir)/src/tscore/.libs/Regression.o \ + $(top_builddir)/src/tscore/.libs/ink_args.o \ + $(top_builddir)/src/tscore/.libs/ParseRules.o \ + $(top_builddir)/src/tscore/.libs/SourceLocation.o \ @OPENSSL_LIBS@ @LIBPCRE@ @LIBTCL@ diff --git a/src/traffic_layout/traffic_layout.cc b/src/traffic_layout/traffic_layout.cc index a0095d142e0..a7bd66d1207 100644 --- a/src/traffic_layout/traffic_layout.cc +++ b/src/traffic_layout/traffic_layout.cc @@ -40,8 +40,9 @@ main(int argc, const char **argv) engine.parser.add_global_usage("traffic_layout CMD [OPTIONS]"); // global options - engine.parser.add_option("--help", "-h", "Print usage information"); - engine.parser.add_option("--run-root", "", "using TS_RUNROOT as sandbox", "", 1); + engine.parser.add_option("--help", "-h", "Print usage information") + .add_option("--run-root", "", "using TS_RUNROOT as sandbox", "", 1) + .add_option("--version", "-V", "Print version string"); // info command engine.parser.add_command("info", "Show the layout as default", [&]() { engine.info(); }) diff --git a/src/tscore/ArgParser.cc b/src/tscore/ArgParser.cc index 349eb657d61..6f3f477355d 100644 --- a/src/tscore/ArgParser.cc +++ b/src/tscore/ArgParser.cc @@ -22,10 +22,13 @@ */ #include "tscore/ArgParser.h" +#include "tscore/ink_file.h" +#include "tscore/I_Version.h" #include #include #include +#include std::string global_usage; std::string parser_program_name; @@ -76,7 +79,7 @@ ArgParser::add_global_usage(std::string const &usage) void ArgParser::help_message(std::string_view err) const { - return _top_level_command.help_message(err); + _top_level_command.help_message(err); } // a graceful way to output help message @@ -103,6 +106,17 @@ ArgParser::Command::help_message(std::string_view err) const if (!_example_usage.empty()) { std::cout << "\nExample Usage: " << _example_usage << std::endl; } + // standard return code + exit(EX_USAGE); +} + +void +ArgParser::Command::version_message() const +{ + // unified version message of ATS + AppVersionInfo appVersionInfo; + appVersionInfo.setup(PACKAGE_NAME, _name.c_str(), PACKAGE_VERSION, __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, ""); + ink_fputln(stdout, appVersionInfo.FullVersionInfoStr); exit(0); } @@ -158,7 +172,16 @@ ArgParser::parse(const char **argv) for (const auto &it : args) { msg = msg + " '" + it + "'"; } - _top_level_command.help_message(msg); + // find the correct level to output help message + ArgParser::Command *command = &_top_level_command; + for (unsigned i = 1; i < _argv.size(); i++) { + auto it = command->_subcommand_list.find(_argv[i]); + if (it == command->_subcommand_list.end()) { + break; + } + command = &it->second; + } + command->help_message(msg); } return ret; } @@ -398,8 +421,12 @@ ArgParser::Command::append_option_data(Arguments &ret, AP_StrVec &args, int inde i -= 1; } } else { + // output version message + if ((args[i] == "--version" || args[i] == "-V") && _option_list.find("--version") != _option_list.end()) { + version_message(); + } // output help message - if (args[i] == "--help" || args[i] == "-h") { + if ((args[i] == "--help" || args[i] == "-h") && _option_list.find("--help") != _option_list.end()) { ArgParser::Command *command = this; // find the correct level to output help messsage for (unsigned i = 1; i < args.size(); i++) { From a639581c85d9b9c9dcb728b03e86efc280b303c3 Mon Sep 17 00:00:00 2001 From: Vijay Mamidi Date: Mon, 12 Nov 2018 15:08:52 +0900 Subject: [PATCH 018/526] release the mutex after the events are cancelled and sessions are destroyed. --- proxy/http2/Http2ConnectionState.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index bb2a2b2b7b0..ef94a933a5d 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -141,10 +141,12 @@ class Http2ConnectionState : public Continuation } cleanup_streams(); - mutex = nullptr; // magic happens - assigning to nullptr frees the ProxyMutex delete local_hpack_handle; + local_hpack_handle = nullptr; delete remote_hpack_handle; + remote_hpack_handle = nullptr; delete dependency_tree; + dependency_tree = nullptr; this->ua_session = nullptr; if (fini_event) { @@ -153,6 +155,8 @@ class Http2ConnectionState : public Continuation if (zombie_event) { zombie_event->cancel(); } + // release the mutex after the events are cancelled and sessions are destroyed. + mutex = nullptr; // magic happens - assigning to nullptr frees the ProxyMutex } // Event handlers From 094ab25154e599d45505a0e40ca8ec3206f13edd Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Mon, 12 Nov 2018 08:03:19 +0900 Subject: [PATCH 019/526] Doc: sort overrides in table This sorts the table alphabetacally based on the TSOverridableConfigKey Value to make for easier finding. A few options are not documented, TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED (and its proxy.config.http.request_buffer_enabled) TS_CONFIG_SSL_CLIENT_VERIFY_SERVER --- .../functions/TSHttpOverridableConfig.en.rst | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index c99f3836b8d..003ee33f10d 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -66,6 +66,8 @@ The following configurations (from ``records.config``) are overridable: TSOverridableConfigKey Value Configuration Value ================================================================== ==================================================================== :c:macro:`TS_CONFIG_BODY_FACTORY_TEMPLATE_BASE` :ts:cv:`proxy.config.body_factory.template_base` +:c:macro:`TS_CONFIG_HTTP_ALLOW_HALF_OPEN` :ts:cv:`proxy.config.http.allow_half_open` +:c:macro:`TS_CONFIG_HTTP_ALLOW_MULTI_RANGE` :ts:cv:`proxy.config.http.allow_multi_range` :c:macro:`TS_CONFIG_HTTP_ANONYMIZE_INSERT_CLIENT_IP` :ts:cv:`proxy.config.http.insert_client_ip` :c:macro:`TS_CONFIG_HTTP_ANONYMIZE_REMOVE_CLIENT_IP` :ts:cv:`proxy.config.http.anonymize_remove_client_ip` :c:macro:`TS_CONFIG_HTTP_ANONYMIZE_REMOVE_COOKIE` :ts:cv:`proxy.config.http.anonymize_remove_cookie` @@ -86,10 +88,10 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_CACHE_HEURISTIC_MAX_LIFETIME` :ts:cv:`proxy.config.http.cache.heuristic_max_lifetime` :c:macro:`TS_CONFIG_HTTP_CACHE_HEURISTIC_MIN_LIFETIME` :ts:cv:`proxy.config.http.cache.heuristic_min_lifetime` :c:macro:`TS_CONFIG_HTTP_CACHE_HTTP` :ts:cv:`proxy.config.http.cache.http` -:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_mismatch` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_CHARSET_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_charset_mismatch` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_ENCODING_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_encoding_mismatch` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_LANGUAGE_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_language_mismatch` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_mismatch` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_AUTHENTICATION` :ts:cv:`proxy.config.http.cache.ignore_authentication` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE` :ts:cv:`proxy.config.http.cache.ignore_client_cc_max_age` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE` :ts:cv:`proxy.config.http.cache.ignore_client_no_cache` @@ -109,8 +111,8 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE` :ts:cv:`proxy.config.http.cache.when_to_revalidate` :c:macro:`TS_CONFIG_HTTP_CHUNKING_ENABLED` :ts:cv:`proxy.config.http.chunking_enabled` :c:macro:`TS_CONFIG_HTTP_CHUNKING_SIZE` :ts:cv:`proxy.config.http.chunking.size` -:c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES` :ts:cv:`proxy.config.http.connect_attempts_max_retries` :c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DEAD_SERVER` :ts:cv:`proxy.config.http.connect_attempts_max_retries_dead_server` +:c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES` :ts:cv:`proxy.config.http.connect_attempts_max_retries` :c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES` :ts:cv:`proxy.config.http.connect_attempts_rr_retries` :c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT` :ts:cv:`proxy.config.http.connect_attempts_timeout` :c:macro:`TS_CONFIG_HTTP_DEFAULT_BUFFER_SIZE` :ts:cv:`proxy.config.http.default_buffer_size` @@ -125,10 +127,10 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_FORWARD_PROXY_AUTH_TO_PARENT` :ts:cv:`proxy.config.http.forward.proxy_auth_to_parent` :c:macro:`TS_CONFIG_HTTP_GLOBAL_USER_AGENT_HEADER` :ts:cv:`proxy.config.http.global_user_agent_header` :c:macro:`TS_CONFIG_HTTP_INSERT_AGE_IN_RESPONSE` :ts:cv:`proxy.config.http.insert_age_in_response` +:c:macro:`TS_CONFIG_HTTP_INSERT_FORWARDED` :ts:cv:`proxy.config.http.insert_forwarded` :c:macro:`TS_CONFIG_HTTP_INSERT_REQUEST_VIA_STR` :ts:cv:`proxy.config.http.insert_request_via_str` :c:macro:`TS_CONFIG_HTTP_INSERT_RESPONSE_VIA_STR` :ts:cv:`proxy.config.http.insert_response_via_str` :c:macro:`TS_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR` :ts:cv:`proxy.config.http.insert_squid_x_forwarded_for` -:c:macro:`TS_CONFIG_HTTP_INSERT_FORWARDED` :ts:cv:`proxy.config.http.insert_forwarded` :c:macro:`TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_IN` :ts:cv:`proxy.config.http.keep_alive_enabled_in` :c:macro:`TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_OUT` :ts:cv:`proxy.config.http.keep_alive_enabled_out` :c:macro:`TS_CONFIG_HTTP_KEEP_ALIVE_NO_ACTIVITY_TIMEOUT_IN` :ts:cv:`proxy.config.http.keep_alive_no_activity_timeout_in` @@ -138,13 +140,19 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME` :ts:cv:`proxy.config.http.negative_caching_lifetime` :c:macro:`TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED` :ts:cv:`proxy.config.http.negative_revalidating_enabled` :c:macro:`TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME` :ts:cv:`proxy.config.http.negative_revalidating_lifetime` +:c:macro:`TS_CONFIG_HTTP_NORMALIZE_AE` :ts:cv:`proxy.config.http.normalize_ae` :c:macro:`TS_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS` :ts:cv:`proxy.config.http.number_of_redirections` +:c:macro:`TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT` :ts:cv:`proxy.config.http.parent_proxy.connect_attempts_timeout` +:c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD` :ts:cv:`proxy.config.http.parent_proxy.fail_threshold` +:c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME` :ts:cv:`proxy.config.http.parent_proxy.retry_time` :c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_TOTAL_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.total_connect_attempts` -:c:macro:`TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB` :ts:cv:`proxy.config.http.parent_proxy.mark_down_hostdb` +:c:macro:`TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.per_parent_connect_attempts` +:c:macro:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH` :ts:cv:`proxy.config.http.per_server.connection.match` +:c:macro:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX` :ts:cv:`proxy.config.http.per_server.connection.max` :c:macro:`TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED` :ts:cv:`proxy.config.http.post.check.content_length.enabled` :c:macro:`TS_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT` :ts:cv:`proxy.config.http.post_connect_attempts_timeout` :c:macro:`TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY` :ts:cv:`proxy.config.http.redirect_use_orig_cache_key` -:c:macro:`TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED` :ts:cv:`proxy.config.http.request_buffer_enabled` +TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED proxy.config.http.request_buffer_enabled :c:macro:`TS_CONFIG_HTTP_REQUEST_HEADER_MAX_SIZE` :ts:cv:`proxy.config.http.request_header_max_size` :c:macro:`TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE` :ts:cv:`proxy.config.http.response_header_max_size` :c:macro:`TS_CONFIG_HTTP_RESPONSE_SERVER_ENABLED` :ts:cv:`proxy.config.http.response_server_enabled` @@ -163,24 +171,16 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_NET_SOCK_PACKET_TOS_OUT` :ts:cv:`proxy.config.net.sock_packet_tos_out` :c:macro:`TS_CONFIG_NET_SOCK_RECV_BUFFER_SIZE_OUT` :ts:cv:`proxy.config.net.sock_recv_buffer_size_out` :c:macro:`TS_CONFIG_NET_SOCK_SEND_BUFFER_SIZE_OUT` :ts:cv:`proxy.config.net.sock_send_buffer_size_out` +:c:macro:`TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB` :ts:cv:`proxy.config.http.parent_proxy.mark_down_hostdb` :c:macro:`TS_CONFIG_SRV_ENABLED` :ts:cv:`proxy.config.srv_enabled` +:c:macro:`TS_CONFIG_SSL_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` +:c:macro:`TS_CONFIG_SSL_CERT_FILEPATH` :ts:cv:`proxy.config.ssl.client.cert.path` +TS_CONFIG_SSL_CLIENT_VERIFY_SERVER :ts:cv:`proxy.config.ssl.client.verify.server` :c:macro:`TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS` :ts:cv:`proxy.config.ssl.hsts_include_subdomains` :c:macro:`TS_CONFIG_SSL_HSTS_MAX_AGE` :ts:cv:`proxy.config.ssl.hsts_max_age` :c:macro:`TS_CONFIG_URL_REMAP_PRISTINE_HOST_HDR` :ts:cv:`proxy.config.url_remap.pristine_host_hdr` :c:macro:`TS_CONFIG_WEBSOCKET_ACTIVE_TIMEOUT` :ts:cv:`proxy.config.websocket.active_timeout` :c:macro:`TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT` :ts:cv:`proxy.config.websocket.no_activity_timeout` -:c:macro:`TS_CONFIG_SSL_CERT_FILEPATH` :ts:cv:`proxy.config.ssl.client.cert.path` -:c:macro:`TS_CONFIG_SSL_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` -:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER` :ts:cv:`proxy.config.ssl.client.verify.server` -:c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD` :ts:cv:`proxy.config.http.parent_proxy.fail_threshold` -:c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME` :ts:cv:`proxy.config.http.parent_proxy.retry_time` -:c:macro:`TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.per_parent_connect_attempts` -:c:macro:`TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT` :ts:cv:`proxy.config.http.parent_proxy.connect_attempts_timeout` -:c:macro:`TS_CONFIG_HTTP_NORMALIZE_AE` :ts:cv:`proxy.config.http.normalize_ae` -:c:macro:`TS_CONFIG_HTTP_ALLOW_MULTI_RANGE` :ts:cv:`proxy.config.http.allow_multi_range` -:c:macro:`TS_CONFIG_HTTP_ALLOW_HALF_OPEN` :ts:cv:`proxy.config.http.allow_half_open` -:c:macro:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX` :ts:cv:`proxy.config.http.per_server.connection.max` -:c:macro:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH` :ts:cv:`proxy.config.http.per_server.connection.match` ================================================================== ==================================================================== Examples From 59a5cc34c215ca1d86e2a39876d26fab08c73c36 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Mon, 12 Nov 2018 17:05:46 +0900 Subject: [PATCH 020/526] Doc: getting started case shouldn't used regex_remap With the getting started simple case using regex_map, we see that pattern used in user@ messages as well as stack overflow questions. Getting rid of that in favor of regular remap rules, and giving a few examples of slightly more complex examples to help gain intuition on how remap works. --- doc/admin-guide/security/index.en.rst | 4 +- doc/getting-started/index.en.rst | 62 +++++++++++++++++++++------ 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/doc/admin-guide/security/index.en.rst b/doc/admin-guide/security/index.en.rst index 411a924a448..22d59b8ff01 100644 --- a/doc/admin-guide/security/index.en.rst +++ b/doc/admin-guide/security/index.en.rst @@ -116,8 +116,8 @@ Client/Traffic Server connections, you must do the following: which your Traffic Server system will be using to terminate SSL connections with clients. :: - ip_dest=1.2.3.4 ssl_cert_name=example.com.pem - ip_dest=* ssl_cert_name=default.pem + dest_ip=1.2.3.4 ssl_cert_name=example.com.pem + dest_ip=* ssl_cert_name=default.pem #. *Optional*: Configure the use of client certificates using the variable :ts:cv:`proxy.config.ssl.client.certification_level` in :file:`records.config`. diff --git a/doc/getting-started/index.en.rst b/doc/getting-started/index.en.rst index b9e7efcb2a9..e65ad67ab2d 100644 --- a/doc/getting-started/index.en.rst +++ b/doc/getting-started/index.en.rst @@ -284,7 +284,12 @@ and want little more than to proxy all requests to our single origin server. This is accomplished with the following rule added to the :file:`remap.config` configuration:: - regex_map http://(.*)/ http://localhost:80/ + map http://www.acme.com/ http://localhost:80/ + +With this mapping rule, all paths that |TS| receives with a Host: header of +``www.acme.com`` will be proxied to ``localhost:80``. For instance, a request +for ``http://www.acme.com/foo/bar`` will be proxied to ``http://localhost:80/foo/bar``, +while requests with other Host: headers will be rejected. It is worth pausing at this point to note that in a reverse proxying scenario, it is |TS| itself which should be responding to HTTP requests made to your @@ -304,13 +309,46 @@ they reconfigure their origin service to listen on port ``8080`` instead of the default, and change |TS| to bind to ``80`` itself. Updating the remap is thus required, and it should now be:: - regex_map http://(.*)/ http://localhost:8080/ + map http://www.acme.com/ http://localhost:8080/ Now all requests made to ``www.acme.com`` are received by |TS| which knows to proxy those requests to ``localhost:8080`` if it cannot already serve them from its cache. Because we enabled pristine host headers earlier, the origin service will continue to receive ``Host: www.acme.com`` in the HTTP request. +If |AW| decides to use |TS| to reverse proxy a second domain ``static.acme.com`` +with a different origin server than the original, they need to make further +changes, as a new remap line needs to be added to handle the additional domain:: + + map http://static.acme.com/ http://origin-static.acme.com/ + +If they also decide to have requests to ``www.acme.com`` with paths that start with +``/api`` to a different origin server. The api origin server shouldn't get the ``/api``, +they will remap it away. And, since the above remap rules catch all paths, +this remap rule needs to be above it:: + + map http://www.acme.com/api/ http://api-origin.acme.com/ + +With this remap rule in place, a request to ``http://www.acme.com/api/example/foo`` +will be proxied to ``http://api-origin.acme.com/example/foo``. + +Finally, if |AW| decides to secure their site with https, they will need two +additional remap rules to handle the https requests. |TS| can translate an inbound +https request to an http request to origin. So, they would have additional remap +rules like:: + + map https://www.acme.com/ http://localhost:8080/ + map https://static.acme.com/ https://origin-static.acme.com/ + +This will require installing a certificate, and adding a line to +:file:`ssl_multicert.config`. Assuming the cert has the static.acme.com alternate +name, and that cert should be presented by default:: + + dest_ip=* ssl_cert_name=/path/to/secret/privatekey/acme.rsa + +Further information about configuring |TS| for TLS can be found :ref:`admin-ssl-termination` +section of the documentation. + Adjust Cache Parameters ~~~~~~~~~~~~~~~~~~~~~~~ @@ -346,12 +384,21 @@ entries: :file:`remap.config`:: - regex_map http://(.*)/ http://localhost:8080/ + map http://www.acme.com/api/ http://api-origin.acme.com/ + map https://www.acme.com/api/ https://api-origin.acme.com/ + map http://www.acme.com/ http://localhost:8080/ + map https://www.acme.com/ http://localhost:8080/ + map http://static.acme.com/ http://origin-static.acme.com/ + map https://static.acme.com/ https://origin-static.acme.com/ :file:`storage.config`:: /cache/trafficserver 500G +:file:`ssl_multicert.config`:: + + ssl_cert_name=/path/to/secret/acme.rsa + Configuring A Forward Proxy --------------------------- @@ -424,15 +471,6 @@ or instead of, the default |TS| logs. The Administrator's Guide discusses logging options in great detail in :ref:`admin-logging`. -Using Traffic Top ------------------ - -Using Stats Over HTTP ---------------------- - -Using Cache Inspector ---------------------- - Further Steps ============= From 9598ebfe2adc2287b03ed536675801fe0f8787b5 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Mon, 12 Nov 2018 13:28:31 -0600 Subject: [PATCH 021/526] Runroot: make runroot_handler support both ink_args and ArgParser --- include/tscore/runroot.h | 2 + src/traffic_layout/traffic_layout.cc | 5 +- src/tscore/runroot.cc | 104 +++++++++++++++++---------- 3 files changed, 70 insertions(+), 41 deletions(-) diff --git a/include/tscore/runroot.h b/include/tscore/runroot.h index 1dd391e394f..0f2efa153a4 100644 --- a/include/tscore/runroot.h +++ b/include/tscore/runroot.h @@ -54,6 +54,8 @@ typedef std::unordered_map RunrootMapType; bool exists(const std::string &dir); bool is_directory(const std::string &directory); +// argparser_runroot_handler should replace runroot_handler below when all program use ArgParser. +void argparser_runroot_handler(std::string const &value, const char *executable, bool json); void runroot_handler(const char **argv, bool json = false); // get a map from default layout diff --git a/src/traffic_layout/traffic_layout.cc b/src/traffic_layout/traffic_layout.cc index a7bd66d1207..72edd203327 100644 --- a/src/traffic_layout/traffic_layout.cc +++ b/src/traffic_layout/traffic_layout.cc @@ -41,7 +41,7 @@ main(int argc, const char **argv) // global options engine.parser.add_option("--help", "-h", "Print usage information") - .add_option("--run-root", "", "using TS_RUNROOT as sandbox", "", 1) + .add_option("--run-root", "", "using TS_RUNROOT as sandbox", "TS_RUNROOT", 1) .add_option("--version", "-V", "Print version string"); // info command @@ -68,7 +68,8 @@ main(int argc, const char **argv) engine.arguments = engine.parser.parse(argv); - runroot_handler(argv, engine.arguments.get("json")); + auto runroot_arg = engine.arguments.get("run-root"); + argparser_runroot_handler(runroot_arg.value(), argv[0], engine.arguments.get("json")); Layout::create(); engine.arguments.invoke(); diff --git a/src/tscore/runroot.cc b/src/tscore/runroot.cc index 5d786d9f5fb..f444d99531d 100644 --- a/src/tscore/runroot.cc +++ b/src/tscore/runroot.cc @@ -101,44 +101,10 @@ get_parent_yaml_path(const std::string &path) return {}; } -// handler for ts runroot -// this function set up runroot_file -void -runroot_handler(const char **argv, bool json) +static void +runroot_extra_handling(const char *executable, bool json) { - std::string prefix = "--run-root"; std::string path; - - // check if we have --run-root... - std::string arg = {}; - - int i = 0; - while (argv[i]) { - std::string_view command = argv[i]; - if (command.substr(0, prefix.size()) == prefix) { - arg = command.data(); - break; - } - i++; - } - - // if --run-root is provided - if (!arg.empty() && arg != prefix) { - // 1. pass in path - prefix += "="; - std::string value = arg.substr(prefix.size(), arg.size() - 1); - path = get_yaml_path(value); - if (!path.empty()) { - if (!json) { - ink_notice("using command line path as RUNROOT"); - } - runroot_file = path; - return; - } else if (!json) { - ink_warning("Unable to access runroot: '%s'", value.c_str()); - } - } - // 2. check Environment variable char *env_val = getenv("TS_RUNROOT"); if (env_val) { @@ -153,7 +119,6 @@ runroot_handler(const char **argv, bool json) ink_warning("Unable to access runroot: '%s' from $TS_RUNROOT", env_val); } } - // 3. find cwd or parent path of cwd to check char cwd[PATH_MAX] = {0}; if (getcwd(cwd, sizeof(cwd)) != nullptr) { @@ -166,10 +131,9 @@ runroot_handler(const char **argv, bool json) return; } } - // 4. installed executable char RealBinPath[PATH_MAX] = {0}; - if ((argv[0] != nullptr) && realpath(argv[0], RealBinPath) != nullptr) { + if ((executable != nullptr) && realpath(executable, RealBinPath) != nullptr) { std::string bindir = RealBinPath; bindir = bindir.substr(0, bindir.find_last_of("/")); // getting the bin dir not executable path path = get_parent_yaml_path(bindir); @@ -185,6 +149,68 @@ runroot_handler(const char **argv, bool json) return; } +// This is a temporary approach to handle runroot with migration to ArgParser. +// When all program use ArgParser, we can remove the runroot_handler below and replace it with this one. +void +argparser_runroot_handler(std::string const &value, const char *executable, bool json) +{ + // 1.if --run-root is provided + if (!value.empty()) { + std::string path = get_yaml_path(value); + if (!path.empty()) { + if (!json) { + ink_notice("using command line path as RUNROOT"); + } + runroot_file = path; + return; + } else if (!json) { + ink_warning("Unable to access runroot: '%s'", value.c_str()); + } + } + runroot_extra_handling(executable, json); +} + +// handler for ts runroot +// this function set up runroot_file +void +runroot_handler(const char **argv, bool json) +{ + std::string prefix = "--run-root"; + std::string path; + + // check if we have --run-root... + std::string arg = {}; + + int i = 0; + while (argv[i]) { + std::string_view command = argv[i]; + if (command.substr(0, prefix.size()) == prefix) { + arg = command.data(); + break; + } + i++; + } + + // if --run-root is provided + if (!arg.empty() && arg != prefix) { + // 1. pass in path + prefix += "="; + std::string value = arg.substr(prefix.size(), arg.size() - 1); + path = get_yaml_path(value); + if (!path.empty()) { + if (!json) { + ink_notice("using command line path as RUNROOT"); + } + runroot_file = path; + return; + } else if (!json) { + ink_warning("Unable to access runroot: '%s'", value.c_str()); + } + } + + runroot_extra_handling(argv[0], json); +} + // return a map of all path with default layout std::unordered_map runroot_map_default() From aee657da043fffe16eb5c71142b1dfda4a0b0dc6 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Sun, 11 Nov 2018 21:54:57 +0900 Subject: [PATCH 022/526] Print thread name instead of thread id --- configure.ac | 40 ++++++++++++++++++++++++++++++++ include/tscore/ink_thread.h | 18 ++++++++++++++ src/tscore/BufferWriterFormat.cc | 7 ++---- src/tscore/Diags.cc | 5 ++-- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index eb758d2e557..eb89e5cd427 100644 --- a/configure.ac +++ b/configure.ac @@ -1807,6 +1807,46 @@ AC_LINK_IFELSE([ AC_MSG_RESULT([no]) ]) +# pthread_getname_np / pthread_get_name_np: +AC_MSG_CHECKING([pthread_getname_np()]) +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ +#if HAVE_PTHREAD_H +#include +#endif +#if PTHREAD_NP_H +#include +#endif + ], [ + char name[[32]]; + pthread_getname_np(pthread_self(), name, sizeof(name)); + ]) + ], [ + AC_DEFINE(HAVE_PTHREAD_GETNAME_NP, 1, [Whether pthread_getname_np() is available]) + AC_MSG_RESULT([yes]) + ], [ + AC_MSG_RESULT([no]) + AC_MSG_CHECKING([pthread_get_name_np()]) + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ + #if HAVE_PTHREAD_H + #include + #endif + #if PTHREAD_NP_H + #include + #endif + ], [ + char name[[32]]; + pthread_get_name_np(pthread_self(), name, sizeof(name)); + ]) + ], [ + AC_DEFINE(HAVE_PTHREAD_GET_NAME_NP, 1, [Whether pthread_get_name_np() is available]) + AC_MSG_RESULT([yes]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) + # BSD-derived systems populate the socket length in the structure itself. It's # redundant to check all of these, but hey, I need the typing practice. AC_CHECK_MEMBER([struct sockaddr.sa_len], [], [], [#include ]) diff --git a/include/tscore/ink_thread.h b/include/tscore/ink_thread.h index 4e321316e2b..5382c9a2a5a 100644 --- a/include/tscore/ink_thread.h +++ b/include/tscore/ink_thread.h @@ -47,6 +47,10 @@ #include #endif +#if HAVE_SYS_PRCTL_H && defined(PR_SET_NAME) +#include +#endif + #define INK_MUTEX_INIT PTHREAD_MUTEX_INITIALIZER #define INK_THREAD_STACK_MIN PTHREAD_STACK_MIN @@ -307,4 +311,18 @@ ink_set_thread_name(const char *name ATS_UNUSED) #endif } +static inline void +ink_get_thread_name(char *name, size_t len) +{ +#if defined(HAVE_PTHREAD_GETNAME_NP) + pthread_getname_np(pthread_self(), name, len); +#elif defined(HAVE_PTHREAD_GET_NAME_NP) + pthread_get_name_np(pthread_self(), name, len); +#elif defined(HAVE_SYS_PRCTL_H) && defined(PR_GET_NAME) + prctl(PR_GET_NAME, name, 0, 0, 0); +#else + snprintf(name, len, "0x%" PRIx64, (uint64_t)ink_thread_self()); +#endif +} + #endif /* #if defined(POSIX_THREAD) */ diff --git a/src/tscore/BufferWriterFormat.cc b/src/tscore/BufferWriterFormat.cc index d5152e6f24c..e3bf9a7843c 100644 --- a/src/tscore/BufferWriterFormat.cc +++ b/src/tscore/BufferWriterFormat.cc @@ -23,6 +23,7 @@ #include "tscore/BufferWriter.h" #include "tscore/bwf_std_format.h" +#include "tscore/ink_thread.h" #include #include #include @@ -993,13 +994,9 @@ BWF_ThreadID(ts::BufferWriter &w, ts::BWFSpec const &spec) void BWF_ThreadName(ts::BufferWriter &w, ts::BWFSpec const &spec) { -#if defined(__FreeBSD_version) - bwformat(w, spec, "thread"sv); // no thread names in FreeBSD. -#else char name[32]; // manual says at least 16, bump that up a bit. - pthread_getname_np(pthread_self(), name, sizeof(name)); + ink_get_thread_name(name, sizeof(name)); bwformat(w, spec, std::string_view{name}); -#endif } static bool BW_INITIALIZED __attribute__((unused)) = []() -> bool { diff --git a/src/tscore/Diags.cc b/src/tscore/Diags.cc index 8d2cd02a185..179be9ec8af 100644 --- a/src/tscore/Diags.cc +++ b/src/tscore/Diags.cc @@ -243,10 +243,9 @@ Diags::print_va(const char *debug_tag, DiagsLevel diags_level, const SourceLocat size_t timestamp_end_offset = format_writer.size(); /////////////////////// - // add the thread id // + // add the thread name // /////////////////////// - format_writer.fill( - snprintf(format_writer.auxBuffer(), format_writer.remaining(), "{0x%" PRIx64 "} ", (uint64_t)ink_thread_self())); + format_writer.print("{thread-name} "); ////////////////////////////////// // append the diag level prefix // From 0135b6380b2d1bb26a62588419cf575929ede180 Mon Sep 17 00:00:00 2001 From: Vijay Mamidi Date: Wed, 14 Nov 2018 13:41:03 +0900 Subject: [PATCH 023/526] Ability to turn off traffic manager listening to proxy ports --- doc/appendices/command-line/traffic_manager.en.rst | 1 + mgmt/LocalManager.cc | 4 ++-- mgmt/LocalManager.h | 3 ++- src/traffic_manager/traffic_manager.cc | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/appendices/command-line/traffic_manager.en.rst b/doc/appendices/command-line/traffic_manager.en.rst index 4011298bbd7..61a7fbdf1f7 100644 --- a/doc/appendices/command-line/traffic_manager.en.rst +++ b/doc/appendices/command-line/traffic_manager.en.rst @@ -36,6 +36,7 @@ Description .. option:: --path FILE .. option:: --proxyBackDoor PORT .. option:: --proxyOff +.. option:: --listenOff .. option:: --proxyPort PORT .. option:: --recordsConf FILE .. option:: --tsArgs ARGUMENTS diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index 4b94d9cdc65..d23c913ae60 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -193,7 +193,7 @@ LocalManager::processRunning() } } -LocalManager::LocalManager(bool proxy_on) : BaseManager(), run_proxy(proxy_on) +LocalManager::LocalManager(bool proxy_on, bool listen) : BaseManager(), run_proxy(proxy_on), listen_for_proxy(listen) { bool found; std::string rundir(RecConfigReadRuntimeDir()); @@ -981,7 +981,7 @@ LocalManager::closeProxyPorts() void LocalManager::listenForProxy() { - if (!run_proxy) { + if (!run_proxy || !listen_for_proxy) { return; } diff --git a/mgmt/LocalManager.h b/mgmt/LocalManager.h index 8feffbd6148..0f4faffabad 100644 --- a/mgmt/LocalManager.h +++ b/mgmt/LocalManager.h @@ -56,7 +56,7 @@ enum ManagementPendingOperation { class LocalManager : public BaseManager { public: - explicit LocalManager(bool proxy_on); + explicit LocalManager(bool proxy_on, bool listen); ~LocalManager(); void initAlarm(); @@ -93,6 +93,7 @@ class LocalManager : public BaseManager bool processRunning(); bool run_proxy; + bool listen_for_proxy; bool proxy_recoverable = true; // false if traffic_server cannot recover with a reboot time_t manager_started_at; time_t proxy_started_at = -1; diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc index d167ec81f7c..a0168969e9b 100644 --- a/src/traffic_manager/traffic_manager.cc +++ b/src/traffic_manager/traffic_manager.cc @@ -84,6 +84,7 @@ static inkcoreapi DiagsConfig *diagsConfig; static char debug_tags[1024] = ""; static char action_tags[1024] = ""; static int proxy_off = false; +static int listen_off = false; static char bind_stdout[512] = ""; static char bind_stderr[512] = ""; @@ -491,6 +492,7 @@ main(int argc, const char **argv) ArgumentDescription argument_descriptions[] = { {"proxyOff", '-', "Disable proxy", "F", &proxy_off, nullptr, nullptr}, + {"listenOff", '-', "Disable traffic manager listen to proxy ports", "F", &listen_off, nullptr, nullptr}, {"path", '-', "Path to the management socket", "S*", &mgmt_path, nullptr, nullptr}, {"recordsConf", '-', "Path to records.config", "S*", &recs_conf, nullptr, nullptr}, {"tsArgs", '-', "Additional arguments for traffic_server", "S*", &tsArgs, nullptr, nullptr}, @@ -572,7 +574,7 @@ main(int argc, const char **argv) #endif ts_host_res_global_init(); ts_session_protocol_well_known_name_indices_init(); - lmgmt = new LocalManager(proxy_off == false); + lmgmt = new LocalManager(proxy_off == false, listen_off == false); RecLocalInitMessage(); lmgmt->initAlarm(); From 49cfc45e5e008eb7c1bdc83c988aca5722f89343 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Tue, 13 Nov 2018 16:51:58 +0900 Subject: [PATCH 024/526] Doc: Remove pipeline references Issue #3712 points out that its unlikely that http1.1 pipelining is not working. Options were removed for it a while back. Similarly, support for http 0.9 has been removed. --- doc/admin-guide/files/records.config.en.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index f7d2f64f920..1638255339e 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -870,14 +870,9 @@ mptcp .. note:: - If HTTP/1.1 is used, then |TS| can use keep-alive connections with - pipelining to origin servers. + If HTTP/1.1 is used, then |TS| can use keep-alive connections to origin servers. - If HTTP/1.0 is used, then |TS| can use keep-alive connections without - pipelining to origin servers. - - If HTTP/0.9 is used, then |TS| does not use keep-alive connections to - origin servers. + If HTTP/1.0 is used, then |TS| can use keep-alive connections to origin servers. .. ts:cv:: CONFIG proxy.config.http.chunking.size INT 4096 :overridable: From 65e7b7560e5497997c096400b247f2640652b22e Mon Sep 17 00:00:00 2001 From: David Calavera Date: Fri, 16 Nov 2018 14:44:03 -0800 Subject: [PATCH 025/526] Transform images when a client accepts image/webp. More browsers besides Chrome have started to support webp as image format. To advertise their support, they include image/webp content type in the Accept header. This change allows this plugin to detect support for webp by checking the Accept header and keeps the old User-Agent check for old versions of Chrome. Signed-off-by: David Calavera --- plugins/experimental/webp_transform/ImageTransform.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/experimental/webp_transform/ImageTransform.cc b/plugins/experimental/webp_transform/ImageTransform.cc index 68ea938a575..1f29a88b8b6 100644 --- a/plugins/experimental/webp_transform/ImageTransform.cc +++ b/plugins/experimental/webp_transform/ImageTransform.cc @@ -92,7 +92,12 @@ class GlobalHookPlugin : public GlobalPlugin { string ctype = transaction.getServerResponse().getHeaders().values("Content-Type"); string user_agent = transaction.getServerRequest().getHeaders().values("User-Agent"); - if (user_agent.find("Chrome") != string::npos && (ctype.find("jpeg") != string::npos || ctype.find("png") != string::npos)) { + string accept = transaction.getServerRequest().getHeaders().values("Accept"); + + bool webp_supported = accept.find("image/webp") != string::npos || user_agent.find("Chrome") != string::npos; + bool image_format = ctype.find("jpeg") != string::npos || ctype.find("png") != string::npos; + + if (webp_supported && image_format) { TS_DEBUG(TAG, "Content type is either jpeg or png. Converting to webp"); transaction.addPlugin(new ImageTransform(transaction)); } From 6f54dfeaa3541b1da9ae4d434a010918bb9f1821 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Fri, 16 Nov 2018 20:05:47 -0600 Subject: [PATCH 026/526] Add case were origin returns 404 to x_remap gold test. --- tests/gold_tests/pluginTest/xdebug/x_remap/four.in | 4 ++++ tests/gold_tests/pluginTest/xdebug/x_remap/out.gold | 11 +++++++++++ .../gold_tests/pluginTest/xdebug/x_remap/x_remap.gold | 2 ++ .../pluginTest/xdebug/x_remap/x_remap.test.py | 1 + 4 files changed, 18 insertions(+) create mode 100644 tests/gold_tests/pluginTest/xdebug/x_remap/four.in diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/four.in b/tests/gold_tests/pluginTest/xdebug/x_remap/four.in new file mode 100644 index 00000000000..1982451bbe8 --- /dev/null +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/four.in @@ -0,0 +1,4 @@ +GET /not_there HTTP/1.1 +Host: two +X-Debug: X-Remap + diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold b/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold index e60a8aae9bf..0619bd2e5c8 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold @@ -58,6 +58,17 @@ X-Remap: from=http://three[0-9]+/, to=http://127.0.0.1:SERVER_PORT/ 0 +====== +HTTP/1.1 404 Not Found +Server: ATS/`` +Date: `` +Age: `` +Transfer-Encoding: chunked +Connection: keep-alive +X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ + +0 + ====== HTTP/1.1 200 OK Date: `` diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold index 55e7f94dba5..d0baa817b03 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold @@ -4,6 +4,8 @@ X_DEBUG MISSING - X_DEBUG MISSING - +X_DEBUG MISSING +- X-Remap, fwd - X_DEBUG MISSING diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py index e901098823c..fb070d0d3cb 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py @@ -68,6 +68,7 @@ def sendMsg(msgFile): sendMsg('one') sendMsg('two') sendMsg('three') +sendMsg('four') sendMsg('fwd1') sendMsg('fwd2') sendMsg('fwd3') From cc5f301a587658f477f9eadf6e31b08176f0a496 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Sat, 17 Nov 2018 16:33:48 +0900 Subject: [PATCH 027/526] Doc: cache.config isn't a substitute for Cache-Control As the first file in the admin-guide config files, and saying that the cache.config file controls the cache, folks think it is *the* way to get objects cached. Having the origin provide caching infor via the Cache-Control: header is the prefered method of communicating cache policies. --- doc/admin-guide/files/cache.config.en.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/admin-guide/files/cache.config.en.rst b/doc/admin-guide/files/cache.config.en.rst index 32e75528e01..40d3a056176 100644 --- a/doc/admin-guide/files/cache.config.en.rst +++ b/doc/admin-guide/files/cache.config.en.rst @@ -22,8 +22,8 @@ cache.config ************ -The :file:`cache.config` file defines how |TS| caches web objects. You can add -caching rules to specify the following: +The :file:`cache.config` file allows you to overrule the origin's cache +policies. You can add caching rules to specify the following: - Not to cache objects from specific IP addresses. - How long to pin particular objects in the cache. @@ -32,6 +32,15 @@ caching rules to specify the following: .. important:: + Generally, using this file to define cache policies is an antipattern. + It's usually better to have the origin specify the cache policy via the + `Cache-Control: `_ header. + That way, all the business logic stays with the content generation. The + origin is in a much better position to know which content can be safely + cached, and for how long. It can make fine grained decisions, changing + Cache-Control: header value per object. This file allows for some overrides + but, is relatively crude compared to what the origin can provide. + After modifying :file:`cache.config`, run :option:`traffic_ctl config reload` to apply changes. From 406bfd3433dfb81ae86acfd1b240cd1e8ef9d0a7 Mon Sep 17 00:00:00 2001 From: Jean Baptiste Favre Date: Sun, 18 Nov 2018 22:56:55 +0100 Subject: [PATCH 028/526] Detect whether we need -latomic or not. Fix mips build --- build/atomic.m4 | 38 ++++++++++++++++++++++++++++++++++++++ configure.ac | 5 +++++ 2 files changed, 43 insertions(+) create mode 100644 build/atomic.m4 diff --git a/build/atomic.m4 b/build/atomic.m4 new file mode 100644 index 00000000000..266eb29d4ac --- /dev/null +++ b/build/atomic.m4 @@ -0,0 +1,38 @@ +dnl -------------------------------------------------------- -*- autoconf -*- +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. + +dnl ----------------------------------------------------------------- +dnl atomic.m4: Trafficserver's autoconf macros for testing atomic support +dnl + +dnl +dnl TS_CHECK_ATOMIC: try to figure out the need for -latomic +dnl +AC_DEFUN([TS_CHECK_ATOMIC], [ + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[#include ]], + [[uint64_t val = 0; __atomic_add_fetch(&val, 1, __ATOMIC_RELAXED);]])], + [AC_DEFINE(HAVE_ATOMIC, 1, [Define to 1 if you have '__atomic' functions.]) + AC_LINK_IFELSE( + [AC_LANG_PROGRAM([[#include ]], + [[uint64_t val = 0; __atomic_add_fetch(&val, 1, __ATOMIC_RELAXED);]])], + [ATOMIC_LIBS=""], + [ATOMIC_LIBS="-latomic"] + )], + [ATOMIC_LIBS=""] + ) + AC_SUBST([ATOMIC_LIBS]) +]) diff --git a/configure.ac b/configure.ac index eb89e5cd427..5a8245d3b8f 100644 --- a/configure.ac +++ b/configure.ac @@ -1166,6 +1166,11 @@ AX_WITH_CURSES CFLAGS="$__saved_CFLAGS" AC_SUBST([CURSES_LDFLAGS],[$curses_ldflags]) +# +# Check for -latomic need (at least for mips arch) +TS_CHECK_ATOMIC +TS_ADDTO([LDFLAGS], [$ATOMIC_LIBS]) + # # Check for SSL presence and usability TS_CHECK_CRYPTO From 3efc1564e397f3aa2e15f64f9a2ce6d66df833f2 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 24 Sep 2018 10:22:30 -0700 Subject: [PATCH 029/526] Don't update records.config when other configs are reloaded Prior to this change, when an operator updated a config file, it's backing config variable would get inserted into records.config on reload. The function modified is only used by the config file signaling mechanism. --- lib/records/P_RecCore.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/records/P_RecCore.cc b/lib/records/P_RecCore.cc index 4c83b8acb21..1da0cc9a067 100644 --- a/lib/records/P_RecCore.cc +++ b/lib/records/P_RecCore.cc @@ -920,7 +920,7 @@ RecSetSyncRequired(char *name, bool lock) r1 = it->second; if (i_am_the_record_owner(r1->rec_type)) { rec_mutex_acquire(&(r1->lock)); - r1->sync_required = REC_SYNC_REQUIRED; + r1->sync_required = REC_PEER_SYNC_REQUIRED; if (REC_TYPE_IS_CONFIG(r1->rec_type)) { r1->config_meta.update_required = REC_UPDATE_REQUIRED; } From e3f72d967a14a62c86f0df00c9fe0fd7418d1cdc Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 12 Nov 2018 07:50:28 +0900 Subject: [PATCH 030/526] An example of our preferred indentation style --- lib/records/RecCore.cc | 17 ++++++++++++++++- lib/records/RecRawStats.cc | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/records/RecCore.cc b/lib/records/RecCore.cc index f264a57f352..6f83226e533 100644 --- a/lib/records/RecCore.cc +++ b/lib/records/RecCore.cc @@ -314,6 +314,7 @@ RecRegisterConfigUpdateCb(const char *name, RecConfigUpdateCb update_cb, void *c if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_CONFIG(r->rec_type)) { /* -- upgrade to support a list of callback functions @@ -389,6 +390,7 @@ RecGetRecordString(const char *name, char *buf, int buf_len, bool lock) } if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (!r->registered || (r->data_type != RECD_STRING)) { err = REC_ERR_FAIL; @@ -468,6 +470,7 @@ RecLookupRecord(const char *name, void (*callback)(const RecRecord *, void *), v if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); callback(r, data); err = REC_ERR_OKAY; @@ -522,6 +525,7 @@ RecGetRecordType(const char *name, RecT *rec_type, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); *rec_type = r->rec_type; err = REC_ERR_OKAY; @@ -546,6 +550,7 @@ RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (!r->registered) { err = REC_ERR_FAIL; @@ -576,6 +581,7 @@ RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lo if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_STAT(r->rec_type)) { *persist_type = r->stat_meta.persist_type; @@ -602,6 +608,7 @@ RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + if (r->registered) { rec_mutex_acquire(&(r->lock)); if (order) { @@ -633,6 +640,7 @@ RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_CONFIG(r->rec_type)) { *update_type = r->config_meta.update_type; @@ -661,6 +669,7 @@ RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_CONFIG(r->rec_type)) { *check_type = r->config_meta.check_type; @@ -689,6 +698,7 @@ RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_CONFIG(r->rec_type)) { *check_expr = r->config_meta.check_expr; @@ -717,7 +727,8 @@ RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; - *buf = (char *)ats_malloc(sizeof(char) * 1024); + + *buf = (char *)ats_malloc(sizeof(char) * 1024); memset(*buf, 0, 1024); err = REC_ERR_OKAY; @@ -767,6 +778,7 @@ RecGetRecordAccessType(const char *name, RecAccessT *access, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); *access = r->config_meta.access_type; err = REC_ERR_OKAY; @@ -791,6 +803,7 @@ RecSetRecordAccessType(const char *name, RecAccessT access, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); r->config_meta.access_type = access; err = REC_ERR_OKAY; @@ -815,6 +828,7 @@ RecGetRecordSource(const char *name, RecSourceT *source, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); *source = r->config_meta.source; err = REC_ERR_OKAY; @@ -902,6 +916,7 @@ RecGetRecord_Xmalloc(const char *name, RecDataT data_type, RecData *data, bool l if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (!r->registered || (r->data_type != data_type)) { err = REC_ERR_FAIL; diff --git a/lib/records/RecRawStats.cc b/lib/records/RecRawStats.cc index dbc2664d677..c83d75cf241 100644 --- a/lib/records/RecRawStats.cc +++ b/lib/records/RecRawStats.cc @@ -535,6 +535,7 @@ RecRegisterRawStatSyncCb(const char *name, RecRawStatSyncCb sync_cb, RecRawStatB ink_rwlock_rdlock(&g_records_rwlock); if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_STAT(r->rec_type)) { if (r->stat_meta.sync_cb) { From ab94cf8efbadabd485db9bc6116818e539715db8 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 16 Nov 2018 16:09:04 +0900 Subject: [PATCH 031/526] Changes remap_purge to update the purge state immediately --- .../experimental/remap_purge/remap_purge.c | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/plugins/experimental/remap_purge/remap_purge.c b/plugins/experimental/remap_purge/remap_purge.c index a8ce00fb2ce..3027543563e 100644 --- a/plugins/experimental/remap_purge/remap_purge.c +++ b/plugins/experimental/remap_purge/remap_purge.c @@ -105,10 +105,10 @@ delete_purge_instance(PurgeInstance *purge) } } -/* This is where we start the PURGE events, setting up the transactino to fail, +/* This is where we start the PURGE events, setting up the transaction to fail, and bump the generation ID, and finally save the state. */ -static int -on_http_cache_lookup_complete(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge) +static void +update_purge_state(PurgeInstance *purge) { FILE *file; @@ -126,14 +126,11 @@ on_http_cache_lookup_complete(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge } TSMutexUnlock(purge->lock); - - TSHttpTxnReenable(txnp, TS_EVENT_HTTP_ERROR); - return TS_SUCCESS; } /* Before we can send the response, we want to modify it to a "200 OK" again, and produce some reasonable body output. */ -static int +static TSReturnCode on_send_response_header(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge) { TSMBuffer bufp; @@ -153,6 +150,7 @@ on_send_response_header(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge) } else { TSHttpTxnReenable(txnp, TS_EVENT_HTTP_ERROR); } + TSContDestroy(contp); return TS_SUCCESS; } @@ -170,10 +168,6 @@ purge_cont(TSCont contp, TSEvent event, void *edata) return on_send_response_header(txnp, contp, purge); break; - case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: - return on_http_cache_lookup_complete(txnp, contp, purge); - break; - default: TSDebug(PLUGIN_NAME, "Unexpected event: %d", event); break; @@ -219,11 +213,13 @@ handle_purge(TSHttpTxn txnp, PurgeInstance *purge) if (path && (path_len >= purge->secret_len)) { int s_path = path_len - 1; - while ((s_path >= 0) && ('/' != path[s_path])) { /* No memrchr in OSX */ + /* Find the last /, essentially memrchr (which does not exist on macOS) */ + while ((s_path >= 0) && ('/' != path[s_path])) { --s_path; } - if (!memcmp(s_path > 0 ? path + s_path + 1 : path, purge->secret, purge->secret_len)) { + if (((path_len - s_path - 1) == purge->secret_len) && + !memcmp(s_path > 0 ? path + s_path + 1 : path, purge->secret, purge->secret_len)) { should_purge = true; } } @@ -238,10 +234,12 @@ handle_purge(TSHttpTxn txnp, PurgeInstance *purge) if (should_purge) { TSCont cont = TSContCreate(purge_cont, TSMutexCreate()); + TSDebug(PLUGIN_NAME, "Setting up continuation for PURGE operation"); TSContDataSet(cont, purge); - TSHttpTxnHookAdd(txnp, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, cont); TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, cont); + update_purge_state(purge); } else if (purge->gen_id > 0) { + TSDebug(PLUGIN_NAME, "Setting request gen_id to %" PRId64, purge->gen_id); TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_CACHE_GENERATION, purge->gen_id); } } From da271d086682e02f344132019e9425ef0a4e5bd6 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 19 Nov 2018 21:56:37 +0000 Subject: [PATCH 032/526] Add test and fix regression with disable_h2 option --- iocore/net/YamlSNIConfig.cc | 6 +++--- iocore/net/YamlSNIConfig.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc index 5366e4b94ff..276b271cfa7 100644 --- a/iocore/net/YamlSNIConfig.cc +++ b/iocore/net/YamlSNIConfig.cc @@ -60,7 +60,7 @@ TsEnumDescriptor POLICY_DESCRIPTOR = {{{"DISABLED", 0}, {"PERMISSIVE", 1}, { TsEnumDescriptor PROPERTIES_DESCRIPTOR = {{{"NONE", 0}, {"SIGNATURE", 0x1}, {"NAME", 0x2}, {"ALL", 0x3}}}; std::set valid_sni_config_keys = {TS_fqdn, - TS_disable_H2, + TS_disable_h2, TS_verify_client, TS_tunnel_route, TS_verify_origin_server, @@ -88,8 +88,8 @@ template <> struct convert { } else { return false; // servername must be present } - if (node[TS_disable_H2]) { - item.disable_h2 = node[TS_disable_H2].as(); + if (node[TS_disable_h2]) { + item.disable_h2 = node[TS_disable_h2].as(); } // enum diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h index 0cb6d64b61d..444ae6b7968 100644 --- a/iocore/net/YamlSNIConfig.h +++ b/iocore/net/YamlSNIConfig.h @@ -28,7 +28,7 @@ #define TSDECL(id) constexpr char TS_##id[] = #id TSDECL(fqdn); -TSDECL(disable_H2); +TSDECL(disable_h2); TSDECL(verify_client); TSDECL(tunnel_route); TSDECL(verify_server_policy); From a28ef5e72316049ae5b56889ef84b5bb96313a20 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 19 Nov 2018 20:51:09 +0000 Subject: [PATCH 033/526] Fix and add tests for requiring TLS certs from user agent. --- iocore/net/SSLUtils.cc | 2 +- tests/gold_tests/tls/ssl/server.key | 43 ++++-- tests/gold_tests/tls/ssl/server.pem | 49 +++--- .../gold_tests/tls/tls_client_verify.test.py | 141 ++++++++++++++++++ .../gold_tests/tls/tls_client_verify2.test.py | 129 ++++++++++++++++ 5 files changed, 318 insertions(+), 46 deletions(-) create mode 100644 tests/gold_tests/tls/tls_client_verify.test.py create mode 100644 tests/gold_tests/tls/tls_client_verify2.test.py diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index bceae8b1cfb..c5a29156880 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -403,7 +403,7 @@ ssl_verify_client_callback(int preverify_ok, X509_STORE_CTX *ctx) SSLNetVConnection *netvc = SSLNetVCAccess(ssl); netvc->callHooks(TS_EVENT_SSL_VERIFY_CLIENT); - return SSL_TLSEXT_ERR_OK; + return preverify_ok; } // Use the certificate callback for openssl 1.0.2 and greater diff --git a/tests/gold_tests/tls/ssl/server.key b/tests/gold_tests/tls/ssl/server.key index 4c7a661a6bd..9cdfc36aa5f 100644 --- a/tests/gold_tests/tls/ssl/server.key +++ b/tests/gold_tests/tls/ssl/server.key @@ -1,15 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E -kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u -SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB -AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal -B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv -sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 -GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe -YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ -pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q -tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA -yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML -lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ -vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCZkEXSlZ+ZFKFg +CPpcDG39e73BuK6E5uE38q2PHh4DV0xcsJnIUx51viqLPwYughxfP0crHyBdXoHV +dW/3WX4gpiGrdiM/dvCouheo0DPaqUUJ2nZKVYh2M57oyeiuJidlKb7BGkfw3HWP +9TV7dVyGWok/cowjopqaLHJWxg/kh2KqvUBD0CHt9Kd1XvgXVmHwE7vCv0j5owv2 +MaExTsFb16uWmVLhl1gNHI2RqCX2yLaebH1DvtbLrit1XErjtaSYeJE9clVRaqT6 +vsvLOhyB5tA9WfZqfBYr/MHDeXQfrbIf+4Cp3aTpq5grc5InIMMH0eOk6/f/4tW+ +nq1lfszZAgMBAAECggEAYvYAqRbXRRVwca0Xel5gO2yU+tSDUw5esWlow8RK3yhR +A6KjV9+Iz6P/UsEIwMwEcLUcrgNfHgybau5Fe4dmqq+lHxQA3xNNP869FIMoB4/x +98mbVYgNau8VRztnAWOBG8ZtMZA4MFZCRMVm8+rL96E8tXCiMwzEyPo/rP/ymfhN +3GRunX+GhfIA79AYNbd7HMVL+cvWWUGUF5Bc5i1wXcLy4I7b9NYtv920BeCLzSFK +BypFB7ku/vKgTcBxe4yxThxPeXPwm4WFzGYKk/Afl1j8tVXCE2U4Y3yykfC0Qk6S +ECZbCKLO2Rxi9fclIDZBHWuKejZhdjHfjeNvZ2vLoQKBgQDJzLmkVLvWAxgl1yvF +U7gwqj/TzYqtVowbjEvTNEnPU1j/hIVI343SVV/EvJmif/iRUop6sRYfLsUjpMsH +CmPysNKL3UtgSYOxLs+0xLhG4OOQRpPSf/uvl9YyWY9G3AqiC7ScthkQjEhZa4c1 +eycYy0jr42kX0OL9MuIH9q0ENQKBgQDCzvGKMs8r5E/Qd3YB9VYB60dx+6G83AHZ +YqIelykObhCdxL9n4K+p4VKKLvgTcCOLYYIkBSWRJWR+ue3s3ey9+XWd2/q4Xvfh +TCjAuO2ibMV+y5ClNlW0fQ/doIVWSDbjO2tZW1jh7YWZ4CtuVrsEisv1sk3KltMB +MguhpTUylQKBgG6TfrncMFzxrx61C+gBmvEXqQffHfkjbnx94OKnSTaQ3jiNHhez +X9v8KhD8o1bWtpay2uyl8pA9qYqBdzqxZ9kJKSW4qd/mCIJjOy87iBpWint5IPD8 +biZmldlbF9ZlJnJq5ZnlclCN/er5r8oPZHoCkj+nieOh8294nUBt25ptAoGAMnPA +EIeaKgbmONpHgLhWPwb9KOL/f1cHT5KA5CVH58nPmdyTqcaCGCAX7Vu+ueIIApgN +SWDf2thxT3S9zuOm5YiO0oRfSZKm5f2AbHE4ciFzgKQd4PvSdH0TN9XT0oW/WVhR +NAI5YcHPIQvyk4/4vXNo4Uf9Z6NqIFwisQmFXoUCgYBK/ZI/HsFsvnR5MV0tFdGM +AdNe6bsQRSZkowoaPxuWbklE4Hn6QvwEmQg3ST2O+vCQV1f1yI6YiWYoptOYscJc +MSs/HxhhaaO5ZsiuPUO6WEPzpNb2CxuIGDDtl83VtUQyjxCmOb6pqqjwzFmZ2bsw +JNMaBCzokrJTxknvauCuSQ== +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/server.pem b/tests/gold_tests/tls/ssl/server.pem index 58b9b9715b7..2b56cc83ea2 100644 --- a/tests/gold_tests/tls/ssl/server.pem +++ b/tests/gold_tests/tls/ssl/server.pem @@ -1,32 +1,21 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E -kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u -SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB -AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal -B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv -sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 -GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe -YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ -pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q -tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA -yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML -lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ -vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z ------END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICszCCAhwCCQD4jSkztmlO1TANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC -VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh -aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u -ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j -b20wHhcNMTcwODI4MDM0NDQ1WhcNMjcwODI2MDM0NDQ1WjCBnTELMAkGA1UEBhMC -VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh -aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u -ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j -b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 -uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE -lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 -Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEATX7975NdhIbJ -glda+sXI9a86GgOpiuKO+vKubRJQZA+UlPf2vHEONjC2+7Y1aZvZYaKYL74vxGky -zkgp6ANSPl45lqD632x0e1Z7vzW5TkqK1JB2/xH2WgDcQZmP0FuQHzVNs4GjghDr -HCp1+sQDhfPB4aLmLFeyN0TkhdH1N3M= +MIIDZDCCAkygAwIBAgIJANod1+h9CtCaMA0GCSqGSIb3DQEBCwUAMEcxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRowGAYDVQQDDBFy +YW5kb20uc2VydmVyLmNvbTAeFw0xODExMTkxNzEwMTlaFw0yODExMTYxNzEwMTla +MEcxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRow +GAYDVQQDDBFyYW5kb20uc2VydmVyLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAJmQRdKVn5kUoWAI+lwMbf17vcG4roTm4TfyrY8eHgNXTFywmchT +HnW+Kos/Bi6CHF8/RysfIF1egdV1b/dZfiCmIat2Iz928Ki6F6jQM9qpRQnadkpV +iHYznujJ6K4mJ2UpvsEaR/DcdY/1NXt1XIZaiT9yjCOimposclbGD+SHYqq9QEPQ +Ie30p3Ve+BdWYfATu8K/SPmjC/YxoTFOwVvXq5aZUuGXWA0cjZGoJfbItp5sfUO+ +1suuK3VcSuO1pJh4kT1yVVFqpPq+y8s6HIHm0D1Z9mp8Fiv8wcN5dB+tsh/7gKnd +pOmrmCtzkicgwwfR46Tr9//i1b6erWV+zNkCAwEAAaNTMFEwHQYDVR0OBBYEFI2y +qm0+UAChDAnLrAINeFOuyUlhMB8GA1UdIwQYMBaAFI2yqm0+UAChDAnLrAINeFOu +yUlhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAA3ZNFbqxcOX +szS5A4EXCepyBJBFejEYy0CsvwQX/ai/pMrw5jqVeF0GAOTpBCVLddyY+ZV1arD2 +Pqi7Qwot9OxEZOzbCBiuMJGotruKgnWFQDHzJ9HA7KDQs270uNESAOG/xW9os9zN +MXApzqfBSR5EIQU5L3RtaiPzoKdQenGQUOj86s0Kon7snDSUzaA2VcfstMWgGvXP +JHtaVusULm0gry32cEap5G5UK+gII6DfLWgFwFGhHHmTz3mKjyGiJQ+09XBtu4lb +ENE+HGRBBA49dUKSr3kwErO4HyHnS0YrsTDnbYURCsGUDma12oijX2sCos6Q4zn8 +3svaouRrucw= -----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py new file mode 100644 index 00000000000..d6298531370 --- /dev/null +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -0,0 +1,141 @@ +''' +Test requiring certificate from user agent +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test various options for requiring certificate from client for mutual authentication TLS +''' + +Test.SkipUnless(Condition.HasProgram("grep", "grep needs to be installed on system for this test to work")) + +ts = Test.MakeATSProcess("ts", select_ports=False) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.ssl.client.certification_level': 2, + 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.TLSv1_3': 0 +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Just map everything through to origin. This test is concentratign on the user-agent side +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}/'.format(server.Variables.Port) +) + +# Scenario 1: Default no client cert required. cert required for bar.com +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + ' verify_client: NONE', +]) + +# to foo.com w/o client cert. Should fail +tr = Test.AddTestRun("Connect to foo.com without cert") +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.StartBefore(server) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to foo.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +# Should fail with badly signed certs +tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to foo.com with cert") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-foo.pem --key signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bar.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bar.com with cert") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bar.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +# Should fail with badly signed certs +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + diff --git a/tests/gold_tests/tls/tls_client_verify2.test.py b/tests/gold_tests/tls/tls_client_verify2.test.py new file mode 100644 index 00000000000..8848dc24acd --- /dev/null +++ b/tests/gold_tests/tls/tls_client_verify2.test.py @@ -0,0 +1,129 @@ +''' +Test requiring certificate from user agent +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test various options for requiring certificate from client for mutual authentication TLS +''' + +Test.SkipUnless(Condition.HasProgram("grep", "grep needs to be installed on system for this test to work")) + +ts = Test.MakeATSProcess("ts", select_ports=False) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.ssl.client.certification_level': 0, + 'proxy.config.ssl.CA.cert.path': '', + 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir) +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Just map everything through to origin. This test is concentratign on the user-agent side +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}/'.format(server.Variables.Port) +) + +# Scenario 1: Default no client cert required. cert required for bar.com +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + ' verify_client: STRICT', +]) + +# to foo.com w/o client cert. Should succeed +tr = Test.AddTestRun("Connect to foo.com without cert") +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.StartBefore(server) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to foo.com with cert") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-foo.pem --key signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bar.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bar.com with cert") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bar.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +# Should fail with badly signed certs +tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") +tr.TimeOut = 5 + From 8c7d6f75ab4951f8bc50a27f6a409c85884f6a6d Mon Sep 17 00:00:00 2001 From: scw00 Date: Tue, 20 Nov 2018 11:22:58 +0800 Subject: [PATCH 034/526] Use LoopTailHandler in UDPNetHandler to support signalActivity --- iocore/net/P_UDPNet.h | 6 +++- iocore/net/UnixUDPNet.cc | 65 +++++++++++++++++++++++++++++++++------- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index 8d803a022ad..9e7d869c42e 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -303,7 +303,7 @@ class UDPQueue void initialize_thread_for_udp_net(EThread *thread); -class UDPNetHandler : public Continuation +class UDPNetHandler : public Continuation, public EThread::LoopTailHandler { public: // engine for outgoing packets @@ -319,12 +319,16 @@ class UDPNetHandler : public Continuation Que(UnixUDPConnection, callback_link) udp_callbacks; Event *trigger_event = nullptr; + EThread *thread = nullptr; ink_hrtime nextCheck; ink_hrtime lastCheck; int startNetEvent(int event, Event *data); int mainNetEvent(int event, Event *data); + int waitForActivity(ink_hrtime timeout) override; + void signalActivity() override; + UDPNetHandler(); }; diff --git a/iocore/net/UnixUDPNet.cc b/iocore/net/UnixUDPNet.cc index 3f73b1bb08c..529a38a23d8 100644 --- a/iocore/net/UnixUDPNet.cc +++ b/iocore/net/UnixUDPNet.cc @@ -66,8 +66,14 @@ initialize_thread_for_udp_net(EThread *thread) new ((ink_dummy_for_new *)get_UDPPollCont(thread)) PollCont(thread->mutex); // The UDPNetHandler cannot be accessed across EThreads. // Because the UDPNetHandler should be called back immediately after UDPPollCont. - nh->mutex = thread->mutex.get(); - + nh->mutex = thread->mutex.get(); + nh->thread = thread; + + PollCont *upc = get_UDPPollCont(thread); + PollDescriptor *upd = upc->pollDescriptor; + // due to ET_UDP is really simple, it should sleep for a long time + // TODO: fixed size + upc->poll_timeout = 100; // This variable controls how often we cleanup the cancelled packets. // If it is set to 0, then cleanup never occurs. REC_ReadConfigInt32(g_udp_periodicFreeCancelledPkts, "proxy.config.udp.free_cancelled_pkts_sec"); @@ -82,8 +88,15 @@ initialize_thread_for_udp_net(EThread *thread) REC_ReadConfigInt32(g_udp_numSendRetries, "proxy.config.udp.send_retries"); g_udp_numSendRetries = g_udp_numSendRetries < 0 ? 0 : g_udp_numSendRetries; - thread->schedule_every(get_UDPPollCont(thread), -9); - thread->schedule_imm(get_UDPNetHandler(thread)); + thread->set_tail_handler(nh); + thread->ep = (EventIO *)ats_malloc(sizeof(EventIO)); + new (thread->ep) EventIO(); + thread->ep->type = EVENTIO_ASYNC_SIGNAL; +#if HAVE_EVENTFD + thread->ep->start(upd, thread->evfd, nullptr, EVENTIO_READ); +#else + thread->ep->start(upd, thread->evpipe[0], nullptr, EVENTIO_READ); +#endif } int @@ -890,6 +903,20 @@ UDPQueue::send(UDPPacket *p) #undef LINK +static void +net_signal_hook_callback(EThread *thread) +{ +#if HAVE_EVENTFD + uint64_t counter; + ATS_UNUSED_RETURN(read(thread->evfd, &counter, sizeof(uint64_t))); +#elif TS_USE_PORT +/* Nothing to drain or do */ +#else + char dummy[1024]; + ATS_UNUSED_RETURN(read(thread->evpipe[0], &dummy[0], 1024)); +#endif +} + UDPNetHandler::UDPNetHandler() { nextCheck = Thread::get_hrtime_updated() + HRTIME_MSECONDS(1000); @@ -911,10 +938,15 @@ int UDPNetHandler::mainNetEvent(int event, Event *e) { ink_assert(trigger_event == e && event == EVENT_POLL); - (void)event; - (void)e; + return this->waitForActivity(net_config_poll_timeout); +} +int +UDPNetHandler::waitForActivity(ink_hrtime timeout) +{ UnixUDPConnection *uc; + PollCont *pc = get_UDPPollCont(this->thread); + pc->do_poll(timeout); /* Notice: the race between traversal of newconn_list and UDPBind() * @@ -943,7 +975,6 @@ UDPNetHandler::mainNetEvent(int event, Event *e) // handle UDP read operations int i, nread = 0; - PollCont *pc = get_UDPPollCont(e->ethread); EventIO *epd = nullptr; for (i = 0; i < pc->pollDescriptor->result; i++) { epd = (EventIO *)get_ev_data(pc->pollDescriptor, i); @@ -973,8 +1004,7 @@ UDPNetHandler::mainNetEvent(int event, Event *e) #endif } } else if (epd->type == EVENTIO_ASYNC_SIGNAL) { - // TODO: receive signal from event system - // net_signal_hook_callback(this->trigger_event->ethread); + net_signal_hook_callback(this->thread); } } // end for @@ -997,7 +1027,7 @@ UDPNetHandler::mainNetEvent(int event, Event *e) udp_callbacks.clear(); while ((uc = q.dequeue())) { ink_assert(uc->mutex && uc->continuation); - if (udpNetInternal.udp_callback(this, uc, trigger_event->ethread)) { // not successful + if (udpNetInternal.udp_callback(this, uc, this->thread)) { // not successful // schedule on a thread of its own. ink_assert(uc->callback_link.next == nullptr); ink_assert(uc->callback_link.prev == nullptr); @@ -1012,3 +1042,18 @@ UDPNetHandler::mainNetEvent(int event, Event *e) return EVENT_CONT; } + +void +UDPNetHandler::signalActivity() +{ +#if HAVE_EVENTFD + uint64_t counter = 1; + ATS_UNUSED_RETURN(write(thread->evfd, &counter, sizeof(uint64_t))); +#elif TS_USE_PORT + PollDescriptor *pd = get_PollDescriptor(thread); + ATS_UNUSED_RETURN(port_send(pd->port_fd, 0, thread->ep)); +#else + char dummy = 1; + ATS_UNUSED_RETURN(write(thread->evpipe[1], &dummy, 1)); +#endif +} From a5f145d9ade9a2ba41448d15c4fd22381f4df4e2 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Sun, 11 Nov 2018 18:06:52 +0900 Subject: [PATCH 035/526] Unify cross event scheduling code --- proxy/http2/Http2Stream.cc | 55 +++++++++++++++++++++----------------- proxy/http2/Http2Stream.h | 13 ++++----- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 2e1e65e6882..429b55fdc5f 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -34,19 +34,16 @@ ClassAllocator http2StreamAllocator("http2StreamAllocator"); int Http2Stream::main_event_handler(int event, void *edata) { - Event *e = static_cast(edata); + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); - Thread *this_thread = this_ethread(); - if (this->get_thread() != this_thread) { - // Send on to the owning thread - if (cross_thread_event == nullptr) { - cross_thread_event = this->get_thread()->schedule_imm(this, event, edata); - } + if (!this->_switch_thread_if_not_on_right_thread(event, edata)) { + // Not on the right thread return 0; } + ink_release_assert(this->_thread == this_ethread()); + + Event *e = static_cast(edata); reentrancy_count++; - ink_release_assert(this->get_thread() == this_ethread()); - SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); if (e == cross_thread_event) { cross_thread_event = nullptr; } else if (e == active_event) { @@ -467,15 +464,13 @@ Http2Stream::update_read_request(int64_t read_len, bool call_update, bool check_ if (closed || parent == nullptr || current_reader == nullptr || read_vio.mutex == nullptr) { return; } - if (this->get_thread() != this_ethread()) { - SCOPED_MUTEX_LOCK(stream_lock, this->mutex, this_ethread()); - if (cross_thread_event == nullptr) { - // Send to the right thread - cross_thread_event = this->get_thread()->schedule_imm(this, VC_EVENT_READ_READY, nullptr); - } + + if (!this->_switch_thread_if_not_on_right_thread(VC_EVENT_READ_READY, nullptr)) { + // Not on the right thread return; } - ink_release_assert(this->get_thread() == this_ethread()); + ink_release_assert(this->_thread == this_ethread()); + SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); if (read_vio.nbytes > 0 && read_vio.ndone <= read_vio.nbytes) { // If this vio has a different buffer, we must copy @@ -532,15 +527,13 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, (buf_reader == nullptr && write_len == 0)) { return; } - if (this->get_thread() != this_ethread()) { - SCOPED_MUTEX_LOCK(stream_lock, this->mutex, this_ethread()); - if (cross_thread_event == nullptr) { - // Send to the right thread - cross_thread_event = this->get_thread()->schedule_imm(this, VC_EVENT_WRITE_READY, nullptr); - } + + if (!this->_switch_thread_if_not_on_right_thread(VC_EVENT_WRITE_READY, nullptr)) { + // Not on the right thread return; } - ink_release_assert(this->get_thread() == this_ethread()); + ink_release_assert(this->_thread == this_ethread()); + Http2ClientSession *parent = static_cast(this->get_parent()); SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); @@ -550,7 +543,7 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, if (this->chunked) { if (chunked_handler.dechunked_buffer && chunked_handler.dechunked_buffer->max_read_avail() > HTTP2_MAX_BUFFER_USAGE) { if (buffer_full_write_event == nullptr) { - buffer_full_write_event = get_thread()->schedule_imm(this, VC_EVENT_WRITE_READY); + buffer_full_write_event = _thread->schedule_imm(this, VC_EVENT_WRITE_READY); } } else { this->response_process_data(is_done); @@ -887,3 +880,17 @@ Http2Stream::release(IOBufferReader *r) current_reader = nullptr; // State machine is on its own way down. this->do_io_close(); } + +bool +Http2Stream::_switch_thread_if_not_on_right_thread(int event, void *edata) +{ + if (this->_thread != this_ethread()) { + SCOPED_MUTEX_LOCK(stream_lock, this->mutex, this_ethread()); + if (cross_thread_event == nullptr) { + // Send to the right thread + cross_thread_event = this->_thread->schedule_imm(this, event, edata); + } + return false; + } + return true; +} diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index b46134d8038..bc74c72bad9 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -180,12 +180,6 @@ class Http2Stream : public ProxyClientTransaction MIOBuffer request_buffer = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; Http2DependencyTree::Node *priority_node = nullptr; - EThread * - get_thread() - { - return _thread; - } - IOBufferReader *response_get_data_reader() const; bool response_is_chunked() const @@ -235,6 +229,13 @@ class Http2Stream : public ProxyClientTransaction Event *send_tracked_event(Event *event, int send_event, VIO *vio); void send_response_body(bool call_update); + /** + * Check if this thread is the right thread to process events for this + * continuation. + * Return true if the caller can continue event processing. + */ + bool _switch_thread_if_not_on_right_thread(int event, void *edata); + HTTPParser http_parser; ink_hrtime _start_time = 0; EThread *_thread = nullptr; From 1fa443cf0f1fcd1b29e2e7f1eeaf576521e3ff08 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 14 Nov 2018 12:56:12 -0600 Subject: [PATCH 036/526] TLS Bridge: Fix remap for UA connection, tweak docs for remap support. --- .../plugins/example-plugins/tls_bridge.en.rst | 42 ++++++++++++------- plugins/experimental/tls_bridge/tls_bridge.cc | 4 +- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst index 1b979fea4c0..0530afa194e 100644 --- a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst +++ b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst @@ -1,18 +1,14 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under the License. .. include:: ../../../common.defs @@ -26,7 +22,8 @@ This plugin is used to provide secured TLS tunnels for connections between a Client and a Service via two gateway |TS| instances. By configuring the |TS| instances the level of security in the -tunnel can be easily controlled for all communications across the tunnels. +tunnel can be easily controlled for all communications across the tunnels without having to update +the client or service. Description =========== @@ -94,6 +91,13 @@ Configuration to run in a restricted environment or use access control (via ``ip_allow.config`` or ``iptables``). + If this is unsuitable then an identity remap rule can be added for the peer |TS|. If the peer |TS| + was named "peer.ats" then the remap rule would be :: + + remap https://peer.ats https://peer.ats + + Remapping will be disabled for the user agent connection and so it will not need a rule. + #. Configure the Ingress |TS| to verify the Peer server certificate:: CONFIG proxy.config.ssl.client.verify.server.policy STRING ENFORCED @@ -114,7 +118,7 @@ Configuration #. Enable the |Name| plugin in ``plugin.config``. The plugin is configured by arguments in ``plugin.config``. These are arguments are in pairs of a *destination* and a *peer*. The - destination is a anchored regular expression which is matched against the host name in the Client + destination is an anchored regular expression which is matched against the host name in the Client ``CONNECT``. The destinations are checked in order and the first match is used to select the Peer |TS|. The peer should be an FQDN or IP address with an optional port. For the example above, if the Peer |TS| was named "peer.example.com" on port 4443 and the Service at ``*.service.com``, the @@ -122,6 +126,9 @@ Configuration tls_bridge.so .*[.]service[.]com peer.example.com:4443 + Note the '.' characters are escaped with brackets so that, for instance, "someservice.com" does + not match the rule. + Notes ===== @@ -161,7 +168,7 @@ If the session is valid then a separate connection to the peer |TS| is created u :code:`TSHttpConnect`. After the ingress |TS| connects to the peer |TS| it sends a duplicate of the Client ``CONNECT`` -request. This is processed by the peer |TS| to connect on to the Service. After this both |TS| +request. This is processed by the peer |TS| to connect to the Service. After this both |TS| instances then tunnel data between the Client and the Service, in effect becoming a transparent tunnel. @@ -243,3 +250,8 @@ socket read. State_3 --> State_1 : CR State_3 --> State_0 : * @enduml + +Debugging +--------- + +Debugging messages for the plugin can be enabled with the "tls_bridge" debug tag. diff --git a/plugins/experimental/tls_bridge/tls_bridge.cc b/plugins/experimental/tls_bridge/tls_bridge.cc index 0a699265700..94657ac1136 100644 --- a/plugins/experimental/tls_bridge/tls_bridge.cc +++ b/plugins/experimental/tls_bridge/tls_bridge.cc @@ -24,7 +24,7 @@ #include "regex.h" #define PLUGIN_NAME "TLS Bridge" -#define PLUGIN_TAG "tls-bridge" +#define PLUGIN_TAG "tls_bridge" using ts::TextView; @@ -606,6 +606,8 @@ CB_Read_Request_Hdr(TSCont contp, TSEvent ev_idx, void *data) TSHttpTxnHookAdd(txn, TS_HTTP_SEND_RESPONSE_HDR_HOOK, actor); // Arrange for cleanup. TSHttpTxnHookAdd(txn, TS_HTTP_TXN_CLOSE_HOOK, actor); + // Skip remap and remap rule requirement - authorized by TLS bridge config. + TSSkipRemappingSet(txn, 1); // Grab the transaction TSHttpTxnIntercept(actor, txn); } From 9cc7b3a7b3a5c908afd2c4938b868b938c124aa4 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Sun, 25 Nov 2018 14:49:56 -0600 Subject: [PATCH 037/526] Issue #4637: Clean up / unify hex conversions. Remove hex conversion in logging that is not used. --- CMakeLists.txt | 1 + proxy/logging/LogFieldAliasMap.h | 30 -------------------------- proxy/logging/LogUtils.cc | 36 -------------------------------- 3 files changed, 1 insertion(+), 66 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8abf0eac84..5fa91bc3303 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,7 @@ CPP_ADD_SOURCES(proxy proxy/http) CPP_ADD_SOURCES(proxy proxy/http2) CPP_ADD_SOURCES(proxy proxy/http/remap) CPP_ADD_SOURCES(proxy proxy/hdrs) +CPP_ADD_SOURCES(proxy proxy/logging) CPP_LIB(iocore iocore iocore) CPP_ADD_SOURCES(iocore iocore/eventsystem) diff --git a/proxy/logging/LogFieldAliasMap.h b/proxy/logging/LogFieldAliasMap.h index 1d605770253..9f69cc36a50 100644 --- a/proxy/logging/LogFieldAliasMap.h +++ b/proxy/logging/LogFieldAliasMap.h @@ -183,34 +183,4 @@ class LogFieldAliasTable : public LogFieldAliasMap } }; -/***************************************************************************** - -The LogFieldAliasTimehex class implements a LogFieldAliasMap that converts time -from their integer value to the "hex" notation and back. - - *****************************************************************************/ - -class LogFieldAliasTimeHex : public LogFieldAliasMap -{ -public: - int - asInt(char *str, IntType *time, bool /* case_sensitive ATS_UNUSED */) const override - { - unsigned long a; - // coverity[secure_coding] - if (sscanf(str, "%lx", (unsigned long *)&a) == 1) { - *time = (IntType)a; - return ALL_OK; - } else { - return INVALID_STRING; - } - } - - int - asString(IntType time, char *buf, size_t bufLen, size_t *numCharsPtr = nullptr) const override - { - return (LogUtils::timestamp_to_hex_str(time, buf, bufLen, numCharsPtr) ? BUFFER_TOO_SMALL : ALL_OK); - } -}; - // LOG_FIELD_ALIAS_MAP_H diff --git a/proxy/logging/LogUtils.cc b/proxy/logging/LogUtils.cc index 94becf250ac..7b4146362c7 100644 --- a/proxy/logging/LogUtils.cc +++ b/proxy/logging/LogUtils.cc @@ -434,42 +434,6 @@ LogUtils::remove_content_type_attributes(char *type_str, int *type_len) } } -/*------------------------------------------------------------------------- - LogUtils::timestamp_to_hex_str - - This routine simply writes the given timestamp integer [time_t] in the equivalent - hexadecimal string format "xxxxxxxxxx" into the provided buffer [buf] of - size [bufLen]. - - It returns 1 if the provided buffer is not big enough to hold the - equivalent ip string (and its null terminator), and 0 otherwise. - If the buffer is not big enough, only the ip "segments" that completely - fit into it are written, and the buffer is null terminated. - -------------------------------------------------------------------------*/ - -int -LogUtils::timestamp_to_hex_str(unsigned ip, char *buf, size_t bufLen, size_t *numCharsPtr) -{ - static const char *table = "0123456789abcdef@"; - int retVal = 1; - int shift = 28; - if (buf && bufLen > 0) { - if (bufLen > 8) { - bufLen = 8; - } - for (retVal = 0; retVal < (int)bufLen;) { - buf[retVal++] = (char)table[((ip >> shift) & 0xf)]; - shift -= 4; - } - - if (numCharsPtr) { - *numCharsPtr = (size_t)retVal; - } - retVal = (retVal == 8) ? 0 : 1; - } - return retVal; -} - /* int LogUtils::ip_to_str (unsigned ip, char *str, unsigned len) From 6665379c0f1090a6c4caac770139c689aa9f7408 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 26 Nov 2018 08:56:12 -0600 Subject: [PATCH 038/526] Docs: Fix proxy.config.net.connections_throttle text. --- doc/admin-guide/files/records.config.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 1638255339e..c14df8842a7 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -396,7 +396,7 @@ Network can handle simultaneously. This is in fact the max number of file descriptors that the :program:`traffic_server` process can have open at any given time. Roughly 10% of these connections are reserved for origin server - connections, i.e. from the default, only ~9,000 client connections can be + connections, i.e. from the default, only ~27,000 client connections can be handled. This should be tuned according to your memory size, and expected work load. If this is set to 0, the throttling logic is disabled. From 2602da13eebe80d873ae3045e8c3da9bade16ac6 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Mon, 12 Nov 2018 10:29:35 -0600 Subject: [PATCH 039/526] Cleanup: replace multiple HashMap with STL unordered_map --- iocore/net/P_SNIActionPerformer.h | 9 +-- iocore/net/P_SSLConfig.h | 2 +- iocore/net/P_SSLSNI.h | 9 +-- iocore/net/SNIActionPerformer.cc | 2 +- iocore/net/SSLSNIConfig.cc | 104 +++++++++++++----------------- iocore/net/SSLUtils.cc | 11 ++-- 6 files changed, 61 insertions(+), 76 deletions(-) diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 001ac00a8b5..488c1cdaa8e 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -31,13 +31,13 @@ #pragma once #include "I_EventSystem.h" -#include "tscore/Map.h" //#include"P_UnixNetProcessor.h" #include #include "P_SSLNextProtocolAccept.h" #include "tscore/ink_inet.h" +#include -extern Map snpsMap; +extern std::unordered_map snpsMap; // enum of all the actions enum AllActions { TS_DISABLE_H2 = 0, @@ -67,8 +67,9 @@ class DisableH2 : public ActionItem auto ssl_vc = reinterpret_cast(cont); auto accept_obj = ssl_vc ? ssl_vc->accept_object : nullptr; if (accept_obj && accept_obj->snpa && ssl_vc) { - auto nps = snpsMap.get(accept_obj->id); - ssl_vc->registerNextProtocolSet(reinterpret_cast(nps)); + if (auto it = snpsMap.find(accept_obj->id); it != snpsMap.end()) { + ssl_vc->registerNextProtocolSet(it->second); + } } return SSL_TLSEXT_ERR_OK; } diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 5ea0c955096..aa86ad9ce0f 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -123,7 +123,7 @@ struct SSLConfigParams : public ConfigInfo { SSL_CTX *client_ctx; - mutable HashMap ctx_map; + mutable std::unordered_map ctx_map; mutable ink_mutex ctxMapLock; SSL_CTX *getClientSSL_CTX(void) const; diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index 33543da8c53..05318a71915 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -31,7 +31,6 @@ #pragma once #include "ProxyConfig.h" -#include "tscore/Map.h" #include "P_SNIActionPerformer.h" #include "tscore/MatcherUtils.h" #include "openssl/ossl_typ.h" @@ -39,6 +38,8 @@ #include #include "YamlSNIConfig.h" +#include + // Properties for the next hop server struct NextHopProperty { const char *name = nullptr; // name of the server @@ -48,9 +49,9 @@ struct NextHopProperty { NextHopProperty(); }; -typedef std::vector actionVector; -typedef HashMap SNIMap; -typedef HashMap NextHopPropertyTable; +using actionVector = std::vector; +using SNIMap = std::unordered_map; +using NextHopPropertyTable = std::unordered_map; struct SNIConfigParams : public ConfigInfo { char *sni_filename = nullptr; diff --git a/iocore/net/SNIActionPerformer.cc b/iocore/net/SNIActionPerformer.cc index d381e061009..fa5116ee361 100644 --- a/iocore/net/SNIActionPerformer.cc +++ b/iocore/net/SNIActionPerformer.cc @@ -35,7 +35,7 @@ #include "P_SSLNextProtocolAccept.h" #include "P_SSLUtils.h" -extern Map snpsMap; +extern std::unordered_map snpsMap; int SNIActionPerformer::PerformAction(Continuation *cont, cchar *servername) diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 7813287c4de..aa963a3a971 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -38,19 +38,20 @@ static ConfigUpdateHandler *sniConfigUpdate; struct NetAccept; -Map snpsMap; +std::unordered_map snpsMap; extern TunnelHashMap TunnelMap; NextHopProperty::NextHopProperty() {} NextHopProperty * SNIConfigParams::getPropertyConfig(cchar *servername) const { - NextHopProperty *nps = nullptr; - nps = next_hop_table.get(servername); - if (!nps) { - nps = wild_next_hop_table.get(servername); + if (auto it = next_hop_table.find(servername); it != next_hop_table.end()) { + return it->second; } - return nps; + if (auto it = wild_next_hop_table.find(servername); it != wild_next_hop_table.end()) { + return it->second; + } + return nullptr; } void @@ -72,10 +73,10 @@ SNIConfigParams::loadSNIConfig() ts::TextView domain{servername, strlen(servername)}; domain.take_prefix_at('.'); if (!domain.empty()) { - wild_sni_action_map.put(ats_stringdup(domain), aiVec); + wild_sni_action_map.emplace(domain, aiVec); } } else { - sni_action_map.put(ats_strdup(servername), aiVec); + sni_action_map.emplace(servername, aiVec); } if (item.tunnel_destination.length()) { @@ -92,15 +93,18 @@ SNIConfigParams::loadSNIConfig() if (certFile) { clientCTX = params->getNewCTX(certFile, keyFile); } - NextHopProperty *nps = new NextHopProperty(); - nps->name = ats_strdup(servername); - nps->verifyServerPolicy = item.verify_server_policy; - nps->verifyServerProperties = item.verify_server_properties; - nps->ctx = clientCTX; - if (wildcard) { - wild_next_hop_table.put(nps->name, nps); - } else { - next_hop_table.put(nps->name, nps); + if (servername) { // a safety check + NextHopProperty *nps = new NextHopProperty(); + + nps->name = ats_strdup(servername); + nps->verifyServerPolicy = item.verify_server_policy; + nps->verifyServerProperties = item.verify_server_properties; + nps->ctx = clientCTX; + if (wildcard) { + wild_next_hop_table.emplace(nps->name, nps); + } else { + next_hop_table.emplace(nps->name, nps); + } } } // end for } @@ -112,29 +116,25 @@ SNIConfigParams::SNIConfigParams() {} actionVector * SNIConfigParams::get(cchar *servername) const { - auto actionVec = sni_action_map.get(servername); - if (!actionVec) { - Vec keys; - wild_sni_action_map.get_keys(keys); - for (int i = 0; i < static_cast(keys.length()); i++) { - std::string_view sv{servername, strlen(servername)}; - std::string_view key_sv{keys.get(i)}; + auto action_it = sni_action_map.find(servername); + if (action_it != sni_action_map.end()) { + for (const auto &it : wild_sni_action_map) { + std::string_view sv{servername}; + std::string_view key_sv{it.first}; if (sv.size() >= key_sv.size() && sv.substr(sv.size() - key_sv.size()) == key_sv) { - return wild_sni_action_map.get(key_sv.data()); + auto wild_action_it = wild_sni_action_map.find(key_sv.data()); + return wild_action_it != wild_sni_action_map.end() ? wild_action_it->second : nullptr; } } } - return actionVec; + return action_it != sni_action_map.end() ? action_it->second : nullptr; } void SNIConfigParams::printSNImap() const { - Vec keys; - sni_action_map.get_keys(keys); - for (size_t i = 0; i < keys.length(); i++) { - Debug("ssl", "Domain name in the map %s: # of registered action items %lu", (char *)keys.get(i), - sni_action_map.get(keys.get(i))->size()); + for (const auto &it : sni_action_map) { + Debug("ssl", "Domain name in the map %s: # of registered action items %lu", it.first.c_str(), it.second->size()); } } @@ -169,42 +169,26 @@ SNIConfigParams::Initialize() void SNIConfigParams::cleanup() { - Vec keys; - sni_action_map.get_keys(keys); - for (int i = keys.length() - 1; i >= 0; i--) { - auto actionVec = sni_action_map.get(keys.get(i)); - for (auto &ai : *actionVec) { + for (const auto &it : sni_action_map) { + auto actionVec = it.second; + for (const auto &ai : *actionVec) { delete ai; } - - actionVec->clear(); + delete actionVec; } - keys.free_and_clear(); - - wild_sni_action_map.get_keys(keys); - for (int i = keys.length() - 1; i >= 0; i--) { - auto actionVec = wild_sni_action_map.get(keys.get(i)); - for (auto &ai : *actionVec) { + for (const auto &it : wild_sni_action_map) { + auto actionVec = it.second; + for (const auto &ai : *actionVec) { delete ai; } - - actionVec->clear(); + delete actionVec; } - keys.free_and_clear(); - - next_hop_table.get_keys(keys); - for (int i = 0; i < static_cast(keys.length()); i++) { - auto *nps = next_hop_table.get(keys.get(i)); - delete (nps); + for (const auto &it : next_hop_table) { + delete it.second; } - keys.free_and_clear(); - - wild_next_hop_table.get_keys(keys); - for (int i = 0; static_cast(keys.length()); i++) { - auto *nps = wild_next_hop_table.get(keys.get(i)); - delete (nps); + for (const auto &it : wild_next_hop_table) { + delete it.second; } - keys.free_and_clear(); } SNIConfigParams::~SNIConfigParams() @@ -229,7 +213,7 @@ SNIConfig::cloneProtoSet() if (na->snpa) { auto snps = na->snpa->cloneProtoSet(); snps->unregisterEndpoint(TS_ALPN_PROTOCOL_HTTP_2_0, nullptr); - snpsMap.put(na->id, snps); + snpsMap.emplace(na->id, snps); } } } diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index c5a29156880..8830d7054ec 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -140,7 +140,7 @@ static ink_mutex *mutex_buf = nullptr; static bool open_ssl_initialized = false; RecRawStatBlock *ssl_rsb = nullptr; -HashMap cipher_map; +std::unordered_map cipher_map; /* Using pthread thread ID and mutex functions directly, instead of * ATS this_ethread / ProxyMutex, so that other linked libraries @@ -1137,8 +1137,8 @@ SSLInitializeStatistics() } // If not already registered ... - if (0 == cipher_map.get(cipherName)) { - cipher_map.put(cipherName, (intptr_t)(ssl_cipher_stats_start + index)); + if (cipherName && cipher_map.find(cipherName) == cipher_map.end()) { + cipher_map.emplace(cipherName, (intptr_t)(ssl_cipher_stats_start + index)); // Register as non-persistent since the order/index is dependent upon configuration. RecRegisterRawStat(ssl_rsb, RECT_PROCESS, statName.c_str(), RECD_INT, RECP_NON_PERSISTENT, (int)ssl_cipher_stats_start + index, RecRawStatSyncSum); @@ -1550,9 +1550,8 @@ ssl_callback_info(const SSL *ssl, int where, int ret) if (cipher) { const char *cipherName = SSL_CIPHER_get_name(cipher); // lookup index of stat by name and incr count - auto data = cipher_map.get(cipherName); - if (data != 0) { - SSL_INCREMENT_DYN_STAT((intptr_t)data); + if (auto it = cipher_map.find(cipherName); it != cipher_map.end()) { + SSL_INCREMENT_DYN_STAT((intptr_t)it->second); } } } From 66610170f2b55dba968172b961288af9710b0839 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Tue, 13 Nov 2018 16:06:42 -0600 Subject: [PATCH 040/526] hostdb: Replace the last TSHashTable with IntrusiveHashMap. --- iocore/hostdb/HostDB.cc | 7 +- iocore/hostdb/P_RefCountCache.h | 116 ++++++++++++++++---------------- 2 files changed, 61 insertions(+), 62 deletions(-) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 5f76b1c96bf..6e50c443d3d 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -1558,14 +1558,13 @@ HostDBContinuation::iterateEvent(int event, Event *e) return EVENT_CONT; } - TSHashTable *partMap = hostDB.refcountcache->get_partition(current_iterate_pos).get_map(); - for (RefCountCachePartition::iterator_type i = partMap->begin(); i != partMap->end(); ++i) { - HostDBInfo *r = (HostDBInfo *)i.m_value->item.get(); + IntrusiveHashMap &partMap = hostDB.refcountcache->get_partition(current_iterate_pos).get_map(); + for (const auto &it : partMap) { + HostDBInfo *r = static_cast(it.item.get()); if (r && !r->is_failed()) { action.continuation->handleEvent(EVENT_INTERVAL, static_cast(r)); } } - current_iterate_pos++; // And reschedule ourselves to pickup the next bucket after HOST_DB_RETRY_PERIOD. Debug("hostdb", "iterateEvent event=%d eventp=%p: completed current iteration %ld of %ld", event, e, current_iterate_pos, diff --git a/iocore/hostdb/P_RefCountCache.h b/iocore/hostdb/P_RefCountCache.h index 5e1b7d6675f..8655d345607 100644 --- a/iocore/hostdb/P_RefCountCache.h +++ b/iocore/hostdb/P_RefCountCache.h @@ -25,7 +25,7 @@ #include #include // TODO: less? just need ET_TASK -#include "tscore/Map.h" +#include "tscore/IntrusiveHashMap.h" #include "tscore/PriorityQueue.h" #include "tscore/List.h" @@ -75,7 +75,8 @@ class RefCountCacheHashEntry { public: Ptr item; - LINK(RefCountCacheHashEntry, item_link); + RefCountCacheHashEntry *_next{nullptr}; + RefCountCacheHashEntry *_prev{nullptr}; PriorityQueueEntry *expiry_entry; RefCountCacheItemMeta meta; @@ -115,24 +116,32 @@ class RefCountCacheHashEntry // Since the hashing values are all fixed size, we can simply use a classAllocator to avoid mallocs extern ClassAllocator> expiryQueueEntry; -struct RefCountCacheHashing { - typedef uint64_t ID; - typedef uint64_t const Key; - typedef RefCountCacheHashEntry Value; - typedef DList(RefCountCacheHashEntry, item_link) ListHead; +struct RefCountCacheLinkage { + using key_type = uint64_t const; + using value_type = RefCountCacheHashEntry; - static ID - hash(Key key) + static value_type *& + next_ptr(value_type *value) + { + return value->_next; + } + static value_type *& + prev_ptr(value_type *value) + { + return value->_prev; + } + static uint64_t + hash_of(key_type key) { return key; } - static Key - key(Value const *value) + static key_type + key_of(value_type *v) { - return value->meta.key; + return v->meta.key; } static bool - equal(Key lhs, Key rhs) + equal(key_type lhs, key_type rhs) { return lhs == rhs; } @@ -143,6 +152,8 @@ struct RefCountCacheHashing { template class RefCountCachePartition { public: + using hash_type = IntrusiveHashMap; + RefCountCachePartition(unsigned int part_num, uint64_t max_size, unsigned int max_items, RecRawStatBlock *rsb = nullptr); Ptr get(uint64_t key); void put(uint64_t key, C *item, int size = 0, int expire_time = 0); @@ -151,15 +162,12 @@ template class RefCountCachePartition void clear(); bool is_full() const; bool make_space_for(unsigned int); - template void dealloc_entry(Iterator ptr); + void dealloc_entry(hash_type::iterator ptr); size_t count() const; void copy(std::vector &items); - typedef typename TSHashTable::iterator iterator_type; - typedef typename TSHashTable::self hash_type; - typedef typename TSHashTable::Location location_type; - TSHashTable *get_map(); + hash_type &get_map(); Ptr lock; // Lock @@ -190,11 +198,10 @@ Ptr RefCountCachePartition::get(uint64_t key) { this->metric_inc(refcountcache_total_lookups_stat, 1); - location_type l = this->item_map.find(key); - if (l.isValid()) { + if (auto it = this->item_map.find(key); it != this->item_map.end()) { // found this->metric_inc(refcountcache_total_hits_stat, 1); - return make_ptr((C *)l.m_value->item.get()); + return make_ptr(static_cast(it->item.get())); } else { return Ptr(); } @@ -241,44 +248,37 @@ template void RefCountCachePartition::erase(uint64_t key, ink_time_t expiry_time) { - location_type l = this->item_map.find(key); - if (l.isValid()) { - if (expiry_time >= 0 && l.m_value->meta.expiry_time != expiry_time) { + if (auto it = this->item_map.find(key); it != this->item_map.end()) { + if (expiry_time >= 0 && it->meta.expiry_time != expiry_time) { return; } - - // TSHashMap does NOT clean up the item-- this remove just removes it from the map - // we are responsible for cleaning it up here - this->item_map.remove(l); - this->dealloc_entry(l); + this->item_map.erase(it); + this->dealloc_entry(it); } } template -template void -RefCountCachePartition::dealloc_entry(Iterator ptr) +RefCountCachePartition::dealloc_entry(hash_type::iterator ptr) { - if (ptr.m_value) { - // decrement usag are not cleaned up. The values are not touched in this method, therefore it is safe - // counters - this->size -= ptr->meta.size; - this->items--; - - this->metric_inc(refcountcache_current_size_stat, -((int64_t)ptr->meta.size)); - this->metric_inc(refcountcache_current_items_stat, -1); - - // remove from expiry queue - if (ptr->expiry_entry != nullptr) { - Debug("refcountcache", "partition %d deleting item from expiry_queue idx=%d", this->part_num, ptr->expiry_entry->index); - - this->expiry_queue.erase(ptr->expiry_entry); - expiryQueueEntry.free(ptr->expiry_entry); - ptr->expiry_entry = nullptr; // To avoid the destruction of `l` calling the destructor again-- and causing issues - } + // decrement usage are not cleaned up. The values are not touched in this method, therefore it is safe + // counters + this->size -= ptr->meta.size; + this->items--; + + this->metric_inc(refcountcache_current_size_stat, -((int64_t)ptr->meta.size)); + this->metric_inc(refcountcache_current_items_stat, -1); - RefCountCacheHashEntry::free(ptr.m_value); + // remove from expiry queue + if (ptr->expiry_entry != nullptr) { + Debug("refcountcache", "partition %d deleting item from expiry_queue idx=%d", this->part_num, ptr->expiry_entry->index); + + this->expiry_queue.erase(ptr->expiry_entry); + expiryQueueEntry.free(ptr->expiry_entry); + ptr->expiry_entry = nullptr; // To avoid the destruction of `l` calling the destructor again-- and causing issues } + + RefCountCacheHashEntry::free(ptr); } template @@ -288,12 +288,12 @@ RefCountCachePartition::clear() // Since the hash nodes embed the list pointers, you can't iterate over the // hash elements and deallocate them, let alone remove them from the hash. // Hence, this monstrosity. - while (this->item_map.count() > 0) { - location_type pos = this->item_map.find(this->item_map.begin().m_value); + auto it = this->item_map.begin(); + while (it != this->item_map.end()) { + auto cur = it; - ink_assert(pos.isValid()); - this->item_map.remove(pos); - this->dealloc_entry(pos); + it = this->item_map.erase(it); + this->dealloc_entry(cur); } } @@ -341,9 +341,9 @@ template void RefCountCachePartition::copy(std::vector &items) { - for (RefCountCachePartition::iterator_type i = this->item_map.begin(); i != this->item_map.end(); ++i) { + for (auto &&it : this->item_map) { RefCountCacheHashEntry *val = RefCountCacheHashEntry::alloc(); - val->set(i.m_value->item.get(), i.m_value->meta.key, i.m_value->meta.size, i.m_value->meta.expiry_time); + val->set(it.item.get(), it.meta.key, it.meta.size, it.meta.expiry_time); items.push_back(val); } } @@ -358,10 +358,10 @@ RefCountCachePartition::metric_inc(RefCountCache_Stats metric_enum, int64_t d } template -TSHashTable * +IntrusiveHashMap & RefCountCachePartition::get_map() { - return &this->item_map; + return this->item_map; } // The header for the cache, this is used to check if the serialized cache is compatible From e137c7d3941be638701a2a2b9427a3a2f9424be1 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Tue, 13 Nov 2018 14:28:38 -0600 Subject: [PATCH 041/526] traffic_ctl: Refactor and convert traffic_ctl to use ArgParser --- include/tscore/runroot.h | 2 +- src/traffic_ctl/alarm.cc | 67 +++------ src/traffic_ctl/config.cc | 243 ++++++++++++--------------------- src/traffic_ctl/host.cc | 113 +++++---------- src/traffic_ctl/metric.cc | 83 ++++------- src/traffic_ctl/plugin.cc | 28 +--- src/traffic_ctl/server.cc | 151 +++++--------------- src/traffic_ctl/storage.cc | 31 ++--- src/traffic_ctl/traffic_ctl.cc | 231 ++++++++++++++++--------------- src/traffic_ctl/traffic_ctl.h | 123 +++++++++-------- 10 files changed, 399 insertions(+), 673 deletions(-) diff --git a/include/tscore/runroot.h b/include/tscore/runroot.h index 0f2efa153a4..5d1b6077052 100644 --- a/include/tscore/runroot.h +++ b/include/tscore/runroot.h @@ -55,7 +55,7 @@ bool exists(const std::string &dir); bool is_directory(const std::string &directory); // argparser_runroot_handler should replace runroot_handler below when all program use ArgParser. -void argparser_runroot_handler(std::string const &value, const char *executable, bool json); +void argparser_runroot_handler(std::string const &value, const char *executable, bool json = false); void runroot_handler(const char **argv, bool json = false); // get a map from default layout diff --git a/src/traffic_ctl/alarm.cc b/src/traffic_ctl/alarm.cc index d3229f386ac..d8df03a5ad7 100644 --- a/src/traffic_ctl/alarm.cc +++ b/src/traffic_ctl/alarm.cc @@ -41,46 +41,38 @@ struct AlarmListPolicy { using CtrlAlarmList = CtrlMgmtList; -static int -alarm_list(unsigned argc, const char **argv) +void +CtrlEngine::alarm_list() { TSMgmtError error; CtrlAlarmList alarms; - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("alarm list", nullptr, 0); - } - error = TSActiveEventGetMlt(alarms.list); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to fetch active alarms"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } while (!alarms.empty()) { char *a = alarms.next(); - printf("%s\n", a); + std::cout << a << std::endl; TSfree(a); } - - return CTRL_EX_OK; } -static int -alarm_clear(unsigned argc, const char **argv) +void +CtrlEngine::alarm_clear() { TSMgmtError error; CtrlAlarmList alarms; - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("alarm clear", nullptr, 0); - } - // First get the active alarms ... error = TSActiveEventGetMlt(alarms.list); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to fetch active alarms"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } // Now resolve them all ... @@ -91,49 +83,26 @@ alarm_clear(unsigned argc, const char **argv) if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to resolve %s", a); TSfree(a); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } TSfree(a); } - - return CTRL_EX_OK; } -static int -alarm_resolve(unsigned argc, const char **argv) +void +CtrlEngine::alarm_resolve() { TSMgmtError error; CtrlAlarmList alarms; - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments == 0) { - return CtrlCommandUsage("alarm resolve ALARM [ALARM ...]", nullptr, 0); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { - error = TSEventResolve(file_arguments[i]); + for (const auto &it : arguments.get("resolve")) { + error = TSEventResolve(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to resolve %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to resolve %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } } - - return CTRL_EX_OK; -} - -int -subcommand_alarm(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {alarm_clear, "clear", "Clear all current alarms"}, - {alarm_list, "list", "List all current alarms"}, - - // Note that we separate resolve one from resolve all for the same reasons that - // we have "metric zero" and "metric clear". - {alarm_resolve, "resolve", "Resolve the listed alarms"}, - /* XXX describe a specific alarm? */ - /* XXX raise an alarm? */ - }; - - return CtrlGenericSubcommand("alarm", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/config.cc b/src/traffic_ctl/config.cc index 35c73035f6d..378fb4785a6 100644 --- a/src/traffic_ctl/config.cc +++ b/src/traffic_ctl/config.cc @@ -181,179 +181,143 @@ format_record(const CtrlMgmtRecord &record, bool recfmt) CtrlMgmtRecordValue value(record); if (recfmt) { - printf("%s %s %s %s\n", rec_labelof(record.rclass()), record.name(), rec_typeof(record.type()), value.c_str()); + std::cout << rec_labelof(record.rclass()) << ' ' << record.name() << ' ' << rec_typeof(record.type()) << ' ' << value.c_str() + << std::endl; } else { - printf("%s: %s\n", record.name(), value.c_str()); + std::cout << record.name() << ": " << value.c_str() << std::endl; } } -static int -config_get(unsigned argc, const char **argv) +void +CtrlEngine::config_get() { - int recfmt = 0; - const ArgumentDescription opts[] = { - {"records", '-', "Emit output in records.config format", "F", &recfmt, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) { - return CtrlCommandUsage("config get [OPTIONS] RECORD [RECORD ...]", opts, countof(opts)); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("get")) { CtrlMgmtRecord record; TSMgmtError error; - error = record.fetch(file_arguments[i]); + error = record.fetch(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } if (REC_TYPE_IS_CONFIG(record.rclass())) { - format_record(record, recfmt); + format_record(record, arguments.get("records")); } } - - return CTRL_EX_OK; } -static int -config_describe(unsigned argc, const char **argv) +void +CtrlEngine::config_describe() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlCommandUsage("config describe RECORD [RECORD ...]"); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("describe")) { TSConfigRecordDescription desc; TSMgmtError error; ink_zero(desc); - error = TSConfigRecordDescribe(file_arguments[i], 0 /* flags */, &desc); + error = TSConfigRecordDescribe(it.c_str(), 0 /* flags */, &desc); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to describe %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to describe %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } - printf("%-16s: %s\n", "Name", desc.rec_name); - printf("%-16s: %s\n", "Current Value", CtrlMgmtRecordValue(desc.rec_type, desc.rec_value).c_str()); - printf("%-16s: %s\n", "Default Value", CtrlMgmtRecordValue(desc.rec_type, desc.rec_default).c_str()); - printf("%-16s: %s\n", "Record Type", rec_classof(desc.rec_class)); - printf("%-16s: %s\n", "Data Type", rec_typeof(desc.rec_type)); - printf("%-16s: %s\n", "Access Control ", rec_accessof(desc.rec_access)); - printf("%-16s: %s\n", "Update Type", rec_updateof(desc.rec_updatetype)); - printf("%-16s: 0x%" PRIx64 "\n", "Update Status", desc.rec_update); - printf("%-16s: %s\n", "Source", rec_sourceof(desc.rec_source)); + std::cout << "Name: " << desc.rec_name << std::endl; + std::cout << "Current Value: " << CtrlMgmtRecordValue(desc.rec_type, desc.rec_value).c_str() << std::endl; + std::cout << "Default Value: " << CtrlMgmtRecordValue(desc.rec_type, desc.rec_default).c_str() << std::endl; + std::cout << "Record Type: " << rec_classof(desc.rec_class) << std::endl; + std::cout << "Data Type: " << rec_typeof(desc.rec_type) << std::endl; + std::cout << "Access Control: " << rec_accessof(desc.rec_access) << std::endl; + std::cout << "Update Type: " << rec_updateof(desc.rec_updatetype) << std::endl; + std::cout << "Update Status: " << desc.rec_update << std::endl; + std::cout << "Source: " << rec_sourceof(desc.rec_source) << std::endl; if (strlen(desc.rec_checkexpr)) { - printf("%-16s: %s, '%s'\n", "Syntax Check", rec_checkof(desc.rec_checktype), desc.rec_checkexpr); + std::cout << "Syntax Check: " << rec_checkof(desc.rec_checktype) << desc.rec_checkexpr << std::endl; } else { - printf("%-16s: %s\n", "Syntax Check", rec_checkof(desc.rec_checktype)); + std::cout << "Syntax Check: " << rec_checkof(desc.rec_checktype) << std::endl; } - - printf("%-16s: %" PRId64 "\n", "Version", desc.rec_version); - printf("%-16s: %" PRId64 "\n", "Order", desc.rec_order); - printf("%-16s: %" PRId64 "\n", "Raw Stat Block", desc.rec_rsb); + std::cout << "Version: " << desc.rec_version << std::endl; + std::cout << "Order: " << desc.rec_order << std::endl; + std::cout << "Raw Stat Block: " << desc.rec_rsb << std::endl; TSConfigRecordDescriptionFree(&desc); } - - return CTRL_EX_OK; } -static int -config_set(unsigned argc, const char **argv) +void +CtrlEngine::config_set() { TSMgmtError error; TSActionNeedT action; - - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 2) { - return CtrlCommandUsage("config set RECORD VALUE"); - } - - error = TSRecordSet(file_arguments[0], file_arguments[1], &action); + auto set_data = arguments.get("set"); + const char *rec_name = set_data[0].c_str(); + const char *rec_val = set_data[1].c_str(); + error = TSRecordSet(rec_name, rec_val, &action); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to set %s", file_arguments[0]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to set %s", rec_name); + status_code = CTRL_EX_ERROR; + return; } switch (action) { case TS_ACTION_SHUTDOWN: - printf("set %s, full shutdown required\n", file_arguments[0]); + std::cout << "set " << rec_name << ", full shutdown required" << std::endl; break; case TS_ACTION_RESTART: - printf("set %s, restart required\n", file_arguments[0]); + std::cout << "set " << rec_name << ", restart required" << std::endl; break; case TS_ACTION_RECONFIGURE: - printf("set %s, please wait 10 seconds for traffic server to sync configuration, restart is not required\n", file_arguments[0]); + std::cout << "set " << rec_name << ", please wait 10 seconds for traffic server to sync configuration, restart is not required" + << std::endl; break; case TS_ACTION_DYNAMIC: default: - printf("set %s\n", file_arguments[0]); + printf("set %s\n", rec_name); break; } - - return CTRL_EX_OK; } -static int -config_match(unsigned argc, const char **argv) +void +CtrlEngine::config_match() { - int recfmt = 0; - const ArgumentDescription opts[] = { - {"records", '-', "Emit output in records.config format", "F", &recfmt, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) { - return CtrlCommandUsage("config match [OPTIONS] REGEX [REGEX ...]", opts, countof(opts)); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("match")) { CtrlMgmtRecordList reclist; TSMgmtError error; // XXX filter the results to only match configuration records. - error = reclist.match(file_arguments[i]); + error = reclist.match(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } while (!reclist.empty()) { CtrlMgmtRecord record(reclist.next()); if (REC_TYPE_IS_CONFIG(record.rclass())) { - format_record(record, recfmt); + format_record(record, arguments.get("records")); } } } - - return CTRL_EX_OK; } -static int -config_reload(unsigned argc, const char **argv) +void +CtrlEngine::config_reload() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("config reload"); - } - TSMgmtError error = TSReconfigure(); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "configuration reload request failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; } -static int -config_status(unsigned argc, const char **argv) +void +CtrlEngine::config_status() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("config status"); - } - CtrlMgmtRecord version; CtrlMgmtRecord configtime; CtrlMgmtRecord starttime; @@ -368,77 +332,57 @@ config_status(unsigned argc, const char **argv) CTRL_MGMT_CHECK(proxy.fetch("proxy.node.config.restart_required.proxy")); CTRL_MGMT_CHECK(manager.fetch("proxy.node.config.restart_required.manager")); - printf("%s\n", CtrlMgmtRecordValue(version).c_str()); - printf("Started at %s", timestr((time_t)starttime.as_int()).c_str()); - printf("Last reconfiguration at %s", timestr((time_t)configtime.as_int()).c_str()); - printf("%s\n", reconfig.as_int() ? "Reconfiguration required" : "Configuration is current"); + std::cout << CtrlMgmtRecordValue(version).c_str() << std::endl; + std::cout << "Started at " << timestr((time_t)starttime.as_int()).c_str(); + std::cout << "Last reconfiguration at " << timestr((time_t)configtime.as_int()).c_str(); + std::cout << (reconfig.as_int() ? "Reconfiguration required" : "Configuration is current") << std::endl; if (proxy.as_int()) { - printf("traffic_server requires restarting\n"); + std::cout << "traffic_server requires restarting" << std::endl; } if (manager.as_int()) { - printf("traffic_manager requires restarting\n"); + std::cout << "traffic_manager requires restarting\n" << std::endl; } - - return CTRL_EX_OK; } -static int -config_defaults(unsigned argc, const char **argv) +void +CtrlEngine::config_defaults() { - int recfmt = 0; - const ArgumentDescription opts[] = { - {"records", '-', "Emit output in records.config format", "F", &recfmt, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage("config diff [OPTIONS]"); - } - TSMgmtError error; CtrlMgmtRecordDescriptionList descriptions; error = descriptions.match(".*"); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to fetch record metadata"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } while (!descriptions.empty()) { TSConfigRecordDescription *desc = descriptions.next(); CtrlMgmtRecordValue deflt(desc->rec_type, desc->rec_default); - if (recfmt) { - printf("%s %s %s %s\n", rec_labelof(desc->rec_class), desc->rec_name, rec_typeof(desc->rec_type), deflt.c_str()); + if (arguments.get("records")) { + std::cout << rec_labelof(desc->rec_class) << ' ' << desc->rec_name << ' ' << rec_typeof(desc->rec_type) << ' ' + << deflt.c_str() << std::endl; } else { - printf("%s: %s\n", desc->rec_name, deflt.c_str()); + std::cout << desc->rec_name << ": " << deflt.c_str() << std::endl; } - TSConfigRecordDescriptionDestroy(desc); } - - return CTRL_EX_OK; } -static int -config_diff(unsigned argc, const char **argv) +void +CtrlEngine::config_diff() { - int recfmt = 0; - const ArgumentDescription opts[] = { - {"records", '-', "Emit output in records.config format", "F", &recfmt, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage("config diff [OPTIONS]"); - } - TSMgmtError error; CtrlMgmtRecordDescriptionList descriptions; error = descriptions.match(".*"); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to fetch record metadata"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } while (!descriptions.empty()) { @@ -468,35 +412,16 @@ config_diff(unsigned argc, const char **argv) CtrlMgmtRecordValue current(desc->rec_type, desc->rec_value); CtrlMgmtRecordValue deflt(desc->rec_type, desc->rec_default); - if (recfmt) { - printf("%s %s %s %s # default: %s\n", rec_labelof(desc->rec_class), desc->rec_name, rec_typeof(desc->rec_type), - current.c_str(), deflt.c_str()); + if (arguments.get("records")) { + std::cout << rec_labelof(desc->rec_class) << ' ' << desc->rec_name << ' ' << rec_typeof(desc->rec_type) << ' ' + << current.c_str() << " # default: " << deflt.c_str() << std::endl; } else { - printf("%s has changed\n", desc->rec_name); - printf("\t%-16s: %s\n", "Current Value", current.c_str()); - printf("\t%-16s: %s\n", "Default Value", deflt.c_str()); + std::cout << desc->rec_name << "has changed" << std::endl; + std::cout << "\tCurrent Value: " << current.c_str() << std::endl; + std::cout << "\tDefault Value: " << deflt.c_str() << std::endl; } } TSConfigRecordDescriptionDestroy(desc); } - - return CTRL_EX_OK; -} - -int -subcommand_config(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {config_defaults, "defaults", "Show default information configuration values"}, - {config_describe, "describe", "Show detailed information about configuration values"}, - {config_diff, "diff", "Show non-default configuration values"}, - {config_get, "get", "Get one or more configuration values"}, - {config_match, "match", "Get configuration matching a regular expression"}, - {config_reload, "reload", "Request a configuration reload"}, - {config_set, "set", "Set a configuration value"}, - {config_status, "status", "Check the configuration status"}, - }; - - return CtrlGenericSubcommand("config", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/host.cc b/src/traffic_ctl/host.cc index 1c69fa30480..2b23bc9dbb6 100644 --- a/src/traffic_ctl/host.cc +++ b/src/traffic_ctl/host.cc @@ -25,121 +25,78 @@ #include "HostStatus.h" #include "records/P_RecUtils.h" -static int -status_get(unsigned argc, const char **argv) +void +CtrlEngine::status_get() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlCommandUsage("host status HOST [HOST ...]", nullptr, 0); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("status")) { CtrlMgmtRecord record; TSMgmtError error; - std::string str = stat_prefix + file_arguments[i]; + std::string str = stat_prefix + it; for (const char *_reason_tag : Reasons::reasons) { std::string _stat = str + "_" + _reason_tag; error = record.fetch(_stat.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } if (REC_TYPE_IS_STAT(record.rclass())) { - printf("%s %s\n", record.name(), CtrlMgmtRecordValue(record).c_str()); + std::cout << record.name() << ' ' << CtrlMgmtRecordValue(record).c_str() << std::endl; } } } - - return CTRL_EX_OK; } -static int -status_down(unsigned argc, const char **argv) +void +CtrlEngine::status_down() { - int down_time = 0; - char *reason = nullptr; - const char *usage = "host down HOST [OPTIONS]"; - - const ArgumentDescription opts[] = { - {"time", 'I', "number of seconds that a host is marked down", "I", &down_time, nullptr, nullptr}, - // memory is allocated for 'reason', if this option is used - {"reason", '-', "reason for marking the host down, one of 'manual|active|local'", "S*", &reason, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) { - return CtrlCommandUsage(usage, opts, countof(opts)); - } + int down_time = 0; + std::string reason = arguments.get("reason").value(); // if reason is not set, set it to manual (default) - if (reason == nullptr) { - reason = ats_strdup(Reasons::MANUAL); + if (reason.empty()) { + reason = Reasons::MANUAL; } - if (!Reasons::validReason(reason)) { - fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason); - return CtrlCommandUsage(usage, opts, countof(opts)); + if (!Reasons::validReason(reason.c_str())) { + fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason.c_str()); + parser.help_message(); } TSMgmtError error = TS_ERR_OKAY; - for (unsigned i = 0; i < n_file_arguments; ++i) { - error = TSHostStatusSetDown(file_arguments[i], down_time, reason); + for (const auto &it : arguments.get("down")) { + error = TSHostStatusSetDown(it.c_str(), down_time, reason.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to set %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to set %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } } - ats_free(reason); - - return CTRL_EX_OK; } -static int -status_up(unsigned argc, const char **argv) +void +CtrlEngine::status_up() { - char *reason = nullptr; - const char *usage = "host up HOST [OPTIONS]"; - - const ArgumentDescription opts[] = { - // memory is allocated for 'reason', if this option is used - {"reason", '-', "reason for marking the host up, one of 'manual|active|local'", "S*", &reason, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) { - return CtrlCommandUsage(usage, nullptr, 0); - } + std::string reason = arguments.get("reason").value(); // if reason is not set, set it to manual (default) - if (reason == nullptr) { - reason = ats_strdup(Reasons::MANUAL); + if (reason.empty()) { + reason = Reasons::MANUAL; } - if (!Reasons::validReason(reason)) { - fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason); - return CtrlCommandUsage(usage, opts, countof(opts)); + if (!Reasons::validReason(reason.c_str())) { + fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason.c_str()); + parser.help_message(); } TSMgmtError error; - for (unsigned i = 0; i < n_file_arguments; ++i) { - error = TSHostStatusSetUp(file_arguments[i], 0, reason); + for (const auto &it : arguments.get("up")) { + error = TSHostStatusSetUp(it.c_str(), 0, reason.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to set %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to set %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } } - ats_free(reason); - - return CTRL_EX_OK; -} - -int -subcommand_host(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {status_get, "status", "Get one or more host statuses"}, - {status_down, "down", "Set down one or more host(s) "}, - {status_up, "up", "Set up one or more host(s) "}, - - }; - - return CtrlGenericSubcommand("host", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/metric.cc b/src/traffic_ctl/metric.cc index 68796b5e62d..1076aceb293 100644 --- a/src/traffic_ctl/metric.cc +++ b/src/traffic_ctl/metric.cc @@ -24,103 +24,72 @@ #include "traffic_ctl.h" #include "records/P_RecUtils.h" -static int -metric_get(unsigned argc, const char **argv) +void +CtrlEngine::metric_get() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlCommandUsage("metric get METRIC [METRIC ...]", nullptr, 0); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("get")) { CtrlMgmtRecord record; TSMgmtError error; - error = record.fetch(file_arguments[i]); + error = record.fetch(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } if (REC_TYPE_IS_STAT(record.rclass())) { - printf("%s %s\n", record.name(), CtrlMgmtRecordValue(record).c_str()); + std::cout << record.name() << ' ' << CtrlMgmtRecordValue(record).c_str() << std::endl; } } - - return CTRL_EX_OK; } -static int -metric_match(unsigned argc, const char **argv) +void +CtrlEngine::metric_match() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlCommandUsage("metric match [OPTIONS] REGEX [REGEX ...]", nullptr, 0); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("match")) { CtrlMgmtRecordList reclist; TSMgmtError error; - error = reclist.match(file_arguments[i]); + error = reclist.match(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } while (!reclist.empty()) { CtrlMgmtRecord record(reclist.next()); if (REC_TYPE_IS_STAT(record.rclass())) { - printf("%s %s\n", record.name(), CtrlMgmtRecordValue(record).c_str()); + std::cout << record.name() << ' ' << CtrlMgmtRecordValue(record).c_str() << std::endl; } } } - - return CTRL_EX_OK; } -static int -metric_clear(unsigned argc, const char **argv) +void +CtrlEngine::metric_clear() { TSMgmtError error; error = TSStatsReset(nullptr); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to clear metrics"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; } -static int -metric_zero(unsigned argc, const char **argv) +void +CtrlEngine::metric_zero() { TSMgmtError error; - for (unsigned i = 0; i < n_file_arguments; ++i) { - error = TSStatsReset(file_arguments[i]); + for (const auto &it : arguments.get("zero")) { + error = TSStatsReset(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to clear %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to clear %s", it.c_str()); + status_code = CTRL_EX_ERROR; } } - - return CTRL_EX_OK; -} - -int -subcommand_metric(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {metric_get, "get", "Get one or more metric values"}, - {metric_clear, "clear", "Clear all metric values"}, - {CtrlUnimplementedCommand, "describe", "Show detailed information about one or more metric values"}, - {metric_match, "match", "Get metrics matching a regular expression"}, - {CtrlUnimplementedCommand, "monitor", "Display the value of a metric over time"}, - - // We could allow clearing all the metrics in the "clear" subcommand, but that seems error-prone. It - // would be too easy to just expect a help message and accidentally nuke all the metrics. - {metric_zero, "zero", "Clear one or more metric values"}, - }; - - return CtrlGenericSubcommand("metric", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/plugin.cc b/src/traffic_ctl/plugin.cc index cec12fac9b5..8935ee7b1de 100644 --- a/src/traffic_ctl/plugin.cc +++ b/src/traffic_ctl/plugin.cc @@ -23,30 +23,16 @@ #include "traffic_ctl.h" -static int -plugin_msg(unsigned argc, const char **argv) +void +CtrlEngine::plugin_msg() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 2) { - return CtrlCommandUsage("plugin msg TAG DATA"); - } - TSMgmtError error; + auto msg_data = arguments.get("msg"); - error = TSLifecycleMessage(file_arguments[0], file_arguments[1], strlen(file_arguments[1]) + 1); + error = TSLifecycleMessage(msg_data[0].c_str(), msg_data[1].c_str(), msg_data[1].size() + 1); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "message '%s' not sent", file_arguments[0]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "message '%s' not sent", msg_data[0].c_str()); + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; -} - -int -subcommand_plugin(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {plugin_msg, "msg", "Send message to plugins - a TAG and the message DATA"}, - }; - - return CtrlGenericSubcommand("plugin", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/server.cc b/src/traffic_ctl/server.cc index d41064ffa61..58271ba4de4 100644 --- a/src/traffic_ctl/server.cc +++ b/src/traffic_ctl/server.cc @@ -23,30 +23,17 @@ #include "traffic_ctl.h" -static int drain = 0; -static int manager = 0; - -static int -restart(unsigned argc, const char **argv) +void +CtrlEngine::server_restart() { TSMgmtError error; - const char *usage = "server restart [OPTIONS]"; - unsigned flags = TS_RESTART_OPT_NONE; - - const ArgumentDescription opts[] = { - {"drain", '-', "Wait for client connections to drain before restarting", "F", &drain, nullptr, nullptr}, - {"manager", '-', "Restart traffic_manager as well as traffic_server", "F", &manager, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage(usage, opts, countof(opts)); - } + unsigned flags = TS_RESTART_OPT_NONE; - if (drain) { + if (arguments.get("drain")) { flags |= TS_RESTART_OPT_DRAIN; } - if (manager) { + if (arguments.get("manager")) { error = TSRestart(flags); } else { error = TSBounce(flags); @@ -54,79 +41,51 @@ restart(unsigned argc, const char **argv) if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server restart failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; -} - -static int -server_restart(unsigned argc, const char **argv) -{ - return restart(argc, argv); } -static int -server_backtrace(unsigned argc, const char **argv) +void +CtrlEngine::server_backtrace() { TSMgmtError error; TSString trace = nullptr; - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("server backtrace"); - } - error = TSProxyBacktraceGet(0, &trace); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server backtrace failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - printf("%s\n", trace); + std::cout << trace << std::endl; TSfree(trace); - return CTRL_EX_OK; } -static int -server_status(unsigned argc, const char **argv) +void +CtrlEngine::server_status() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("server status"); - } - switch (TSProxyStateGet()) { case TS_PROXY_ON: - printf("Proxy -- on\n"); + std::cout << "Proxy -- on" << std::endl; break; case TS_PROXY_OFF: - printf("Proxy -- off\n"); + std::cout << "Proxy -- off" << std::endl; break; case TS_PROXY_UNDEFINED: - printf("Proxy status undefined\n"); + std::cout << "Proxy status undefined" << std::endl; break; } - - // XXX Surely we can report more useful status that this !?!! - - return CTRL_EX_OK; } -static int -server_stop(unsigned argc, const char **argv) +void +CtrlEngine::server_stop() { TSMgmtError error; - const char *usage = "server stop [OPTIONS]"; - unsigned flags = TS_RESTART_OPT_NONE; + unsigned flags = TS_RESTART_OPT_NONE; - const ArgumentDescription opts[] = { - {"drain", '-', "Wait for client connections to drain before stopping", "F", &drain, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage(usage, opts, countof(opts)); - } - - if (drain) { + if (arguments.get("drain")) { flags |= TS_STOP_OPT_DRAIN; } @@ -134,62 +93,36 @@ server_stop(unsigned argc, const char **argv) if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server stop failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; } -static int -server_start(unsigned argc, const char **argv) +void +CtrlEngine::server_start() { TSMgmtError error; - int cache = 0; - int hostdb = 0; unsigned clear = TS_CACHE_CLEAR_NONE; - const ArgumentDescription opts[] = { - {"clear-cache", '-', "Clear the disk cache on startup", "F", &cache, nullptr, nullptr}, - {"clear-hostdb", '-', "Clear the DNS cache on startup", "F", &hostdb, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage("server start [OPTIONS]", opts, countof(opts)); - } - - clear |= cache ? TS_CACHE_CLEAR_CACHE : TS_CACHE_CLEAR_NONE; - clear |= hostdb ? TS_CACHE_CLEAR_HOSTDB : TS_CACHE_CLEAR_NONE; + clear |= arguments.get("clear-cache") ? TS_CACHE_CLEAR_CACHE : TS_CACHE_CLEAR_NONE; + clear |= arguments.get("clear-hostdb") ? TS_CACHE_CLEAR_HOSTDB : TS_CACHE_CLEAR_NONE; error = TSProxyStateSet(TS_PROXY_ON, clear); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server start failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; } -static int -server_drain(unsigned argc, const char **argv) +void +CtrlEngine::server_drain() { TSMgmtError error; - const char *usage = "server drain [OPTIONS]"; - - int no_new_connection = 0; - int undo = 0; - const ArgumentDescription opts[] = { - {"no-new-connection", 'N', "Wait for new connections down to threshold before starting draining", "F", &no_new_connection, - nullptr, nullptr}, - {"undo", 'U', "Recover server from the drain mode", "F", &undo, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage(usage, opts, countof(opts)); - } - if (undo) { + if (arguments.get("undo")) { error = TSDrain(TS_DRAIN_OPT_UNDO); - } else if (no_new_connection) { + } else if (arguments.get("no-new-connection")) { error = TSDrain(TS_DRAIN_OPT_IDLE); } else { error = TSDrain(TS_DRAIN_OPT_NONE); @@ -197,21 +130,7 @@ server_drain(unsigned argc, const char **argv) if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server drain failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; -} - -int -subcommand_server(unsigned argc, const char **argv) -{ - const subcommand commands[] = {{server_backtrace, "backtrace", "Show a full stack trace of the traffic_server process"}, - {server_restart, "restart", "Restart Traffic Server"}, - {server_start, "start", "Start the proxy"}, - {server_status, "status", "Show the proxy status"}, - {server_stop, "stop", "Stop the proxy"}, - {server_drain, "drain", "Drain the requests"}}; - - return CtrlGenericSubcommand("server", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/storage.cc b/src/traffic_ctl/storage.cc index ebd96071eba..1e55bc94fde 100644 --- a/src/traffic_ctl/storage.cc +++ b/src/traffic_ctl/storage.cc @@ -23,33 +23,18 @@ #include "traffic_ctl.h" -static int -storage_offline(unsigned argc, const char **argv) +void +CtrlEngine::storage_offline() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments == 0) { - return CtrlCommandUsage("storage offline DEVICE [DEVICE ...]"); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + auto offline_data = arguments.get("offline"); + for (const auto &it : offline_data) { TSMgmtError error; - error = TSStorageDeviceCmdOffline(file_arguments[i]); + error = TSStorageDeviceCmdOffline(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to take %s offline", file_arguments[0]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to take %s offline", offline_data[0].c_str()); + status_code = CTRL_EX_ERROR; + return; } } - - return CTRL_EX_OK; -} - -int -subcommand_storage(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {storage_offline, "offline", "Take one or more storage volumes offline"}, - {CtrlUnimplementedCommand, "status", "Show the storage configuration"}, - }; - - return CtrlGenericSubcommand("storage", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/traffic_ctl.cc b/src/traffic_ctl/traffic_ctl.cc index be61cf096dd..8efb92a8112 100644 --- a/src/traffic_ctl/traffic_ctl.cc +++ b/src/traffic_ctl/traffic_ctl.cc @@ -28,8 +28,6 @@ #include "tscore/I_Layout.h" #include "tscore/runroot.h" -AppVersionInfo CtrlVersionInfo; - const char * CtrlMgmtRecord::name() const { @@ -145,120 +143,137 @@ CtrlMgmtError(TSMgmtError err, const char *fmt, ...) } } -int -CtrlSubcommandUsage(const char *name, const subcommand *cmds, unsigned ncmds, const ArgumentDescription *desc, unsigned ndesc) -{ - const char *opt = ndesc ? "[OPTIONS]" : ""; - const char *sep = (ndesc && name) ? " " : ""; - - fprintf(stderr, "Usage: traffic_ctl %s%s%s CMD [ARGS ...]\n\nSubcommands:\n", name ? name : "", sep, opt); - - for (unsigned i = 0; i < ncmds; ++i) { - fprintf(stderr, " %-16s%s\n", cmds[i].name, cmds[i].help); - } - - if (ndesc) { - usage(desc, ndesc, "\nOptions:"); - } - - return CTRL_EX_USAGE; -} - -int -CtrlCommandUsage(const char *msg, const ArgumentDescription *desc, unsigned ndesc) -{ - fprintf(stderr, "Usage: traffic_ctl %s\n", msg); - - if (ndesc) { - usage(desc, ndesc, "\nOptions:"); - } - - return CTRL_EX_USAGE; -} - -bool -CtrlProcessArguments(int /* argc */, const char **argv, const ArgumentDescription *desc, unsigned ndesc) -{ - n_file_arguments = 0; - return process_args_ex(&CtrlVersionInfo, desc, ndesc, argv); -} - -int -CtrlUnimplementedCommand(unsigned /* argc */, const char **argv) +void +CtrlEngine::CtrlUnimplementedCommand(std::string_view command) { - fprintf(stderr, "'%s' command is not implemented\n", *argv); - return CTRL_EX_UNIMPLEMENTED; + fprintf(stderr, "'%s' command is not implemented\n", command.data()); + status_code = CTRL_EX_UNIMPLEMENTED; } -int -CtrlGenericSubcommand(const char *name, const subcommand *cmds, unsigned ncmds, unsigned argc, const char **argv) -{ - CtrlCommandLine cmdline; - - // Process command line arguments and dump into variables - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlSubcommandUsage(name, cmds, ncmds, nullptr, 0); - } - - cmdline.init(n_file_arguments, file_arguments); - - for (unsigned i = 0; i < ncmds; ++i) { - if (strcmp(file_arguments[0], cmds[i].name) == 0) { - return cmds[i].handler(cmdline.argc(), cmdline.argv()); - } - } - - return CtrlSubcommandUsage(name, cmds, ncmds, nullptr, 0); -} - -static const subcommand commands[] = { - {subcommand_alarm, "alarm", "Manipulate alarms"}, - {subcommand_config, "config", "Manipulate configuration records"}, - {subcommand_metric, "metric", "Manipulate performance metrics"}, - {subcommand_server, "server", "Stop, restart and examine the server"}, - {subcommand_storage, "storage", "Manipulate cache storage"}, - {subcommand_plugin, "plugin", "Interact with plugins"}, - {subcommand_host, "host", "Interact with host status"}, -}; - int main(int argc, const char **argv) { - CtrlCommandLine cmdline; - int debug = false; - - CtrlVersionInfo.setup(PACKAGE_NAME, "traffic_ctl", PACKAGE_VERSION, __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, ""); - program_name = CtrlVersionInfo.AppStr; - - ArgumentDescription argument_descriptions[] = { - {"debug", '-', "Enable debugging output", "F", &debug, nullptr, nullptr}, - {"help", 'h', "Print usage information", nullptr, nullptr, nullptr, - [](const ArgumentDescription *args, unsigned nargs, const char *arg_unused) { - CtrlSubcommandUsage(nullptr, commands, countof(commands), args, nargs); - }}, - VERSION_ARGUMENT_DESCRIPTION(), - RUNROOT_ARGUMENT_DESCRIPTION(), - }; + CtrlEngine engine; + + engine.parser.add_global_usage("traffic_ctl [OPTIONS] CMD [ARGS ...]"); + engine.parser.require_commands(); + + engine.parser.add_option("--debug", "", "Enable debugging output") + .add_option("--version", "-V", "Print version string") + .add_option("--help", "-h", "Print usage information") + .add_option("--run-root", "", "using TS_RUNROOT as sandbox", "TS_RUNROOT", 1); + + auto &alarm_command = engine.parser.add_command("alarm", "Manipulate alarms").require_commands(); + auto &config_command = engine.parser.add_command("config", "Manipulate configuration records").require_commands(); + auto &metric_command = engine.parser.add_command("metric", "Manipulate performance metrics").require_commands(); + auto &server_command = engine.parser.add_command("server", "Stop, restart and examine the server").require_commands(); + auto &storage_command = engine.parser.add_command("storage", "Manipulate cache storage").require_commands(); + auto &plugin_command = engine.parser.add_command("plugin", "Interact with plugins").require_commands(); + auto &host_command = engine.parser.add_command("host", "Interact with host status").require_commands(); + + // alarm commands + alarm_command.add_command("clear", "Clear all current alarms", [&]() { engine.alarm_clear(); }) + .add_example_usage("traffic_ctl alarm clear"); + alarm_command.add_command("list", "List all current alarms", [&]() { engine.alarm_list(); }) + .add_example_usage("traffic_ctl alarm list"); + alarm_command.add_command("resolve", "Resolve the listed alarms", "", MORE_THAN_ONE_ARG_N, [&]() { engine.alarm_resolve(); }) + .add_example_usage("traffic_ctl alarm resolve ALARM [ALARM ...]"); + + // config commands + config_command.add_command("defaults", "Show default information configuration values", [&]() { engine.config_defaults(); }) + .add_example_usage("traffic_ctl config defaults [OPTIONS]") + .add_option("--records", "", "Emit output in records.config format"); + config_command + .add_command("describe", "Show detailed information about configuration values", "", MORE_THAN_ONE_ARG_N, + [&]() { engine.config_describe(); }) + .add_example_usage("traffic_ctl config describe RECORD [RECORD ...]"); + config_command.add_command("diff", "Show non-default configuration values", [&]() { engine.config_diff(); }) + .add_example_usage("traffic_ctl config diff [OPTIONS]") + .add_option("--records", "", "Emit output in records.config format"); + config_command.add_command("get", "Get one or more configuration values", "", MORE_THAN_ONE_ARG_N, [&]() { engine.config_get(); }) + .add_example_usage("traffic_ctl config get [OPTIONS] RECORD [RECORD ...]") + .add_option("--records", "", "Emit output in records.config format"); + config_command + .add_command("match", "Get configuration matching a regular expression", "", MORE_THAN_ONE_ARG_N, + [&]() { engine.config_match(); }) + .add_example_usage("traffic_ctl config match [OPTIONS] REGEX [REGEX ...]") + .add_option("--records", "", "Emit output in records.config format"); + config_command.add_command("reload", "Request a configuration reload", [&]() { engine.config_reload(); }) + .add_example_usage("traffic_ctl config reload"); + config_command.add_command("status", "Check the configuration status", [&]() { engine.config_status(); }) + .add_example_usage("traffic_ctl config status"); + config_command.add_command("set", "Set a configuration value", "", 2, [&]() { engine.config_set(); }) + .add_example_usage("traffic_ctl config set RECORD VALUE"); + + // host commands + host_command.add_command("status", "Get one or more host statuses", "", MORE_THAN_ONE_ARG_N, [&]() { engine.status_get(); }) + .add_example_usage("traffic_ctl host status HOST [HOST ...]"); + host_command.add_command("down", "Set down one or more host(s)", "", MORE_THAN_ONE_ARG_N, [&]() { engine.status_down(); }) + .add_example_usage("traffic_ctl host down HOST [OPTIONS]") + .add_option("--time", "-I", "number of seconds that a host is marked down", "", 1) + .add_option("--reason", "", "reason for marking the host down, one of 'manual|active|local"); + host_command.add_command("up", "Set up one or more host(s)", "", MORE_THAN_ONE_ARG_N, [&]() { engine.status_up(); }) + .add_example_usage("traffic_ctl host up METRIC value") + .add_option("--reason", "", "reason for marking the host up, one of 'manual|active|local"); + + // metric commands + metric_command.add_command("get", "Get one or more metric values", "", MORE_THAN_ONE_ARG_N, [&]() { engine.metric_get(); }) + .add_example_usage("traffic_ctl metric get METRIC [METRIC ...]"); + metric_command.add_command("clear", "Clear all metric values", [&]() { engine.metric_clear(); }); + metric_command.add_command("describe", "Show detailed information about one or more metric values", "", MORE_THAN_ONE_ARG_N, + [&]() { engine.CtrlUnimplementedCommand("describe"); }); // not implemented + metric_command.add_command("match", "Get metrics matching a regular expression", "", MORE_THAN_ZERO_ARG_N, + [&]() { engine.metric_match(); }); + metric_command.add_command("monitor", "Display the value of a metric over time", "", MORE_THAN_ZERO_ARG_N, + [&]() { engine.CtrlUnimplementedCommand("monitor"); }); // not implemented + metric_command.add_command("zero", "Clear one or more metric values", "", MORE_THAN_ONE_ARG_N, [&]() { engine.metric_zero(); }); + + // plugin command + plugin_command.add_command("msg", "Send message to plugins - a TAG and the message DATA", "", 2, [&]() { engine.plugin_msg(); }) + .add_example_usage("traffic_ctl plugin msg TAG DATA"); + + // server commands + server_command.add_command("backtrace", "Show a full stack trace of the traffic_server process", + [&]() { engine.server_backtrace(); }); + server_command.add_command("restart", "Restart Traffic Server", [&]() { engine.server_backtrace(); }) + .add_example_usage("traffic_ctl server restart [OPTIONS]") + .add_option("--drain", "", "Wait for client connections to drain before restarting") + .add_option("--manager", "", "Restart traffic_manager as well as traffic_server"); + server_command.add_command("start", "Start the proxy", [&]() { engine.server_start(); }) + .add_example_usage("traffic_ctl server start [OPTIONS]") + .add_option("--clear-cache", "", "Clear the disk cache on startup") + .add_option("--clear-hostdb", "", "Clear the DNS cache on startup"); + server_command.add_command("status", "Show the proxy status", [&]() { engine.server_status(); }) + .add_example_usage("traffic_ctl server status"); + server_command.add_command("stop", "Stop the proxy", [&]() { engine.server_stop(); }) + .add_example_usage("traffic_ctl server stop [OPTIONS]") + .add_option("--drain", "", "Wait for client connections to drain before stopping"); + server_command.add_command("drain", "Drain the requests", [&]() { engine.server_drain(); }) + .add_example_usage("traffic_ctl server drain [OPTIONS]") + .add_option("--no-new-connection", "-N", "Wait for new connections down to threshold before starting draining") + .add_option("--undo", "-U", "Recover server from the drain mode"); + + // storage commands + storage_command + .add_command("offline", "Take one or more storage volumes offline", "", MORE_THAN_ONE_ARG_N, + [&]() { engine.storage_offline(); }) + .add_example_usage("storage offline DEVICE [DEVICE ...]"); + storage_command.add_command("status", "Show the storage configuration", "", MORE_THAN_ZERO_ARG_N, + [&]() { engine.CtrlUnimplementedCommand("status"); }); // not implemented + + // parse the arguments + engine.arguments = engine.parser.parse(argv); BaseLogFile *base_log_file = new BaseLogFile("stderr"); - diags = new Diags(program_name, "" /* tags */, "" /* actions */, base_log_file); + diags = new Diags("traffic_ctl", "" /* tags */, "" /* actions */, base_log_file); - // Process command line arguments and dump into variables - if (!CtrlProcessArguments(argc, argv, argument_descriptions, countof(argument_descriptions))) { - return CtrlSubcommandUsage(nullptr, commands, countof(commands), argument_descriptions, countof(argument_descriptions)); - } - - if (debug) { + if (engine.arguments.get("debug")) { diags->activate_taglist("traffic_ctl", DiagsTagType_Debug); diags->config.enabled[DiagsTagType_Debug] = true; diags->show_location = SHOW_LOCATION_DEBUG; } - if (n_file_arguments < 1) { - return CtrlSubcommandUsage(nullptr, commands, countof(commands), argument_descriptions, countof(argument_descriptions)); - } - - runroot_handler(argv); + argparser_runroot_handler(engine.arguments.get("--run-root").value(), argv[0]); Layout::create(); RecProcessInit(RECM_STAND_ALONE, diags); LibRecordsConfigInit(); @@ -272,16 +287,10 @@ main(int argc, const char **argv) // error. TSInit(rundir, static_cast(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS)); - for (unsigned i = 0; i < countof(commands); ++i) { - if (strcmp(file_arguments[0], commands[i].name) == 0) { - CtrlCommandLine cmdline; - - cmdline.init(n_file_arguments, file_arguments); - return commands[i].handler(cmdline.argc(), cmdline.argv()); - } - } + engine.arguments.invoke(); // Done with the mgmt API. TSTerminate(); - return CtrlSubcommandUsage(nullptr, commands, countof(commands), argument_descriptions, countof(argument_descriptions)); + + return engine.status_code; } diff --git a/src/traffic_ctl/traffic_ctl.h b/src/traffic_ctl/traffic_ctl.h index 991c790bfa8..f02fe27b51b 100644 --- a/src/traffic_ctl/traffic_ctl.h +++ b/src/traffic_ctl/traffic_ctl.h @@ -30,28 +30,22 @@ #include "tscore/ink_args.h" #include "tscore/I_Version.h" #include "tscore/BaseLogFile.h" +#include "tscore/ArgParser.h" #include #include +#include -struct subcommand { - int (*handler)(unsigned, const char **); - const char *name; - const char *help; -}; - -extern AppVersionInfo CtrlVersionInfo; +// Exit status codes, following BSD's sysexits(3) +constexpr int CTRL_EX_OK = 0; +constexpr int CTRL_EX_ERROR = 2; +constexpr int CTRL_EX_UNIMPLEMENTED = 3; +constexpr int CTRL_EX_USAGE = EX_USAGE; +constexpr int CTRL_EX_UNAVAILABLE = 69; #define CtrlDebug(...) Debug("traffic_ctl", __VA_ARGS__) void CtrlMgmtError(TSMgmtError err, const char *fmt, ...) TS_PRINTFLIKE(2, 3); -int CtrlSubcommandUsage(const char *name, const subcommand *cmds, unsigned ncmds, const ArgumentDescription *desc, unsigned ndesc); -int CtrlCommandUsage(const char *msg, const ArgumentDescription *desc = nullptr, unsigned ndesc = 0); - -bool CtrlProcessArguments(int argc, const char **argv, const ArgumentDescription *desc, unsigned ndesc); -int CtrlUnimplementedCommand(unsigned argc, const char **argv); - -int CtrlGenericSubcommand(const char *, const subcommand *cmds, unsigned ncmds, unsigned argc, const char **argv); #define CTRL_MGMT_CHECK(expr) \ do { \ @@ -59,7 +53,8 @@ int CtrlGenericSubcommand(const char *, const subcommand *cmds, unsigned ncmds, if (e != TS_ERR_OKAY) { \ CtrlDebug("%s failed with status %d", #expr, e); \ CtrlMgmtError(e, nullptr); \ - return CTRL_EX_ERROR; \ + status_code = CTRL_EX_ERROR; \ + return; \ } \ } while (0) @@ -167,47 +162,59 @@ struct CtrlMgmtRecordList : CtrlMgmtList { TSMgmtError match(const char *); }; -struct CtrlCommandLine { - CtrlCommandLine() { this->args.push_back(nullptr); } - void - init(unsigned argc, const char **argv) - { - this->args.clear(); - for (unsigned i = 0; i < argc; ++i) { - this->args.push_back(argv[i]); - } - - // Always nullptr-terminate to keep ink_args happy. Note that we adjust arg() accordingly. - this->args.push_back(nullptr); - } - - unsigned - argc() - { - return args.size() - 1; - } - - const char ** - argv() - { - return &args[0]; - } - -private: - std::vector args; +// this is a engine for traffic_ctl containing the ArgParser and all the methods +// it also has a status code which can be set by these methods to return +struct CtrlEngine { + // the parser for traffic_ctl + ts::ArgParser parser; + // parsed arguments + ts::Arguments arguments; + // the return status code from functions + // By default it is set to CTRL_EX_OK so we don't need to set it + // in each method when they finish successfully. + int status_code = CTRL_EX_OK; + + // All traffic_ctl methods: + // umimplemented command + void CtrlUnimplementedCommand(std::string_view command); + + // alarm methods + void alarm_list(); + void alarm_clear(); + void alarm_resolve(); + + // config methods + void config_defaults(); + void config_diff(); + void config_status(); + void config_reload(); + void config_match(); + void config_get(); + void config_set(); + void config_describe(); + + // host methods + void status_get(); + void status_down(); + void status_up(); + + // metric methods + void metric_get(); + void metric_match(); + void metric_clear(); + void metric_zero(); + + // metric methods + void plugin_msg(); + + // server methods + void server_restart(); + void server_backtrace(); + void server_status(); + void server_stop(); + void server_start(); + void server_drain(); + + // storage methods + void storage_offline(); }; - -int subcommand_alarm(unsigned argc, const char **argv); -int subcommand_config(unsigned argc, const char **argv); -int subcommand_metric(unsigned argc, const char **argv); -int subcommand_server(unsigned argc, const char **argv); -int subcommand_storage(unsigned argc, const char **argv); -int subcommand_plugin(unsigned argc, const char **argv); -int subcommand_host(unsigned argc, const char **argv); - -// Exit status codes, following BSD's sysexits(3) -#define CTRL_EX_OK 0 -#define CTRL_EX_ERROR 2 -#define CTRL_EX_UNIMPLEMENTED 3 -#define CTRL_EX_USAGE EX_USAGE -#define CTRL_EX_UNAVAILABLE 69 From 9b7be409e61b95b73e29eca0a57e87e52b4431e3 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Mon, 12 Nov 2018 10:31:35 -0600 Subject: [PATCH 042/526] TCL: Replace RawHashTable with STL --- include/tscore/RawHashTable.h | 382 ------------------ proxy/hdrs/HttpCompat.h | 1 - proxy/http/HttpBodyFactory.cc | 223 ++++------ proxy/http/HttpBodyFactory.h | 21 +- .../unit_tests/test_error_page_selection.cc | 27 +- src/tscore/Makefile.am | 2 - src/tscore/RawHashTable.cc | 31 -- 7 files changed, 107 insertions(+), 580 deletions(-) delete mode 100644 include/tscore/RawHashTable.h delete mode 100644 src/tscore/RawHashTable.cc diff --git a/include/tscore/RawHashTable.h b/include/tscore/RawHashTable.h deleted file mode 100644 index 4d0cb5141d8..00000000000 --- a/include/tscore/RawHashTable.h +++ /dev/null @@ -1,382 +0,0 @@ -/** @file - - C++ wrapper around libts hash tables - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - @section details Details - - These C++ RawHashTables are a C++ wrapper around libts hash tables. - They expose an interface very analogous to ink_hash_table, for better - or for worse. See HashTable for a more C++-oriented hash table. - -*/ - -#pragma once - -#include "tscore/ink_apidefs.h" -#include "tscore/ink_hash_table.h" - -////////////////////////////////////////////////////////////////////////////// -// -// Constants and Type Definitions -// -////////////////////////////////////////////////////////////////////////////// - -typedef enum { - RawHashTable_KeyType_String = InkHashTableKeyType_String, - RawHashTable_KeyType_Word = InkHashTableKeyType_Word -} RawHashTable_KeyType; - -typedef InkHashTableKey RawHashTable_Key; -typedef InkHashTableValue RawHashTable_Value; -typedef InkHashTableEntry RawHashTable_Binding; -typedef InkHashTableIteratorState RawHashTable_IteratorState; - -////////////////////////////////////////////////////////////////////////////// -// -// The RawHashTable Class -// -////////////////////////////////////////////////////////////////////////////// - -class RawHashTable -{ -private: - InkHashTable *ht; - RawHashTable_KeyType key_type; - bool deallocate_values_on_destruct; - -public: - inkcoreapi RawHashTable(RawHashTable_KeyType key_type, bool deallocate_values_on_destruct = false); - virtual ~RawHashTable(); - - // - // these are the simplest accessor functions - // - - bool getValue(RawHashTable_Key key, RawHashTable_Value *value_ptr); - void setValue(RawHashTable_Key key, RawHashTable_Value value_ptr); - bool isBound(RawHashTable_Key key); - bool unbindKey(RawHashTable_Key key); - void replaceString(char *key, char *string); - - // - // these functions allow you to manipulate the (key,value) bindings directly - // - - RawHashTable_Binding *getCurrentBinding(RawHashTable_Key key); - RawHashTable_Binding *getOrCreateBinding(RawHashTable_Key key, bool *was_new = nullptr); - - void setBindingValue(RawHashTable_Binding *binding, RawHashTable_Value value); - RawHashTable_Key getKeyFromBinding(RawHashTable_Binding *binding); - RawHashTable_Value getValueFromBinding(RawHashTable_Binding *binding); - - // - // these functions allow you to iterate through RawHashTable bindings - // - - RawHashTable_Binding *firstBinding(RawHashTable_IteratorState *state_ptr); - RawHashTable_Binding *nextBinding(RawHashTable_IteratorState *state_ptr); -}; - -////////////////////////////////////////////////////////////////////////////// -// -// Inline Methods -// -////////////////////////////////////////////////////////////////////////////// - -/** - This routine gets the value associated with key. If the key has a - binding, the value is stored through value_ptr and true is returned. If - the key DOES NOT have a binding, false is returned. - -*/ -inline bool -RawHashTable::getValue(RawHashTable_Key key, RawHashTable_Value *value_ptr) -{ - int is_bound; - - is_bound = ink_hash_table_lookup(ht, (InkHashTableKey)key, (InkHashTableValue *)value_ptr); - return (is_bound ? true : false); -} - -/** - This routine sets the value associated with key to the value. If - a value is previously bound to the key, the previous value is left - dangling. The caller is responsible to freeing any previous binding - values needing freeing before calling setValue. - - If the key has a binding, the value is stored through value_ptr and - true is returned. If the key DOES NOT have a binding, false is returned. - -*/ -inline void -RawHashTable::setValue(RawHashTable_Key key, RawHashTable_Value value) -{ - ink_hash_table_insert(ht, (InkHashTableKey)key, (InkHashTableValue)value); -} - -/** - This routine sets the value associated with key to the value pointed to - by value_ptr. If a value is previously bound to the key, the previous - value is left dangling. The caller is responsible to freeing any - previous value before setValue. - - If the key has a binding, the value is stored through value_ptr and - true is returned. If the key DOES NOT have a binding, false is returned. - -*/ -inline bool -RawHashTable::isBound(RawHashTable_Key key) -{ - int status = ink_hash_table_isbound(ht, (InkHashTableKey)key); - return (status ? true : false); -} - -/** - This routine removes any association for key from the hash table. If - data was bound to key, the binding will be deleted, but the value will - not be deallocated. The caller is responsible to freeing any previous - value before unbindKey. - - @return true if the key was previously bound, false otherwise. - -*/ -inline bool -RawHashTable::unbindKey(RawHashTable_Key key) -{ - int status; - - status = ink_hash_table_delete(ht, (InkHashTableKey)key); - return (status ? true : false); -} - -/** - This rather specialized routine binds a malloc-allocated string value - to the key, freeing any previous value. The key must be a string, - and the hash table must have been constructed as having key_type - RawHashTable_KeyType_String. - -*/ -inline void -RawHashTable::replaceString(char *key, char *string) -{ - // if (key_type != RawHashTable_KeyType_String) - // { - // throw BadKeyType(); - // } - - ink_hash_table_replace_string(ht, key, string); -} - -/** - This function looks up a binding for key in the hash table, and returns - a pointer to the binding data structure directly inside the hash table, - or NULL if there is no binding. - -*/ -inline RawHashTable_Binding * -RawHashTable::getCurrentBinding(RawHashTable_Key key) -{ - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_lookup_entry(ht, (InkHashTableKey)key); - return ((RawHashTable_Binding *)he_ptr); -} - -/** - This function looks up a binding for key in the hash table, creates - a binding if one doesn't exist, and returns a pointer to the binding - data structure directly inside the hash table. - - If was_new is not NULL, true is stored through was_new. If no binding - previously existed, false is stored through was_new if a binding - previously existed. - -*/ -inline RawHashTable_Binding * -RawHashTable::getOrCreateBinding(RawHashTable_Key key, bool *was_new) -{ - int _was_new; - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_get_entry(ht, (InkHashTableKey)key, &_was_new); - *was_new = (_was_new ? true : false); - return ((RawHashTable_Binding *)he_ptr); -} - -/** - This function looks up a binding for key in the hash table, creates - a binding if one doesn't exist, and returns a pointer to the binding - data structure directly inside the hash table. - - If was_new is not NULL, true is stored through was_new. If no binding - previously existed, false is stored through was_new if a binding - previously existed. - -*/ -inline void -RawHashTable::setBindingValue(RawHashTable_Binding *binding, RawHashTable_Value value) -{ - ink_hash_table_set_entry(ht, (InkHashTableEntry *)binding, (InkHashTableValue)value); -} - -/** - This function takes a binding and extracts the key. - -*/ -inline RawHashTable_Key -RawHashTable::getKeyFromBinding(RawHashTable_Binding *binding) -{ - InkHashTableKey ht_key; - - ht_key = ink_hash_table_entry_key(ht, (InkHashTableEntry *)binding); - return ((RawHashTable_Key)ht_key); -} - -/** - This function takes a binding and extracts the value. - -*/ -inline RawHashTable_Value -RawHashTable::getValueFromBinding(RawHashTable_Binding *binding) -{ - InkHashTableValue ht_value; - - ht_value = ink_hash_table_entry_value(ht, (InkHashTableEntry *)binding); - return ((RawHashTable_Value)ht_value); -} - -/** - This function takes a hash table, initializes an interator data - structure to point to the first binding in the hash table, and returns - the first binding, or NULL if there are none. - -*/ -inline RawHashTable_Binding * -RawHashTable::firstBinding(RawHashTable_IteratorState *state_ptr) -{ - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_iterator_first(ht, (InkHashTableIteratorState *)state_ptr); - return ((RawHashTable_Binding *)he_ptr); -} - -inline RawHashTable::RawHashTable(RawHashTable_KeyType akey_type, bool adeallocate_values_on_destruct) -{ - RawHashTable::key_type = akey_type; - RawHashTable::deallocate_values_on_destruct = adeallocate_values_on_destruct; - ht = ink_hash_table_create((InkHashTableKeyType)key_type); -} - -inline RawHashTable::~RawHashTable() -{ - if (deallocate_values_on_destruct) { - ink_hash_table_destroy_and_free_values(ht); - } else { - ink_hash_table_destroy(ht); - } -} - -/** - This function takes a hash table and a pointer to iterator state, - and advances to the next binding in the hash table, if any. If there - in a next binding, a pointer to the binding is returned, else NULL. - -*/ -inline RawHashTable_Binding * -RawHashTable::nextBinding(RawHashTable_IteratorState *state_ptr) -{ - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_iterator_next(ht, (InkHashTableIteratorState *)state_ptr); - return ((RawHashTable_Binding *)he_ptr); -} - -////////////////////////////////////////////////////////////////////////////// -// -// The RawHashTableIter Class -// -////////////////////////////////////////////////////////////////////////////// - -class RawHashTableIter -{ -public: - RawHashTableIter(RawHashTable &ht); - ~RawHashTableIter(); - - RawHashTable_Value &operator++(); // get next - RawHashTable_Value &operator()() const; // get current - operator const void *() const; // is valid - - RawHashTable_Value &value() const; // get current value - const char *key() const; // get current key - -private: - RawHashTable &m_ht; - RawHashTable_Binding *m_currentBinding; - RawHashTable_IteratorState m_hashIterState; -}; - -////////////////////////////////////////////////////////////////////////////// -// -// Inline Methods -// -////////////////////////////////////////////////////////////////////////////// - -inline RawHashTable_Value & -RawHashTableIter::operator()() const -{ - return (m_currentBinding->clientData); -} - -inline RawHashTable_Value & -RawHashTableIter::operator++() -{ - m_currentBinding = m_ht.nextBinding(&m_hashIterState); - return (m_currentBinding->clientData); -} - -inline RawHashTableIter::operator const void *() const -{ - return ((m_currentBinding != nullptr) ? this : nullptr); -} - -inline RawHashTable_Value & -RawHashTableIter::value() const -{ - return (m_currentBinding->clientData); -} - -inline const char * -RawHashTableIter::key() const -{ - return (m_currentBinding->key.string); -} - -inline RawHashTableIter::RawHashTableIter(RawHashTable &ht) : m_ht(ht), m_currentBinding(nullptr) -{ - m_currentBinding = m_ht.firstBinding(&m_hashIterState); - return; -} - -inline RawHashTableIter::~RawHashTableIter() -{ - return; -} diff --git a/proxy/hdrs/HttpCompat.h b/proxy/hdrs/HttpCompat.h index 3bf0dc334a3..8c237995f83 100644 --- a/proxy/hdrs/HttpCompat.h +++ b/proxy/hdrs/HttpCompat.h @@ -25,7 +25,6 @@ #include "tscore/ink_string++.h" #include "MIME.h" -#include "tscore/RawHashTable.h" #include "tscore/Diags.h" class HttpCompat diff --git a/proxy/http/HttpBodyFactory.cc b/proxy/http/HttpBodyFactory.cc index 53edaa6864b..9d5433b53d7 100644 --- a/proxy/http/HttpBodyFactory.cc +++ b/proxy/http/HttpBodyFactory.cc @@ -190,50 +190,29 @@ HttpBodyFactory::fabricate_with_old_api(const char *type, HttpTransact::State *c } unlock(); - return (buffer); + return buffer; } void HttpBodyFactory::dump_template_tables(FILE *fp) { - RawHashTable *h1, *h2; - RawHashTable_Key k1, k2; - RawHashTable_Value v1, v2; - RawHashTable_Binding *b1, *b2; - RawHashTable_IteratorState i1, i2; - HttpBodySet *body_set; - lock(); - - h1 = table_of_sets; - - if (h1 != nullptr) { - /////////////////////////////////////////// - // loop over set->body-types hash table // - /////////////////////////////////////////// - - for (b1 = h1->firstBinding(&i1); b1 != nullptr; b1 = h1->nextBinding(&i1)) { - k1 = table_of_sets->getKeyFromBinding(b1); - v1 = table_of_sets->getValueFromBinding(b1); - body_set = (HttpBodySet *)v1; - - if (body_set != nullptr) { - fprintf(fp, "set %s: name '%s', lang '%s', charset '%s'\n", k1, body_set->set_name, body_set->content_language, - body_set->content_charset); + if (table_of_sets) { + for (const auto &it1 : *table_of_sets.get()) { + HttpBodySet *body_set = static_cast(it1.second); + if (body_set) { + fprintf(fp, "set %s: name '%s', lang '%s', charset '%s'\n", it1.first.c_str(), body_set->set_name, + body_set->content_language, body_set->content_charset); /////////////////////////////////////////// // loop over body-types->body hash table // /////////////////////////////////////////// ink_assert(body_set->is_sane()); - h2 = body_set->table_of_pages; - - for (b2 = h2->firstBinding(&i2); b2 != nullptr; b2 = h2->nextBinding(&i2)) { - k2 = table_of_sets->getKeyFromBinding(b2); - v2 = table_of_sets->getValueFromBinding(b2); - HttpBodyTemplate *t = (HttpBodyTemplate *)v2; - - fprintf(fp, " %-30s: %" PRId64 " bytes\n", k2, t->byte_count); + if (body_set->table_of_pages) { + for (const auto &it2 : *body_set->table_of_pages.get()) { + fprintf(fp, " %-30s: %" PRId64 " bytes\n", it2.first.c_str(), it2.second->byte_count); + } } } } @@ -254,7 +233,7 @@ config_callback(const char * /* name ATS_UNUSED */, RecDataT /* data_type ATS_UN { HttpBodyFactory *body_factory = (HttpBodyFactory *)cookie; body_factory->reconfigure(); - return (0); + return 0; } void @@ -372,7 +351,7 @@ HttpBodyFactory::HttpBodyFactory() HttpBodyFactory::~HttpBodyFactory() { // FIX: need to implement destructor - delete table_of_sets; + table_of_sets.reset(nullptr); } // LOCKING: must be called with lock taken @@ -457,13 +436,12 @@ const char * HttpBodyFactory::determine_set_by_host(HttpTransact::State *context) { const char *set; - RawHashTable_Value v; int host_len = context->hh_info.host_len; char host_buffer[host_len + 1]; strncpy(host_buffer, context->hh_info.request_host, host_len); host_buffer[host_len] = '\0'; - if (table_of_sets->getValue((RawHashTable_Key)host_buffer, &v)) { - set = table_of_sets->getKeyFromBinding(table_of_sets->getCurrentBinding((RawHashTable_Key)host_buffer)); + if (auto it = table_of_sets->find(host_buffer); it != table_of_sets->end()) { + set = it->first.c_str(); } else { set = "default"; } @@ -471,8 +449,9 @@ HttpBodyFactory::determine_set_by_host(HttpTransact::State *context) } const char * -HttpBodyFactory::determine_set_by_language(RawHashTable *table_of_sets, StrList *acpt_language_list, StrList *acpt_charset_list, - float *Q_best_ptr, int *La_best_ptr, int *Lc_best_ptr, int *I_best_ptr) +HttpBodyFactory::determine_set_by_language(std::unique_ptr &table_of_sets, StrList *acpt_language_list, + StrList *acpt_charset_list, float *Q_best_ptr, int *La_best_ptr, int *Lc_best_ptr, + int *I_best_ptr) { float Q, Ql, Qc, Q_best; int I, Idummy, I_best; @@ -480,13 +459,6 @@ HttpBodyFactory::determine_set_by_language(RawHashTable *table_of_sets, StrList int is_the_default_set; const char *set_best; - RawHashTable_Key k1; - RawHashTable_Value v1; - RawHashTable_Binding *b1; - RawHashTable_IteratorState i1; - RawHashTable *table_of_pages; - HttpBodySetRawData *body_set; - set_best = "default"; Q_best = 0.00001; La_best = 0; @@ -506,20 +478,15 @@ HttpBodyFactory::determine_set_by_language(RawHashTable *table_of_sets, StrList goto done; } - if (table_of_sets != nullptr) { + if (table_of_sets) { /////////////////////////////////////////// // loop over set->body-types hash table // /////////////////////////////////////////// + for (const auto &it : *table_of_sets.get()) { + const char *set_name = it.first.c_str(); + HttpBodySetRawData *body_set = it.second; - for (b1 = table_of_sets->firstBinding(&i1); b1 != nullptr; b1 = table_of_sets->nextBinding(&i1)) { - k1 = table_of_sets->getKeyFromBinding(b1); - v1 = table_of_sets->getValueFromBinding(b1); - const char *set_name = (const char *)k1; - - body_set = (HttpBodySetRawData *)v1; - table_of_pages = body_set->table_of_pages; - - if ((set_name == nullptr) || (table_of_pages == nullptr)) { + if ((it.first.empty()) || (body_set->table_of_pages == nullptr)) { continue; } @@ -635,7 +602,7 @@ HttpBodyFactory::determine_set_by_language(RawHashTable *table_of_sets, StrList *La_best_ptr = La_best; *Lc_best_ptr = Lc_best; *I_best_ptr = I_best; - return (set_best); + return set_best; } // LOCKING: must be called with lock taken @@ -648,46 +615,42 @@ HttpBodyFactory::determine_set_by_language(StrList *acpt_language_list, StrList set_best = determine_set_by_language(table_of_sets, acpt_language_list, acpt_charset_list, &Q_best, &La_best, &Lc_best, &I_best); - return (set_best); + return set_best; } // LOCKING: must be called with lock taken HttpBodyTemplate * HttpBodyFactory::find_template(const char *set, const char *type, HttpBodySet **body_set_return) { - RawHashTable_Value v; - Debug("body_factory", "calling find_template(%s,%s)", set, type); *body_set_return = nullptr; - if (table_of_sets == nullptr) { - return (nullptr); + if (table_of_sets == nullptr || !set || !type) { + return nullptr; } - if (table_of_sets->getValue((RawHashTable_Key)set, &v)) { - HttpBodySet *body_set = (HttpBodySet *)v; - RawHashTable *table_of_types = body_set->table_of_pages; - - if (table_of_types == nullptr) { - return (nullptr); + if (auto it = table_of_sets->find(set); it != table_of_sets->end()) { + HttpBodySet *body_set = static_cast(it->second); + if (body_set->table_of_pages == nullptr) { + return nullptr; } - if (table_of_types->getValue((RawHashTable_Key)type, &v)) { - HttpBodyTemplate *t = (HttpBodyTemplate *)v; + if (auto it_page = body_set->table_of_pages->find(type); it_page != body_set->table_of_pages->end()) { + HttpBodyTemplate *t = it_page->second; if ((t == nullptr) || (!t->is_sane())) { - return (nullptr); + return nullptr; } *body_set_return = body_set; Debug("body_factory", "find_template(%s,%s) -> (file %s, length %" PRId64 ", lang '%s', charset '%s')", set, type, t->template_pathname, t->byte_count, body_set->content_language, body_set->content_charset); - return (t); + return t; } } Debug("body_factory", "find_template(%s,%s) -> NULL", set, type); - return (nullptr); + return nullptr; } // LOCKING: must be called with lock taken @@ -705,17 +668,17 @@ HttpBodyFactory::is_response_suppressed(HttpTransact::State *context) } else */ if (response_suppression_mode == 0) { - return (false); + return false; } else if (response_suppression_mode == 1) { - return (true); + return true; } else if (response_suppression_mode == 2) { if (context->req_flavor == HttpTransact::REQ_FLAVOR_INTERCEPTED) { - return (true); + return true; } else { - return (false); + return false; } } else { - return (false); + return false; } } @@ -723,69 +686,43 @@ HttpBodyFactory::is_response_suppressed(HttpTransact::State *context) void HttpBodyFactory::nuke_template_tables() { - RawHashTable *h1, *h2; - RawHashTable_Value v1, v2; - RawHashTable_Binding *b1, *b2; - RawHashTable_IteratorState i1, i2; - HttpBodySet *body_set; - HttpBodyTemplate *hbt; - - h1 = table_of_sets; - - if (h1) { + if (table_of_sets) { Debug("body_factory", "deleting pre-existing template tables"); } else { Debug("body_factory", "no pre-existing template tables"); } - if (h1 != nullptr) { + if (table_of_sets) { /////////////////////////////////////////// // loop over set->body-types hash table // /////////////////////////////////////////// - - for (b1 = h1->firstBinding(&i1); b1 != nullptr; b1 = h1->nextBinding(&i1)) { - v1 = h1->getValueFromBinding(b1); - - body_set = (HttpBodySet *)v1; + for (const auto &it : *table_of_sets.get()) { + HttpBodySet *body_set = static_cast(it.second); ink_assert(body_set->is_sane()); - h2 = body_set->table_of_pages; - - if (h2 != nullptr) { - body_set->table_of_pages = nullptr; - + if (body_set->table_of_pages) { /////////////////////////////////////////// // loop over body-types->body hash table // /////////////////////////////////////////// - - for (b2 = h2->firstBinding(&i2); b2 != nullptr; b2 = h2->nextBinding(&i2)) { - v2 = h2->getValueFromBinding(b2); - if (v2) { - // need a cast here - hbt = (HttpBodyTemplate *)v2; - delete hbt; - } + for (const auto &it_page : *body_set->table_of_pages.get()) { + delete it_page.second; } - - delete h2; + body_set->table_of_pages.reset(nullptr); } - delete body_set; } - delete h1; + table_of_sets.reset(nullptr); } - - table_of_sets = nullptr; } // LOCKING: must be called with lock taken -RawHashTable * +std::unique_ptr HttpBodyFactory::load_sets_from_directory(char *set_dir) { DIR *dir; struct dirent *dirEntry; - RawHashTable *new_table_of_sets; + std::unique_ptr new_table_of_sets; if (set_dir == nullptr) { - return (nullptr); + return nullptr; } Debug("body_factory", "load_sets_from_directory(%s)", set_dir); @@ -798,10 +735,10 @@ HttpBodyFactory::load_sets_from_directory(char *set_dir) if (dir == nullptr) { Warning("can't open response template directory '%s' (%s)", set_dir, (strerror(errno) ? strerror(errno) : "unknown reason")); Warning("no response templates --- using default error pages"); - return (nullptr); + return nullptr; } - new_table_of_sets = new RawHashTable(RawHashTable_KeyType_String); + new_table_of_sets.reset(new HttpBodyFactory::BodySetTable); ////////////////////////////////////////// // loop over each language subdirectory // @@ -837,13 +774,13 @@ HttpBodyFactory::load_sets_from_directory(char *set_dir) HttpBodySet *body_set = load_body_set_from_directory(dirEntry->d_name, subdir); if (body_set != nullptr) { Debug("body_factory", " %s -> %p", dirEntry->d_name, body_set); - new_table_of_sets->setValue((RawHashTable_Key)(dirEntry->d_name), (RawHashTable_Value)body_set); + new_table_of_sets->emplace(dirEntry->d_name, body_set); } } closedir(dir); - return (new_table_of_sets); + return new_table_of_sets; } // LOCKING: must be called with lock taken @@ -931,7 +868,7 @@ HttpBodyFactory::load_body_set_from_directory(char *set_name, char *tmpl_dir) } closedir(dir); - return (body_set); + return body_set; } //////////////////////////////////////////////////////////////////////// @@ -956,9 +893,7 @@ HttpBodySet::~HttpBodySet() ats_free(set_name); ats_free(content_language); ats_free(content_charset); - if (table_of_pages) { - delete table_of_pages; - } + table_of_pages.reset(nullptr); } int @@ -972,15 +907,11 @@ HttpBodySet::init(char *set, char *dir) ink_filepath_make(info_path, sizeof(info_path), dir, ".body_factory_info"); fd = open(info_path, O_RDONLY); if (fd < 0) { - return (-1); + return -1; } this->set_name = ats_strdup(set); - - if (this->table_of_pages) { - delete (this->table_of_pages); - } - this->table_of_pages = new RawHashTable(RawHashTable_KeyType_String); + this->table_of_pages.reset(new TemplateTable); lineno = 0; @@ -1079,37 +1010,37 @@ HttpBodySet::init(char *set, char *dir) } close(fd); - return (lines_added); + return lines_added; } HttpBodyTemplate * HttpBodySet::get_template_by_name(const char *name) { - RawHashTable_Value v; - Debug("body_factory", " calling get_template_by_name(%s)", name); if (table_of_pages == nullptr) { - return (nullptr); + return nullptr; } - if (table_of_pages->getValue((RawHashTable_Key)name, &v)) { - HttpBodyTemplate *t = (HttpBodyTemplate *)v; + if (auto it = table_of_pages->find(name); it != table_of_pages->end()) { + HttpBodyTemplate *t = it->second; if ((t == nullptr) || (!t->is_sane())) { - return (nullptr); + return nullptr; } Debug("body_factory", " get_template_by_name(%s) -> (file %s, length %" PRId64 ")", name, t->template_pathname, t->byte_count); - return (t); + return t; } Debug("body_factory", " get_template_by_name(%s) -> NULL", name); - return (nullptr); + return nullptr; } void HttpBodySet::set_template_by_name(const char *name, HttpBodyTemplate *t) { - table_of_pages->setValue((RawHashTable_Key)name, (RawHashTable_Value)t); + if (name) { + table_of_pages->emplace(name, t); + } } //////////////////////////////////////////////////////////////////////// @@ -1158,10 +1089,10 @@ HttpBodyTemplate::load_from_file(char *dir, char *file) // coverity[fs_check_call] status = stat(path, &stat_buf); if (status != 0) { - return (0); + return 0; } if (!S_ISREG(stat_buf.st_mode)) { - return (0); + return 0; } /////////////////// @@ -1171,7 +1102,7 @@ HttpBodyTemplate::load_from_file(char *dir, char *file) // coverity[toctou] fd = open(path, O_RDONLY); if (fd < 0) { - return (0); + return 0; } //////////////////////////////////////// @@ -1192,7 +1123,7 @@ HttpBodyTemplate::load_from_file(char *dir, char *file) Warning("reading template file '%s', got %" PRId64 " bytes instead of %" PRId64 " (%s)", path, bytes_read, new_byte_count, (strerror(errno) ? strerror(errno) : "unknown error")); ats_free(new_template_buffer); - return (0); + return 0; } Debug("body_factory", " read %" PRId64 " bytes from '%s'", new_byte_count, path); @@ -1206,7 +1137,7 @@ HttpBodyTemplate::load_from_file(char *dir, char *file) byte_count = new_byte_count; template_pathname = ats_strdup(path); - return (1); + return 1; } char * @@ -1224,5 +1155,5 @@ HttpBodyTemplate::build_instantiated_buffer(HttpTransact::State *context, int64_ Debug("body_factory_instantiation", " after instantiation: [%s]", buffer); Debug("body_factory", " returning %" PRId64 " byte instantiated buffer", *buflen_return); - return (buffer); + return buffer; } diff --git a/proxy/http/HttpBodyFactory.h b/proxy/http/HttpBodyFactory.h index 5d78fb46b20..165b24017a2 100644 --- a/proxy/http/HttpBodyFactory.h +++ b/proxy/http/HttpBodyFactory.h @@ -61,9 +61,11 @@ #include "HttpConfig.h" #include "HttpCompat.h" #include "HttpTransact.h" -#include "tscore/RawHashTable.h" #include "tscore/ink_sprintf.h" +#include +#include + #define HTTP_BODY_TEMPLATE_MAGIC 0xB0DFAC00 #define HTTP_BODY_SET_MAGIC 0xB0DFAC55 #define HTTP_BODY_FACTORY_MAGIC 0xB0DFACFF @@ -109,11 +111,12 @@ class HttpBodyTemplate //////////////////////////////////////////////////////////////////////// struct HttpBodySetRawData { - unsigned int magic = 0; + using TemplateTable = std::unordered_map; + unsigned int magic = 0; char *set_name; char *content_language; char *content_charset; - RawHashTable *table_of_pages; + std::unique_ptr table_of_pages; }; //////////////////////////////////////////////////////////////////////// @@ -162,6 +165,7 @@ class HttpBodySet : public HttpBodySetRawData class HttpBodyFactory { public: + using BodySetTable = std::unordered_map; HttpBodyFactory(); ~HttpBodyFactory(); @@ -197,8 +201,9 @@ class HttpBodyFactory void dump_template_tables(FILE *fp = stderr); void reconfigure(); - static const char *determine_set_by_language(RawHashTable *table_of_sets, StrList *acpt_language_list, StrList *acpt_charset_list, - float *Q_best_ptr, int *La_best_ptr, int *Lc_best_ptr, int *I_best_ptr); + static const char *determine_set_by_language(std::unique_ptr &table_of_sets, StrList *acpt_language_list, + StrList *acpt_charset_list, float *Q_best_ptr, int *La_best_ptr, int *Lc_best_ptr, + int *I_best_ptr); private: char *fabricate(StrList *acpt_language_list, StrList *acpt_charset_list, const char *type, HttpTransact::State *context, @@ -225,7 +230,7 @@ class HttpBodyFactory // initialization methods // //////////////////////////// void nuke_template_tables(); - RawHashTable *load_sets_from_directory(char *set_dir); + std::unique_ptr load_sets_from_directory(char *set_dir); HttpBodySet *load_body_set_from_directory(char *set_name, char *tmpl_dir); ///////////////////////////////////////////////// @@ -254,6 +259,6 @@ class HttpBodyFactory //////////////////// unsigned int magic = HTTP_BODY_FACTORY_MAGIC; // magic for sanity checks/debugging ink_mutex mutex; // prevents reconfig/read races - bool callbacks_established = false; // all config variables present - RawHashTable *table_of_sets = nullptr; // sets of template hash tables + bool callbacks_established = false; // all config variables present + std::unique_ptr table_of_sets; // sets of template hash tables }; diff --git a/proxy/http/unit_tests/test_error_page_selection.cc b/proxy/http/unit_tests/test_error_page_selection.cc index 483c8e2c350..004aff537bf 100644 --- a/proxy/http/unit_tests/test_error_page_selection.cc +++ b/proxy/http/unit_tests/test_error_page_selection.cc @@ -74,16 +74,16 @@ TEST_CASE("error page selection test", "[http]") int nsets = sizeof(sets) / sizeof(sets[0]); int ntests = sizeof(tests) / sizeof(tests[0]); // (1) build fake hash table of sets - RawHashTable *table_of_sets = new RawHashTable(RawHashTable_KeyType_String); + std::unique_ptr table_of_sets; + table_of_sets.reset(new HttpBodyFactory::BodySetTable); for (i = 0; i < nsets; i++) { - HttpBodySetRawData *body_set; - body_set = (HttpBodySetRawData *)ats_malloc(sizeof(HttpBodySetRawData)); - body_set->magic = 0; - body_set->set_name = (char *)(sets[i].set_name); - body_set->content_language = (char *)(sets[i].content_language); - body_set->content_charset = (char *)(sets[i].content_charset); - body_set->table_of_pages = (RawHashTable *)1; // hack --- can't be NULL - table_of_sets->setValue((RawHashTable_Key)(body_set->set_name), (RawHashTable_Value)body_set); + HttpBodySetRawData *body_set = new HttpBodySetRawData; + body_set->magic = 0; + body_set->set_name = strdup(sets[i].set_name); + body_set->content_language = strdup(sets[i].content_language); + body_set->content_charset = strdup(sets[i].content_charset); + body_set->table_of_pages.reset(new HttpBodySetRawData::TemplateTable); + table_of_sets->emplace(body_set->set_name, body_set); } // (2) for each test, parse accept headers into lists, and test matching for (i = 0; i < ntests; i++) { @@ -104,5 +104,12 @@ TEST_CASE("error page selection test", "[http]") REQUIRE(La_best == tests[i].expected_La); REQUIRE(I_best == tests[i].expected_I); } - delete table_of_sets; + for (const auto &it : *table_of_sets.get()) { + ats_free(it.second->set_name); + ats_free(it.second->content_language); + ats_free(it.second->content_charset); + it.second->table_of_pages.reset(nullptr); + delete it.second; + } + table_of_sets.reset(nullptr); } diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index 8c3c9f985f8..0ab50153561 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -183,8 +183,6 @@ libtscore_la_SOURCES = \ ParseRules.h \ PriorityQueue.h \ Ptr.h \ - RawHashTable.cc \ - RawHashTable.h \ RbTree.cc \ RbTree.h \ Regex.cc \ diff --git a/src/tscore/RawHashTable.cc b/src/tscore/RawHashTable.cc deleted file mode 100644 index 7bfdaa146cf..00000000000 --- a/src/tscore/RawHashTable.cc +++ /dev/null @@ -1,31 +0,0 @@ -/** @file - - C++ wrapper around libts hash tables - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - @section details Details - - These C++ RawHashTables are a C++ wrapper around libts hash tables. - They expose an interface very analogous to ink_hash_table, for better - or for worse. See HashTable for a more C++-oriented hash table. - -*/ - -#include "tscore/RawHashTable.h" From 1a5fc3fed09b59fb5d693e3c2629e0027f65c8f7 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Sat, 17 Nov 2018 21:37:57 +0900 Subject: [PATCH 043/526] Doc: removing stale while revalidate plugin doc (marked deprecated in 7) The stale while revalidate plugin was marked deprecated in the ATS 7 documentation. Removing now in the 8 docs. --- doc/admin-guide/plugins/index.en.rst | 6 ----- .../plugins/stale_while_revalidate.en.rst | 25 ------------------- 2 files changed, 31 deletions(-) delete mode 100644 doc/admin-guide/plugins/stale_while_revalidate.en.rst diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index dd7dd5b884d..9d8d9b0c5f5 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -160,7 +160,6 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Signed URLs SSL Headers SSL Session Reuse - Stale While Revalidate System Statistics Traffic Dump WebP Transform @@ -228,11 +227,6 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`SSL Headers ` Populate request headers with SSL session information. -:doc:`Stale While Revalidate ` - :deprecated: - - Refresh content asynchronously while serving stale data. - :doc:`System Stats ` Inserts system statistics in to the stats list diff --git a/doc/admin-guide/plugins/stale_while_revalidate.en.rst b/doc/admin-guide/plugins/stale_while_revalidate.en.rst deleted file mode 100644 index 76e192bf663..00000000000 --- a/doc/admin-guide/plugins/stale_while_revalidate.en.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. _admin-plugins-tale-while-revalidate: - -Stale While Revalidate Plugin -***************************** - - :deprecated: - -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -refresh content asynchronously while serving stale data From db0c1f9d3ddd8fb120a4dd9534039c45bd9156ed Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Mon, 26 Nov 2018 13:51:12 -0600 Subject: [PATCH 044/526] Cleanup: Remove Map.h including HashMap, Vec and TSHashTable --- include/tscore/Map.h | 2228 ----------------------------- iocore/net/P_SNIActionPerformer.h | 4 +- iocore/net/P_SSLConfig.h | 2 +- iocore/net/P_SSLNetProcessor.h | 1 - iocore/net/P_SSLSNI.h | 4 +- iocore/net/SNIActionPerformer.cc | 2 +- iocore/net/SSLConfig.cc | 2 +- iocore/net/SSLNetVConnection.cc | 2 +- iocore/net/SSLSNIConfig.cc | 12 +- iocore/net/SSLSessionCache.h | 1 - src/tscore/Makefile.am | 5 +- src/tscore/suppression.txt | 1 - src/tscore/unit_tests/test_Map.cc | 215 --- src/tscore/unit_tests/test_Vec.cc | 388 ----- 14 files changed, 15 insertions(+), 2852 deletions(-) delete mode 100644 include/tscore/Map.h delete mode 100644 src/tscore/unit_tests/test_Map.cc delete mode 100644 src/tscore/unit_tests/test_Vec.cc diff --git a/include/tscore/Map.h b/include/tscore/Map.h deleted file mode 100644 index 5114bbf7a70..00000000000 --- a/include/tscore/Map.h +++ /dev/null @@ -1,2228 +0,0 @@ -/** @file - - A set of Map templates. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#pragma once - -#include -#include -#include -#include - -#include "tscore/defalloc.h" -#include "tscore/ink_assert.h" -#include "tscore/Diags.h" - -#include "tscore/List.h" - -#define MAP_INTEGRAL_SIZE (1 << (2)) -//#define MAP_INITIAL_SHIFT ((2)+1) -//#define MAP_INITIAL_SIZE (1 << MAP_INITIAL_SHIFT) - -// Simple Vector class, also supports open hashed sets -#define VEC_INTEGRAL_SHIFT_DEFAULT 2 /* power of 2 (1 << VEC_INTEGRAL_SHIFT)*/ -#define VEC_INTEGRAL_SIZE (1 << (S)) -#define VEC_INITIAL_SHIFT ((S) + 1) -#define VEC_INITIAL_SIZE (1 << VEC_INITIAL_SHIFT) - -#define SET_LINEAR_SIZE 4 /* must be <= than VEC_INTEGRAL_SIZE */ -#define SET_INITIAL_INDEX 2 - -template // S must be a power of 2 -class Vec -{ -public: - size_t n; - size_t i; // size index for sets, reserve for vectors - C *v; - C e[VEC_INTEGRAL_SIZE]; - - Vec(); - Vec(const Vec &vv); - Vec(const C c); - ~Vec(); - - C &operator[](int i) const { return v[i]; } - C get(size_t i) const; - void add(C a); - void - push_back(C a) - { - add(a); - } // std::vector name - bool add_exclusive(C a); - C &add(); - void drop(); - C pop(); - void reset(); - void clear(); - void free_and_clear(); - void delete_and_clear(); - void set_clear(); - C *set_add(C a); - void set_remove(C a); // expensive, use BlockHash for cheaper remove - C *set_add_internal(C a); - bool set_union(Vec &v); - int set_intersection(Vec &v); - int some_intersection(Vec &v); - int some_disjunction(Vec &v); - int some_difference(Vec &v); - void set_intersection(Vec &v, Vec &result); - void set_disjunction(Vec &v, Vec &result); - void set_difference(Vec &v, Vec &result); - size_t set_count() const; - size_t count() const; - C *in(C a); - C *set_in(C a); - C first_in_set(); - C *set_in_internal(C a); - void set_expand(); - ssize_t index(C a) const; - void set_to_vec(); - void vec_to_set(); - void move(Vec &v); - void copy(const Vec &v); - void fill(size_t n); - void append(const Vec &v); - template void append(const C *src, CountType count); - void prepend(const Vec &v); - void remove_index(int index); - void - remove(C a) - { - int i = index(a); - if (i >= 0) - remove_index(i); - } - C &insert(size_t index); - void insert(size_t index, Vec &vv); - void insert(size_t index, C a); - void - push(C a) - { - insert(0, a); - } - void reverse(); - void reserve(size_t n); - C * - end() const - { - return v + n; - } - C & - first() const - { - return v[0]; - } - C & - last() const - { - return v[n - 1]; - } - Vec & - operator=(Vec &v) - { - this->copy(v); - return *this; - } - unsigned - length() const - { - return n; - } - // vector::size() intentionally not implemented because it should mean "bytes" not count of elements - int write(int fd); - int read(int fd); - void qsort(bool (*lt)(C, C)); - void qsort(bool (*lt)(const C &, const C &)); - static void swap(C *p1, C *p2); - -private: - void move_internal(Vec &v); - void copy_internal(const Vec &v); - void add_internal(C a); - C &add_internal(); - void addx(); -}; - -// c -- class, p -- pointer to elements of v, v -- vector -#define forv_Vec(_c, _p, _v) \ - if ((_v).n) \ - for (_c *qq__##_p = (_c *)0, *_p = (_v).v[0]; \ - ((uintptr_t)(qq__##_p) < (_v).length()) && ((_p = (_v).v[(intptr_t)qq__##_p]), 1); \ - qq__##_p = (_c *)(((intptr_t)qq__##_p) + 1)) -#define for_Vec(_c, _p, _v) \ - if ((_v).n) \ - for (_c *qq__##_p = (_c *)0, _p = (_v).v[0]; \ - ((uintptr_t)(qq__##_p) < (_v).length()) && ((_p = (_v).v[(intptr_t)qq__##_p]), 1); \ - qq__##_p = (_c *)(((intptr_t)qq__##_p) + 1)) -#define forvp_Vec(_c, _p, _v) \ - if ((_v).n) \ - for (_c *qq__##_p = (_c *)0, *_p = &(_v).v[0]; \ - ((uintptr_t)(qq__##_p) < (_v).length()) && ((_p = &(_v).v[(intptr_t)qq__##_p]), 1); \ - qq__##_p = (_c *)(((intptr_t)qq__##_p) + 1)) - -template class Accum -{ -public: - Vec asset; - Vec asvec; - void - add(C c) - { - if (asset.set_add(c)) - asvec.add(c); - } - void - add(Vec v) - { - for (int i = 0; i < v.n; i++) - if (v.v[i]) - add(v.v[i]); - } - void - clear() - { - asset.clear(); - asvec.clear(); - } -}; - -const uintptr_t prime2[] = {1, 3, 7, 13, 31, 61, 127, 251, 509, 1021, - 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, - 2097143, 4194301, 8388593, 16777213, 33554393, 67108859, 134217689, 268435399, 536870909}; - -// primes generated with map_mult.c -const uintptr_t open_hash_primes[256] = { - 0x02D4AF27, 0x1865DFC7, 0x47C62B43, 0x35B4889B, 0x210459A1, 0x3CC51CC7, 0x02ADD945, 0x0607C4D7, 0x558E6035, 0x0554224F, - 0x5A281657, 0x1C458C7F, 0x7F8BE723, 0x20B9BA99, 0x7218AA35, 0x64B10C2B, 0x548E8983, 0x5951218F, 0x7AADC871, 0x695FA5B1, - 0x40D40FCB, 0x20E03CC9, 0x55E9920F, 0x554CE08B, 0x7E78B1D7, 0x7D965DF9, 0x36A520A1, 0x1B0C6C11, 0x33385667, 0x2B0A7B9B, - 0x0F35AE23, 0x0BD608FB, 0x2284ADA3, 0x6E6C0687, 0x129B3EED, 0x7E86289D, 0x1143C24B, 0x1B6C7711, 0x1D87BB41, 0x4C7E635D, - 0x67577999, 0x0A0113C5, 0x6CF085B5, 0x14A4D0FB, 0x4E93E3A7, 0x5C87672B, 0x67F3CA17, 0x5F944339, 0x4C16DFD7, 0x5310C0E3, - 0x2FAD1447, 0x4AFB3187, 0x08468B7F, 0x49E56C51, 0x6280012F, 0x097D1A85, 0x34CC9403, 0x71028BD7, 0x6DEDC7E9, 0x64093291, - 0x6D78BB0B, 0x7A03B465, 0x2E044A43, 0x1AE58515, 0x23E495CD, 0x46102A83, 0x51B78A59, 0x051D8181, 0x5352CAC9, 0x57D1312B, - 0x2726ED57, 0x2E6BC515, 0x70736281, 0x5938B619, 0x0D4B6ACB, 0x44AB5E2B, 0x0029A485, 0x002CE54F, 0x075B0591, 0x3EACFDA9, - 0x0AC03411, 0x53B00F73, 0x2066992D, 0x76E72223, 0x55F62A8D, 0x3FF92EE1, 0x17EE0EB3, 0x5E470AF1, 0x7193EB7F, 0x37A2CCD3, - 0x7B44F7AF, 0x0FED8B3F, 0x4CC05805, 0x7352BF79, 0x3B61F755, 0x523CF9A3, 0x1AAFD219, 0x76035415, 0x5BE84287, 0x6D598909, - 0x456537E9, 0x407EA83F, 0x23F6FFD5, 0x60256F39, 0x5D8EE59F, 0x35265CEB, 0x1D4AD4EF, 0x676E2E0F, 0x2D47932D, 0x776BB33B, - 0x6DE1902B, 0x2C3F8741, 0x5B2DE8EF, 0x686DDB3B, 0x1D7C61C7, 0x1B061633, 0x3229EA51, 0x7FCB0E63, 0x5F22F4C9, 0x517A7199, - 0x2A8D7973, 0x10DCD257, 0x41D59B27, 0x2C61CA67, 0x2020174F, 0x71653B01, 0x2FE464DD, 0x3E7ED6C7, 0x164D2A71, 0x5D4F3141, - 0x5F7BABA7, 0x50E1C011, 0x140F5D77, 0x34E80809, 0x04AAC6B3, 0x29C42BAB, 0x08F9B6F7, 0x461E62FD, 0x45C2660B, 0x08BF25A7, - 0x5494EA7B, 0x0225EBB7, 0x3C5A47CF, 0x2701C333, 0x457ED05B, 0x48CDDE55, 0x14083099, 0x7C69BDAB, 0x7BF163C9, 0x41EE1DAB, - 0x258B1307, 0x0FFAD43B, 0x6601D767, 0x214DBEC7, 0x2852CCF5, 0x0009B471, 0x190AC89D, 0x5BDFB907, 0x15D4E331, 0x15D22375, - 0x13F388D5, 0x12ACEDA5, 0x3835EA5D, 0x2587CA35, 0x06756643, 0x487C6F55, 0x65C295EB, 0x1029F2E1, 0x10CEF39D, 0x14C2E415, - 0x444825BB, 0x24BE0A2F, 0x1D2B7C01, 0x64AE3235, 0x5D2896E5, 0x61BBBD87, 0x4A49E86D, 0x12C277FF, 0x72C81289, 0x5CF42A3D, - 0x332FF177, 0x0DAECD23, 0x6000ED1D, 0x203CDDE1, 0x40C62CAD, 0x19B9A855, 0x782020C3, 0x6127D5BB, 0x719889A7, 0x40E4FCCF, - 0x2A3C8FF9, 0x07411C7F, 0x3113306B, 0x4D7CA03F, 0x76119841, 0x54CEFBDF, 0x11548AB9, 0x4B0748EB, 0x569966B1, 0x45BC721B, - 0x3D5A376B, 0x0D8923E9, 0x6D95514D, 0x0F39A367, 0x2FDAD92F, 0x721F972F, 0x42D0E21D, 0x5C5952DB, 0x7394D007, 0x02692C55, - 0x7F92772F, 0x025F8025, 0x34347113, 0x560EA689, 0x0DCC21DF, 0x09ECC7F5, 0x091F3993, 0x0E0B52AB, 0x497CAA55, 0x0A040A49, - 0x6D8F0CC5, 0x54F41609, 0x6E0CB8DF, 0x3DCB64C3, 0x16C365CD, 0x6D6B9FB5, 0x02B9382B, 0x6A5BFAF1, 0x1669D75F, 0x13CFD4FD, - 0x0FDF316F, 0x21F3C463, 0x6FC58ABF, 0x04E45BE7, 0x1911225B, 0x28CD1355, 0x222084E9, 0x672AD54B, 0x476FC267, 0x6864E16D, - 0x20AEF4FB, 0x603C5FB9, 0x55090595, 0x1113B705, 0x24E38493, 0x5291AF97, 0x5F5446D9, 0x13A6F639, 0x3D501313, 0x37E02017, - 0x236B0ED3, 0x60F246BF, 0x01E02501, 0x2D2F66BD, 0x6BF23609, 0x16729BAF}; - -/* IMPLEMENTATION */ - -template inline Vec::Vec() : n(0), i(0), v(nullptr) -{ - memset(static_cast(&e[0]), 0, sizeof(e)); -} - -template inline Vec::Vec(const Vec &vv) -{ - copy(vv); -} - -template inline Vec::Vec(C c) -{ - n = 1; - i = 0; - v = &e[0]; - e[0] = c; -} - -template -inline C -Vec::get(size_t i) const -{ - if (i < n) { - return v[i]; - } else { - return C(); - } -} - -template -inline void -Vec::add(C a) -{ - if (n & (VEC_INTEGRAL_SIZE - 1)) - v[n++] = a; - else if (!v) - (v = e)[n++] = a; - else - add_internal(a); -} - -template -inline C & -Vec::add() -{ - C *ret; - if (n & (VEC_INTEGRAL_SIZE - 1)) - ret = &v[n++]; - else if (!v) - ret = &(v = e)[n++]; - else - ret = &add_internal(); - return *ret; -} - -template -inline void -Vec::drop() -{ - if (n && 0 == --n) - clear(); -} - -template -inline C -Vec::pop() -{ - if (!n) - return 0; - n--; - C ret = v[n]; - if (!n) - clear(); - return ret; -} - -template -inline void -Vec::set_clear() -{ - memset(v, 0, n * sizeof(C)); -} - -template -inline C * -Vec::set_add(C a) -{ - if (n < SET_LINEAR_SIZE) { - for (C *c = v; c < v + n; c++) - if (*c == a) - return nullptr; - add(a); - return &v[n - 1]; - } - if (n == SET_LINEAR_SIZE) { - Vec vv(*this); - clear(); - for (C *c = vv.v; c < vv.v + vv.n; c++) { - set_add_internal(*c); - } - } - return set_add_internal(a); -} - -template -void -Vec::set_remove(C a) -{ - Vec tmp; - tmp.move(*this); - for (C *c = tmp.v; c < tmp.v + tmp.n; c++) - if (*c != a) - set_add(a); -} - -template -inline size_t -Vec::count() const -{ - int x = 0; - for (C *c = v; c < v + n; c++) - if (*c) - x++; - return x; -} - -template -inline C * -Vec::in(C a) -{ - for (C *c = v; c < v + n; c++) - if (*c == a) - return c; - return nullptr; -} - -template -inline bool -Vec::add_exclusive(C a) -{ - if (!in(a)) { - add(a); - return true; - } else - return false; -} - -template -inline C * -Vec::set_in(C a) -{ - if (n <= SET_LINEAR_SIZE) - return in(a); - return set_in_internal(a); -} - -template -inline C -Vec::first_in_set() -{ - for (C *c = v; c < v + n; c++) - if (*c) - return *c; - return 0; -} - -template -inline ssize_t -Vec::index(C a) const -{ - for (C *c = v; c < v + n; c++) { - if (*c == a) { - return c - v; - } - } - return -1; -} - -template -inline void -Vec::move_internal(Vec &vv) -{ - n = vv.n; - i = vv.i; - if (vv.v == &vv.e[0]) { - memcpy(e, &vv.e[0], sizeof(e)); - v = e; - } else - v = vv.v; -} - -template -inline void -Vec::move(Vec &vv) -{ - move_internal(vv); - vv.v = nullptr; - vv.clear(); -} - -template -inline void -Vec::copy(const Vec &vv) -{ - n = vv.n; - i = vv.i; - if (vv.v == &vv.e[0]) { - memcpy(e, &vv.e[0], sizeof(e)); - v = e; - } else { - if (vv.v) - copy_internal(vv); - else - v = nullptr; - } -} - -template -inline void -Vec::fill(size_t nn) -{ - for (size_t i = n; i < nn; i++) - add() = 0; -} - -template -inline void -Vec::append(const Vec &vv) -{ - for (C *c = vv.v; c < vv.v + vv.n; c++) - if (*c != 0) - add(*c); -} - -template -template -inline void -Vec::append(const C *src, CountType count) -{ - reserve(length() + count); - for (CountType c = 0; c < count; ++c) { - add(src[c]); - } -} - -template -inline void -Vec::prepend(const Vec &vv) -{ - if (vv.n) { - int oldn = n; - fill(n + vv.n); - if (oldn) - memmove(&v[vv.n], &v[0], oldn * sizeof(v[0])); - memcpy(&v[0], vv.v, vv.n * sizeof(v[0])); - } -} - -template -void -Vec::add_internal(C a) -{ - addx(); - v[n++] = a; -} - -template -C & -Vec::add_internal() -{ - addx(); - return v[n++]; -} - -template -C * -Vec::set_add_internal(C c) -{ - size_t j, k; - if (n) { - uintptr_t h = (uintptr_t)c; - h = h % n; - for (k = h, j = 0; j < i + 3; j++) { - if (!v[k]) { - v[k] = c; - return &v[k]; - } else if (v[k] == c) { - return nullptr; - } - k = (k + open_hash_primes[j]) % n; - } - } - Vec vv; - vv.move_internal(*this); - set_expand(); - if (vv.v) { - set_union(vv); - } - return set_add(c); -} - -template -C * -Vec::set_in_internal(C c) -{ - size_t j, k; - if (n) { - uintptr_t h = (uintptr_t)c; - h = h % n; - for (k = h, j = 0; j < i + 3; j++) { - if (!v[k]) - return nullptr; - else if (v[k] == c) - return &v[k]; - k = (k + open_hash_primes[j]) % n; - } - } - return nullptr; -} - -template -bool -Vec::set_union(Vec &vv) -{ - bool changed = false; - for (size_t i = 0; i < vv.n; i++) { - if (vv.v[i]) { - changed = set_add(vv.v[i]) || changed; - } - } - return changed; -} - -template -int -Vec::set_intersection(Vec &vv) -{ - Vec tv; - tv.move(*this); - int changed = 0; - for (int i = 0; i < tv.n; i++) - if (tv.v[i]) { - if (vv.set_in(tv.v[i])) - set_add(tv.v[i]); - else - changed = 1; - } - return changed; -} - -template -int -Vec::some_intersection(Vec &vv) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (vv.set_in(v[i])) - return 1; - return 0; -} - -template -int -Vec::some_disjunction(Vec &vv) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (!vv.set_in(v[i])) - return 1; - for (int i = 0; i < vv.n; i++) - if (vv.v[i]) - if (!set_in(vv.v[i])) - return 1; - return 0; -} - -template -void -Vec::set_intersection(Vec &vv, Vec &result) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (vv.set_in(v[i])) - result.set_add(v[i]); -} - -template -void -Vec::set_disjunction(Vec &vv, Vec &result) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (!vv.set_in(v[i])) - result.set_add(v[i]); - for (int i = 0; i < vv.n; i++) - if (vv.v[i]) - if (!set_in(vv.v[i])) - result.set_add(vv.v[i]); -} - -template -void -Vec::set_difference(Vec &vv, Vec &result) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (!vv.set_in(v[i])) - result.set_add(v[i]); -} - -template -int -Vec::some_difference(Vec &vv) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (!vv.set_in(v[i])) - return 1; - return 0; -} - -template -size_t -Vec::set_count() const -{ - size_t x = 0; - for (size_t i = 0; i < n; i++) { - if (v[i]) { - x++; - } - } - return x; -} - -template -void -Vec::set_to_vec() -{ - C *x = &v[0], *y = x; - for (; y < v + n; y++) { - if (*y) { - if (x != y) - *x = *y; - x++; - } - } - if (i) { - i = prime2[i]; // convert set allocation to reserve - if (i - n > 0) - memset(&v[n], 0, (i - n) * (sizeof(C))); - } else { - i = 0; - if (v == &e[0] && VEC_INTEGRAL_SIZE - n > 0) - memset(&v[n], 0, (VEC_INTEGRAL_SIZE - n) * (sizeof(C))); - } -} - -template -void -Vec::vec_to_set() -{ - Vec vv; - vv.move(*this); - for (C *c = vv.v; c < vv.v + vv.n; c++) - set_add(*c); -} - -template -void -Vec::remove_index(int index) -{ - if (n > 1) - memmove(&v[index], &v[index + 1], (n - 1 - index) * sizeof(v[0])); - n--; - if (n <= 0) - v = e; -} - -template -void -Vec::insert(size_t index, C a) -{ - add(); - memmove(&v[index + 1], &v[index], (n - index - 1) * sizeof(C)); - v[index] = a; -} - -template -void -Vec::insert(size_t index, Vec &vv) -{ - fill(n + vv.n); - memmove(&v[index + vv.n], &v[index], (n - index - 1) * sizeof(C)); - for (int x = 0; x < vv.n; x++) - v[index + x] = vv[x]; -} - -template -C & -Vec::insert(size_t index) -{ - add(); - memmove(&v[index + 1], &v[index], (n - index - 1) * sizeof(C)); - memset(&v[index], 0, sizeof(C)); - return v[index]; -} - -template -void -Vec::reverse() -{ - for (int i = 0; i < n / 2; i++) { - C *s = &v[i], *e = &v[n - 1 - i]; - C t; - memcpy(&t, s, sizeof(t)); - memcpy(s, e, sizeof(t)); - memcpy(e, &t, sizeof(t)); - } -} - -template -void -Vec::copy_internal(const Vec &vv) -{ - int l = n, nl = (1 + VEC_INITIAL_SHIFT); - l = l >> VEC_INITIAL_SHIFT; - while (l) { - l = l >> 1; - nl++; - } - nl = 1 << nl; - v = (C *)A::alloc(nl * sizeof(C)); - memcpy(static_cast(v), vv.v, n * sizeof(C)); - memset(static_cast(v + n), 0, (nl - n) * sizeof(C)); - if (i > n) // reset reserve - i = 0; -} - -template -void -Vec::set_expand() -{ - if (!n) - i = SET_INITIAL_INDEX; - else - i = i + 1; - n = prime2[i]; - v = (C *)A::alloc(n * sizeof(C)); - memset(static_cast(v), 0, n * sizeof(C)); -} - -template -inline void -Vec::reserve(size_t x) -{ - if (x <= n) - return; - unsigned xx = 1 << VEC_INITIAL_SHIFT; - while (xx < x) - xx *= 2; - i = xx; - void *vv = (void *)v; - v = (C *)A::alloc(i * sizeof(C)); - if (vv && n) - memcpy(v, vv, n * sizeof(C)); - memset(&v[n], 0, (i - n) * sizeof(C)); - if (vv && vv != e) - A::free(vv); -} - -template -inline void -Vec::addx() -{ - if (!v) { - v = e; - return; - } - if (v == e) { - v = (C *)A::alloc(VEC_INITIAL_SIZE * sizeof(C)); - memcpy(static_cast(v), &e[0], n * sizeof(C)); - ink_assert(n < VEC_INITIAL_SIZE); - memset(static_cast(&v[n]), 0, (VEC_INITIAL_SIZE - n) * sizeof(C)); - } else { - if ((n & (n - 1)) == 0) { - size_t nl = n * 2; - if (nl <= i) { - return; - } else { - i = 0; - } - void *vv = (void *)v; - v = (C *)A::alloc(nl * sizeof(C)); - memcpy(static_cast(v), vv, n * sizeof(C)); - memset(static_cast(&v[n]), 0, n * sizeof(C)); - A::free(vv); - } - } -} - -template -inline void -Vec::reset() -{ - v = nullptr; - n = 0; - i = 0; -} - -template -inline void -Vec::clear() -{ - if (v && v != e) - A::free(v); - reset(); -} - -template -inline void -Vec::free_and_clear() -{ - for (size_t x = 0; x < (n); x++) - A::free((void *)v[x]); - clear(); -} - -template -inline void -Vec::delete_and_clear() -{ - for (size_t x = 0; x < n; x++) { - if (v[x]) { - delete v[x]; - } - } - clear(); -} - -template inline Vec::~Vec() -{ - if (v && v != e) - A::free(v); -} - -template -inline int -marshal_size(Vec &v) -{ - int l = sizeof(int) * 2; - for (int x = 0; x < v.n; x++) - l += ::marshal_size(v.v[x]); - return l; -} - -template -inline int -marshal(Vec &v, char *buf) -{ - char *x = buf; - *(int *)x = v.n; - x += sizeof(int); - *(int *)x = v.i; - x += sizeof(int); - for (int i = 0; i < v.n; i++) - x += ::marshal(v.v[i], x); - return x - buf; -} - -template -inline int -unmarshal(Vec &v, char *buf) -{ - char *x = buf; - v.n = *(int *)x; - x += sizeof(int); - v.i = *(int *)x; - x += sizeof(int); - if (v.n) { - v.v = (C *)A::alloc(sizeof(C) * v.n); - memset(v.v, 0, sizeof(C) * v.n); - } else - v.v = v.e; - for (int i = 0; i < v.n; i++) - x += ::unmarshal(v.v[i], x); - return x - buf; -} - -template -inline int -Vec::write(int fd) -{ - int r = 0, t = 0; - if ((r = ::write(fd, this, sizeof(*this))) < 0) - return r; - t += r; - if ((r = ::write(fd, v, n * sizeof(C))) < 0) - return r; - t += r; - return t; -} - -template -inline int -Vec::read(int fd) -{ - int r = 0, t = 0; - if ((r = ::read(fd, this, sizeof(*this))) < 0) - return r; - t += r; - v = (C *)A::alloc(sizeof(C) * n); - memset(v, 0, sizeof(C) * n); - if ((r = ::read(fd, v, n * sizeof(C))) < 0) - return r; - t += r; - return t; -} - -template -inline void -Vec::swap(C *p1, C *p2) -{ - C t = *p1; - *p1 = *p2; - *p2 = t; -} - -template -inline void -qsort_Vec(C *left, C *right, bool (*lt)(C, C)) -{ - if (right - left < 5) { - for (C *y = right - 1; y > left; y--) { - for (C *x = left; x < y; x++) { - if (lt(x[1], x[0])) { - C t = x[0]; - x[0] = x[1]; - x[1] = t; - } - } - } - } else { - C *center = left + ((right - left) / 2); - C median; - - // find the median - if (lt(*center, *left)) { // order left and center - Vec::swap(center, left); - } - if (lt(*(right - 1), *left)) { // order left and right - Vec::swap(right - 1, left); - } - if (lt(*(right - 1), *center)) { // order right and center - Vec::swap((right - 1), center); - } - Vec::swap(center, right - 2); // stash the median one from the right for now - median = *(right - 2); // the median of left, center and right values - - // now partition, pivoting on the median value - // l ptr is +1 b/c we already put the lowest of the incoming left, center - // and right in there, ignore it for now - // r ptr is -2 b/c we already put the biggest of the 3 values in (right-1) - // and the median in (right -2) - C *l = left + 1, *r = right - 2; - - // move l and r until they have something to do - while (lt(median, *(r - 1))) { - r--; - } - while (l < r && lt(*l, median)) { - l++; - } - // until l and r meet, - // compare l and median - // swap l for r if l is larger than median - while (l < r) { - if (lt(*l, median)) { - l++; - } else { - Vec::swap(l, r - 1); - r--; - } - } - - Vec::swap(l, right - 2); // restore median to its rightful place - - // recurse for the littles (left segment) - qsort_Vec(left, l, lt); - // recurse for the bigs (right segment) - qsort_Vec(l + 1, right, lt); - } -} - -template -inline void -qsort_VecRef(C *left, C *right, bool (*lt)(const C &, const C &), unsigned int *p_ctr) -{ - if (right - left < 5) { - for (C *y = right - 1; y > left; y--) { - for (C *x = left; x < y; x++) { - if (lt(x[1], x[0])) { - C t = x[0]; - x[0] = x[1]; - x[1] = t; - } - } - } - } else { - C *center = left + ((right - left) / 2); - C median; - - // find the median - if (lt(*center, *left)) { // order left and center - Vec::swap(center, left); - } - if (lt(*(right - 1), *left)) { // order left and right - Vec::swap(right - 1, left); - } - if (lt(*(right - 1), *center)) { // order right and center - Vec::swap((right - 1), center); - } - Vec::swap(center, right - 2); // stash the median one from the right for now - median = *(right - 2); // the median of left, center and right values - - // now partition, pivoting on the median value - // l ptr is +1 b/c we already put the lowest of the incoming left, center - // and right in there, ignore it for now - // r ptr is -2 b/c we already put the biggest of the 3 values in (right-1) - // and the median in (right -2) - C *l = left + 1, *r = right - 2; - - // move l and r until they have something to do - while (lt(median, *(r - 1))) { - r--; - } - while (l < r && lt(*l, median)) { - l++; - } - // until l and r meet, - // compare l and median - // swap l for r if l is larger than median - while (l < r) { - if (lt(*l, median)) { - l++; - } else { - Vec::swap(l, r - 1); - r--; - } - } - - Vec::swap(l, right - 2); // restore median to its rightful place - - // recurse for the littles (left segment) - qsort_VecRef(left, l, lt, p_ctr); - // recurse for the bigs (right segment) - qsort_VecRef(l + 1, right, lt, p_ctr); - } - (*p_ctr)++; -} - -template -inline void -Vec::qsort(bool (*lt)(C, C)) -{ - if (n) - qsort_Vec(&v[0], end(), lt); -} - -template -inline void -Vec::qsort(bool (*lt)(const C &, const C &)) -{ - static unsigned int ctr = 0; - if (n) - qsort_VecRef(&v[0], end(), lt, &ctr); - Debug("qsort", "took %u iterations to sort %ld elements", ctr, n); -} -void test_vec(); - -typedef const char cchar; - -template -static inline char * -_dupstr(cchar *s, cchar *e = nullptr) -{ - int l = e ? e - s : strlen(s); - char *ss = (char *)A::alloc(l + 1); - memcpy(ss, s, l); - ss[l] = 0; - return ss; -} - -// Simple direct mapped Map (pointer hash table) and Environment - -template class MapElem -{ -public: - K key; - C value; - bool - operator==(MapElem &e) - { - return e.key == key; - } - operator uintptr_t(void) { return (uintptr_t)(uintptr_t)key; } - MapElem(K const &akey, C const &avalue) : key(akey), value(avalue) {} - MapElem(MapElem const &e) : key(e.key), value(e.value) {} - MapElem() : key(), value() {} -}; - -template class Map : public Vec, A> -{ -public: - typedef MapElem ME; - typedef Vec PType; - using PType::n; - using PType::i; - using PType::v; - ME *put(K akey, C avalue); - ME *put(K akey); - C get(K akey); - C *getp(K akey); - void get_keys(Vec &keys) const; - void get_keys_set(Vec &keys); - void get_values(Vec &values); - void map_union(Map &m); - bool some_disjunction(Map &m) const; -}; - -template class HashFns -{ -public: - static uintptr_t hash(C a); - static int equal(C a, C b); -}; - -template class HashSetFns -{ -public: - static uintptr_t hash(C a); - static uintptr_t hash(K a); - static int equal(C a, C b); - static int equal(K a, C b); -}; - -template class HashMap : public Map -{ -public: - typedef MapElem value_type; ///< What's stored in the table. - using Map::n; - using Map::i; - using Map::v; - using Map::e; - HashMap() {} - HashMap(C c) : invalid_value(c) {} - MapElem *get_internal(K akey) const; - C get(K akey) const; - value_type *put(K akey, C avalue); - void get_keys(Vec &keys) const; - void get_values(Vec &values); - -private: - C invalid_value = 0; // return this object if key is not present -}; - -#define form_Map(_c, _p, _v) \ - if ((_v).n) \ - for (_c *qq__##_p = (_c *)0, *_p = &(_v).v[0]; ((uintptr_t)(qq__##_p) < (_v).n) && ((_p = &(_v).v[(uintptr_t)qq__##_p]) || 1); \ - qq__##_p = (_c *)(((uintptr_t)qq__##_p) + 1)) \ - if ((_p)->key) - -template class HashSet : public Vec -{ -public: - typedef Vec V; - using V::n; - using V::i; - using V::v; - using V::e; - C get(K akey); - C *put(C avalue); -}; - -class StringHashFns -{ -public: - static uintptr_t - hash(cchar *s) - { - uintptr_t h = 0; - // 31 changed to 27, to avoid prime2 in vec.cpp - while (*s) - h = h * 27 + (unsigned char)*s++; - return h; - } - static int - equal(cchar *a, cchar *b) - { - return !strcmp(a, b); - } -}; - -class CaseStringHashFns -{ -public: - static uintptr_t - hash(cchar *s) - { - uintptr_t h = 0; - // 31 changed to 27, to avoid prime2 in vec.cpp - while (*s) - h = h * 27 + (unsigned char)toupper(*s++); - return h; - } - static int - equal(cchar *a, cchar *b) - { - return !strcasecmp(a, b); - } -}; - -class PointerHashFns -{ -public: - static uintptr_t - hash(void *s) - { - return (uintptr_t)(uintptr_t)s; - } - static int - equal(void *a, void *b) - { - return a == b; - } -}; - -template class ChainHash : public Map, A> -{ -public: - using Map, A>::n; - using Map, A>::v; - typedef ConsCell ChainCons; - C put(C c); - C get(C c); - C put_bag(C c); - int get_bag(C c, Vec &v); - int del(C avalue); - void get_elements(Vec &elements); -}; - -template -class ChainHashMap : public Map, A>, A> -{ -public: - using Map, A>, A>::n; - using Map, A>, A>::v; - MapElem *put(K akey, C avalue); - C get(K akey); - int del(K akey); - MapElem *put_bag(K akey, C c); - int get_bag(K akey, Vec &v); - void get_keys(Vec &keys); - void get_values(Vec &values); -}; - -template class StringChainHash : public ChainHash -{ -public: - cchar *canonicalize(cchar *s, cchar *e); - cchar * - canonicalize(cchar *s) - { - return canonicalize(s, s + strlen(s)); - } -}; - -template class NBlockHash -{ -public: - int n; - int i; - C *v; - C e[N]; - - C * - end() - { - return last(); - } - int - length() - { - return N * n; - } - C *first(); - C *last(); - C put(C c); - C get(C c); - C *assoc_put(C *c); - C *assoc_get(C *c); - int del(C c); - void clear(); - void reset(); - int count(); - void size(int p2); - void copy(const NBlockHash &hh); - void move(NBlockHash &hh); - NBlockHash(); - NBlockHash(NBlockHash &hh) - { - v = e; - copy(hh); - } -}; - -/* use forv_Vec on BlockHashes */ - -#define DEFAULT_BLOCK_HASH_SIZE 4 -template class BlockHash : public NBlockHash -{ -}; -typedef BlockHash StringBlockHash; - -template class Env -{ -public: - typedef ConsCell EnvCons; - void put(K akey, C avalue); - C get(K akey); - void push(); - void pop(); - void - clear() - { - store.clear(); - scope.clear(); - } - - Env() {} - Map *, A> store; - List, A> scope; - List *get_bucket(K akey); -}; - -/* IMPLEMENTATION */ - -template -inline C -Map::get(K akey) -{ - MapElem e(akey, (C)0); - MapElem *x = this->set_in(e); - if (x) - return x->value; - return (C)0; -} - -template -inline C * -Map::getp(K akey) -{ - MapElem e(akey, (C)0); - MapElem *x = this->set_in(e); - if (x) - return &x->value; - return 0; -} - -template -inline MapElem * -Map::put(K akey, C avalue) -{ - MapElem e(akey, avalue); - MapElem *x = this->set_in(e); - if (x) { - x->value = avalue; - return x; - } else - return this->set_add(e); -} - -template -inline MapElem * -Map::put(K akey) -{ - MapElem e(akey, 0); - MapElem *x = this->set_in(e); - if (x) - return x; - else - return this->set_add(e); -} - -template -inline void -Map::get_keys(Vec &keys) const -{ - for (size_t i = 0; i < n; i++) - if (v[i].key) - keys.add(v[i].key); -} - -template -inline void -Map::get_keys_set(Vec &keys) -{ - for (int i = 0; i < n; i++) - if (v[i].key) - keys.set_add(v[i].key); -} - -template -inline void -Map::get_values(Vec &values) -{ - for (int i = 0; i < n; i++) - if (v[i].key) - values.set_add(v[i].value); - values.set_to_vec(); -} - -template -inline void -Map::map_union(Map &m) -{ - for (int i = 0; i < m.n; i++) - if (m.v[i].key) - put(m.v[i].key, m.v[i].value); -} - -template -inline bool -Map::some_disjunction(Map &m) const -{ - for (size_t i = 0; i < m.n; i++) { - if (m.v[i].key && get(m.v[i].key) != m.v[i].value) { - return true; - } - } - for (size_t i = 0; i < n; i++) { - if (v[i].key && m.get(v[i].key) != v[i].value) { - return true; - } - } - return false; -} - -template -inline void -map_set_add(Map *, A> &m, K akey, C avalue) -{ - Vec *v = m.get(akey); - if (!v) - m.put(akey, (v = new Vec)); - v->set_add(avalue); -} - -template -inline void -map_set_add(Map *, A> &m, K akey, Vec *madd) -{ - Vec *v = m.get(akey); - if (!v) - m.put(akey, (v = new Vec)); - v->set_union(*madd); -} - -template -inline C -HashSet::get(K akey) -{ - if (!n) - return 0; - if (n <= MAP_INTEGRAL_SIZE) { - for (C *c = v; c < v + n; c++) - if (c) - if (AHashFns::equal(akey, *c)) - return *c; - return 0; - } - uintptr_t h = AHashFns::hash(akey); - h = h % n; - for (int k = h, j = 0; j < i + 3; j++) { - if (!v[k]) - return 0; - else if (AHashFns::equal(akey, v[k])) - return v[k]; - k = (k + open_hash_primes[j]) % n; - } - return 0; -} - -template -inline C * -HashSet::put(C avalue) -{ - if (n < MAP_INTEGRAL_SIZE) { - if (!v) - v = e; - for (int i = 0; i < n; i++) - if (AHashFns::equal(avalue, v[i])) - return &v[i]; - v[n] = avalue; - n++; - return &v[n - 1]; - } - if (n > MAP_INTEGRAL_SIZE) { - uintptr_t h = AHashFns::hash(avalue); - h = h % n; - for (int k = h, j = 0; j < i + 3; j++) { - if (!v[k]) { - v[k] = avalue; - return &v[k]; - } - k = (k + open_hash_primes[j]) % n; - } - } else - i = SET_INITIAL_INDEX - 1; // will be incremented in set_expand - HashSet vv(*this); - Vec::set_expand(); - for (int i = 0; i < vv.n; i++) - if (vv.v[i]) - put(vv.v[i]); - return put(avalue); -} - -template -inline MapElem * -HashMap::get_internal(K akey) const -{ - if (!n) - return nullptr; - if (n <= MAP_INTEGRAL_SIZE) { - for (MapElem *c = v; c < v + n; c++) - if (c->key) - if (AHashFns::equal(akey, c->key)) - return c; - return nullptr; - } - uintptr_t h = AHashFns::hash(akey); - h = h % n; - for (size_t k = h, j = 0; j < i + 3; j++) { - if (!v[k].key) - return nullptr; - else if (AHashFns::equal(akey, v[k].key)) - return &v[k]; - k = (k + open_hash_primes[j]) % n; - } - return nullptr; -} - -template -inline C -HashMap::get(K akey) const -{ - MapElem *x = get_internal(akey); - if (!x) - return invalid_value; - return x->value; -} - -template -inline MapElem * -HashMap::put(K akey, C avalue) -{ - MapElem *x = get_internal(akey); - if (x) { - x->value = avalue; - return x; - } else { - if (n < MAP_INTEGRAL_SIZE) { - if (!v) - v = e; - v[n].key = akey; - v[n].value = avalue; - n++; - return &v[n - 1]; - } - if (n > MAP_INTEGRAL_SIZE) { - uintptr_t h = AHashFns::hash(akey); - h = h % n; - for (size_t k = h, j = 0; j < i + 3; j++) { - if (!v[k].key) { - v[k].key = akey; - v[k].value = avalue; - return &v[k]; - } - k = (k + open_hash_primes[j]) % n; - } - } else - i = SET_INITIAL_INDEX - 1; // will be incremented in set_expand - } - HashMap vv(*this); - Map::set_expand(); - for (size_t i = 0; i < vv.n; i++) { - if (vv.v && vv.v[i] && vv.v[i].key) { - put(vv.v[i].key, vv.v[i].value); - } - } - return put(akey, avalue); -} - -template -inline void -HashMap::get_keys(Vec &keys) const -{ - Map::get_keys(keys); -} - -template -inline void -HashMap::get_values(Vec &values) -{ - Map::get_values(values); -} - -/* ---------------------------------------------------------------------------------------------- */ -/** A hash map usable by ATS core. - - This class depends on the @c DLL class from @c List.h. It assumes it can uses instances of that - class to store chains of elements. - - Values stored in this container are not destroyed when the container is destroyed. These must be - released by the client. - - Duplicate keys are allowed. Clients must walk the list for multiple entries. - @see @c Location::operator++() - - By default the table automatically expands to limit the average chain length. This can be tuned. If set - to @c MANUAL then the table will expand @b only when explicitly requested to do so by the client. - @see @c ExpansionPolicy - @see @c setExpansionPolicy() - @see @c setExpansionLimit() - @see @c expand() - - All the parameters for the hash map are passed via the template argument @a H. This is a struct - that contains both type definitions and static methods. It must have - - - No state (cheap and risk free to copy). - - - All required methods are static methods. - - @a ID is a @c typedef for the hash type. This is the type of the value produced by the hash function. It must be - a numeric type. - - @a Key is a @c typedef for the key type. This is passed to the @a hash function and used for equality - checking of elements. It is presumed cheap to copy. If the underlying key is not a simple type - then @a Key should be declared as a constant pointer or a constant reference. The hash table - will never attempt to modify a key. - - @a Value is a @c typedef for the value type, the type of the element stored in the hash table. - - @a ListHead is @c typedef for the @c DLL compatible class that can serve as the anchor for a chain of - @a Value instances. This is use both as data to be stored in a bucket and for access to next and - previous pointers from instances of @a Value. - - Method @c hash converts a @c Key to a hash value. The key argument can be by value or by constant reference. - @code - ID hash(Key key); - @endcode - - Method @c key extracts the key from a @c Value instance. - @code - Key key(Value const*); - @endcode - - Method @c equal checks for equality between a @c Key and a @c Value. The key argument can be a - constant reference or by value. The arguments should be @c const if not by value. - - @code - bool equal (Key lhs, Key rhs); - bool equal (Key key, Value const* value); - @endcode - - Example for @c HttpServerSession keyed by the origin server IP address. - - @code - struct Hasher { - typedef uint32_t ID; - typedef sockaddr const* Key; - typedef HttpServerSession Value; - typedef DList(HttpServerSession, ip_hash_link) ListHead; - - static uint32_t hash(sockaddr const* key) { return ats_ip_hash(key); } - static sockaddr const* key(HttpServerSession const* value) { return &value->ip.sa } - static bool equal(sockaddr const* lhs, sockaddr const* rhs) { return ats_ip_eq(lhs, rhs); } - // Alternatively - // static ID hash(Key* key); - // static Key key(Value* value); - // static bool equal(Key lhs, Key rhs); - @endcode - - In @c HttpServerSession is the definition - - @code - LINK(HttpServerSession, ip_hash_link); - @endcode - - which creates the internal links used by @c TSHashTable. - - */ -template -class TSHashTable -{ -public: - typedef TSHashTable self; ///< Self reference type. - - // Make embedded types easier to use by importing them to the class namespace. - typedef H Hasher; ///< Rename and promote. - typedef typename Hasher::ID ID; ///< ID type. - typedef typename Hasher::Key Key; ///< Key type. - typedef typename Hasher::Value Value; ///< Stored value (element) type. - typedef typename Hasher::ListHead ListHead; ///< Anchor for chain. - - /// When the hash table is expanded. - enum ExpansionPolicy { - MANUAL, ///< Client must explicitly expand the table. - AVERAGE, ///< Table expands if average chain length exceeds limit. [default] - MAXIMUM ///< Table expands if any chain length exceeds limit. - }; - - /** Hash bucket. - This is stored in the base array, anchoring the open chaining. - - @internal The default values are selected so that zero initialization is correct. Be careful if you - change that. - */ - struct Bucket { - ListHead m_chain; ///< Chain of elements. - size_t m_count; ///< # of elements in chain. - - /** Internal chain for iteration. - - Iteration is tricky because it needs to skip over empty buckets and detect end of buckets. - Both of these are difficult inside the iterator without excess data. So we chain the - non-empty buckets and let the iterator walk that. This makes end detection easy and - iteration on sparse data fast. If we make it a doubly linked list adding and removing buckets - is very fast as well. - */ - LINK(Bucket, m_link); - - /** Do the values in this bucket have different keys? - - @internal This can have a false positive, but that's OK, better than the expense of being - exact. What we want is to avoid expanding to shorten the chain if it won't help, which it - won't if all the keys are the same. - - @internal Because we've selected the default to be @c false so we can use @c Vec which zero fills empty elements. - */ - bool m_mixed_p; - - /// Default constructor - equivalent to zero filled. - Bucket() : m_count(0), m_mixed_p(false) { ink_zero(m_link); } - }; - - /** Information about locating a value in the hash table. - - An instance of this returned when searching for a key in the table. It can then be used to - check if a matching key was found, and to iterate over equivalent keys. Note this iterator - will touch only values which have a matching key. - - @internal It's not really an iterator, although similar. - @internal we store the ID (hashed key value) for efficiency - we can get the actual key via the - @a m_value member. - */ - struct Location { - Value *m_value; ///< The value located. - Bucket *m_bucket; ///< Containing bucket of value. - ID m_id; ///< ID (hashed key). - size_t m_distance; ///< How many values in the chain we've gone past to get here. - - /// Default constructor - empty location. - Location() : m_value(nullptr), m_bucket(nullptr), m_id(0), m_distance(0) {} - /// Check for location being valid (referencing a value). - bool - isValid() const - { - return nullptr != m_value; - } - - /// Automatically cast to a @c Value* for convenience. - /// @note This lets you assign the return of @c find to a @c Value*. - /// @note This also permits the use of this class directly as a boolean expression. - operator Value *() const { return m_value; } - /// Dereference. - Value &operator*() const { return *m_value; } - /// Dereference. - Value *operator->() const { return m_value; } - /// Find next value with matching key (prefix). - Location & - operator++() - { - if (m_value) - this->advance(); - return *this; - } - /// Find next value with matching key (postfix). - Location & - operator++(int) - { - Location zret(*this); - if (m_value) - this->advance(); - return zret; - } - - protected: - /// Move to next matching value, no checks. - void advance(); - - friend class TSHashTable; - }; - - /** Standard iterator for walking the table. - This iterates over all elements. - @internal Iterator is @a end if @a m_value is @c nullptr. - */ - struct iterator { - Value *m_value; ///< Current location. - Bucket *m_bucket; ///< Current bucket; - - iterator() : m_value(0), m_bucket(0) {} - iterator &operator++(); - iterator operator++(int); - Value &operator*() { return *m_value; } - Value *operator->() { return m_value; } - bool - operator==(iterator const &that) - { - return m_bucket == that.m_bucket && m_value == that.m_value; - } - bool - operator!=(iterator const &that) - { - return !(*this == that); - } - - protected: - /// Internal iterator constructor. - iterator(Bucket *b, Value *v) : m_value(v), m_bucket(b) {} - friend class TSHashTable; - }; - - iterator begin(); ///< First element. - iterator end(); ///< Past last element. - - /// The default starting number of buckets. - static size_t const DEFAULT_BUCKET_COUNT = 7; ///< POOMA. - /// The default expansion policy limit. - static size_t const DEFAULT_EXPANSION_LIMIT = 4; ///< Value from previous version. - - /** Constructor (also default). - Constructs an empty table with at least @a nb buckets. - */ - TSHashTable(size_t nb = DEFAULT_BUCKET_COUNT); - - /** Insert a value in to the table. - The @a value must @b NOT already be in a table of this type. - @note The value itself is put in the table, @b not a copy. - */ - void insert(Value *value); - - /** Find a value that matches @a key. - - @note This finds the first value with a matching @a key. No other properties - of the value are examined. - - @return The @c Location of the value. Use @c Location::isValid() to check for success. - */ - Location find(Key key); - - /** Get a @c Location for @a value. - - This is a bit obscure but needed in certain cases. It should only be used on a @a value that - is already known to be in the table. It just does the bucket lookup and uses that and the @a - value to construct a @c Location that can be used with other methods. The @a m_distance value - is not set in this case for performance reasons. - */ - Location find(Value *value); - - /** Remove the value at @a location from the table. - - This method assumes a @a location is consistent. Be very careful if you modify a @c Location. - - @note This does @b not clean up the removed elements. Use carefully to avoid leaks. - - @return @c true if the value was removed, @c false otherwise. - */ - bool remove(Location const &location); - - /** Remove @b all values with @a key. - - @note This does @b not clean up the removed elements. Use carefully to avoid leaks. - - @return @c true if any value was removed, @c false otherwise. - */ - bool remove(Key key); - - /** Remove all values from the table. - - The values are not cleaned up. The values are not touched in this method, therefore it is safe - to destroy them first and then @c clear this table. - */ - void clear(); - - /// Get the number of elements in the table. - size_t - count() const - { - return m_count; - } - - /// Get the number of buckets in the table. - size_t - bucketCount() const - { - return m_array.n; - } - - /// Enable or disable expanding the table when chains are long. - void - setExpansionPolicy(ExpansionPolicy p) - { - m_expansion_policy = p; - } - /// Get the current expansion policy. - ExpansionPolicy - getExpansionPolicy() const - { - return m_expansion_policy; - } - /// Set the limit value for the expansion policy. - void - setExpansionLimit(size_t n) - { - m_expansion_limit = n; - } - /// Set the limit value for the expansion policy. - size_t - expansionLimit() const - { - return m_expansion_limit; - } - - /** Expand the hash. - - Useful primarily when the expansion policy is set to @c MANUAL. - */ - void expand(); - -protected: - typedef Vec Array; ///< Bucket array. - - size_t m_count; ///< # of elements stored in the table. - ExpansionPolicy m_expansion_policy; ///< When to exand the table. - size_t m_expansion_limit; ///< Limit value for expansion. - Array m_array; ///< Bucket storage. - /// Make available to nested classes statically. - // We must reach inside the link hackery because we're in a template and - // must use typename. Older compilers don't handle typename outside of - // template context so if we put typename in the base definition it won't - // work in non-template classes. - typedef DLL BucketChain; - /// List of non-empty buckets. - BucketChain m_bucket_chain; - - /** Get the ID and bucket for key. - Fills @a m_id and @a m_bucket in @a location from @a key. - */ - void findBucket(Key key, Location &location); - - // noncopyable - TSHashTable(const TSHashTable &) = delete; - TSHashTable &operator=(const TSHashTable &) = delete; -}; - -template -typename TSHashTable::iterator -TSHashTable::begin() -{ - // Get the first non-empty bucket, if any. - Bucket *b = m_bucket_chain.head; - return b && b->m_chain.head ? iterator(b, b->m_chain.head) : this->end(); -} - -template -typename TSHashTable::iterator -TSHashTable::end() -{ - return iterator(nullptr, nullptr); -} - -template -typename TSHashTable::iterator & -TSHashTable::iterator::operator++() -{ - if (m_value) { - if (nullptr == (m_value = ListHead::next(m_value))) { // end of bucket, next bucket. - if (nullptr != (m_bucket = BucketChain::next(m_bucket))) { // found non-empty next bucket. - m_value = m_bucket->m_chain.head; - ink_assert(m_value); // if bucket is in chain, must be non-empty. - } - } - } - return *this; -} - -template -typename TSHashTable::iterator -TSHashTable::iterator::operator++(int) -{ - iterator prev(*this); - ++*this; - return prev; -} - -template -TSHashTable::TSHashTable(size_t nb) : m_count(0), m_expansion_policy(AVERAGE), m_expansion_limit(DEFAULT_EXPANSION_LIMIT) -{ - if (nb) { - int idx = 1; - while (prime2[idx] < nb) - ++idx; - m_array.n = 1; // anything non-zero. - m_array.i = idx - 1; - } - m_array.set_expand(); -} - -template -void -TSHashTable::Location::advance() -{ - Key key = Hasher::key(m_value); - // assumes valid location with correct key, advance to next matching key or make location invalid. - do { - ++m_distance; - m_value = ListHead::next(m_value); - } while (m_value && !Hasher::equal(key, Hasher::key(m_value))); -} - -template -void -TSHashTable::findBucket(Key key, Location &location) -{ - location.m_id = Hasher::hash(key); - location.m_bucket = &(m_array[location.m_id % m_array.n]); -} - -template -typename TSHashTable::Location -TSHashTable::find(Key key) -{ - Location zret; - Value *v; - - this->findBucket(key, zret); // zret gets updated to match the bucket. - v = zret.m_bucket->m_chain.head; - // Search for first matching key. - while (nullptr != v && !Hasher::equal(key, Hasher::key(v))) - v = ListHead::next(v); - zret.m_value = v; - return zret; -} - -template -typename TSHashTable::Location -TSHashTable::find(Value *value) -{ - Location zret; - this->findBucket(Hasher::key(value), zret); - if (zret.m_bucket->m_chain.in(value)) // just checks value links and chain head. - zret.m_value = value; - return zret; -} - -template -void -TSHashTable::insert(Value *value) -{ - Key key = Hasher::key(value); - Bucket *bucket = &(m_array[Hasher::hash(key) % m_array.n]); - - // Bad client if already in a list! - ink_assert(!bucket->m_chain.in(value)); - - // Mark mixed if not already marked and we're adding a different key. - if (!bucket->m_mixed_p && !bucket->m_chain.empty() && !Hasher::equal(key, Hasher::key(bucket->m_chain.head))) - bucket->m_mixed_p = true; - - bucket->m_chain.push(value); - ++m_count; - if (1 == ++(bucket->m_count)) // not empty, put it on the non-empty list. - m_bucket_chain.push(bucket); - // auto expand if appropriate. - if ((AVERAGE == m_expansion_policy && (m_count / m_array.n) > m_expansion_limit) || - (MAXIMUM == m_expansion_policy && bucket->m_count > m_expansion_limit && bucket->m_mixed_p)) - this->expand(); -} - -template -bool -TSHashTable::remove(Location const &l) -{ - bool zret = false; - if (l.isValid()) { - ink_assert(l.m_bucket->m_count); - ink_assert(l.m_bucket->m_chain.head); - l.m_bucket->m_chain.remove(l.m_value); - --m_count; - --(l.m_bucket->m_count); - if (0 == l.m_bucket->m_count) // if it's now empty, take it out of the non-empty bucket chain. - m_bucket_chain.remove(l.m_bucket); - else if (1 == l.m_bucket->m_count) // if count drops to 1, then it's not mixed any more. - l.m_bucket->m_mixed_p = false; - zret = true; - } - return zret; -} - -template -bool -TSHashTable::remove(Key key) -{ - Location loc = this->find(key); - bool zret = loc.isValid(); - while (loc.isValid()) { - Location target(loc); - loc.advance(); - this->remove(target); - } - return zret; -} - -template -void -TSHashTable::clear() -{ - Bucket null_bucket; - // Remove the values but not the actual buckets. - for (size_t i = 0; i < m_array.n; ++i) { - m_array[i] = null_bucket; - } - // Clear container data. - m_count = 0; - m_bucket_chain.clear(); -} - -template -void -TSHashTable::expand() -{ - Bucket *b = m_bucket_chain.head; // stash before reset. - ExpansionPolicy org_expansion_policy = m_expansion_policy; - Array tmp; - tmp.move(m_array); // stash the current array here. - // Reset to empty state. - m_count = 0; - m_bucket_chain.clear(); - - // Because we moved the array, we have to copy back a couple of things to make - // the expansion actually expand. How this is supposed to work without leaks or - // mucking about in the internal is unclear to me. - m_array.n = 1; // anything non-zero. - m_array.i = tmp.i; // set the base index. - m_array.set_expand(); // bumps array size up to next index value. - - m_expansion_policy = MANUAL; // disable any auto expand while we're expanding. - // Move the values from the stashed array to the expanded hash. - while (b) { - Value *v = b->m_chain.head; - while (v) { - b->m_chain.remove(v); // clear local pointers to be safe. - this->insert(v); - v = b->m_chain.head; // next value, because previous was removed. - } - b = BucketChain::next(b); // these buckets are in the stashed array so pointers are still valid. - } - // stashed array gets cleaned up when @a tmp goes out of scope. - m_expansion_policy = org_expansion_policy; // reset to original value. -} - -/* ---------------------------------------------------------------------------------------------- */ diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 488c1cdaa8e..739abbb10b7 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -98,7 +98,7 @@ class SNI_IpAllow : public ActionItem IpMap ip_map; public: - SNI_IpAllow(std::string const &ip_allow_list, cchar *servername) + SNI_IpAllow(std::string const &ip_allow_list, const char *servername) { // the server identified by item.fqdn requires ATS to do IP filtering if (ip_allow_list.length()) { @@ -147,5 +147,5 @@ class SNIActionPerformer { public: SNIActionPerformer() = default; - static int PerformAction(Continuation *cont, cchar *servername); + static int PerformAction(Continuation *cont, const char *servername); }; diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index aa86ad9ce0f..b67944ba8b8 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -127,7 +127,7 @@ struct SSLConfigParams : public ConfigInfo { mutable ink_mutex ctxMapLock; SSL_CTX *getClientSSL_CTX(void) const; - SSL_CTX *getNewCTX(cchar *client_cert, cchar *key_file) const; + SSL_CTX *getNewCTX(const char *client_cert, const char *key_file) const; void initialize(); void cleanup(); diff --git a/iocore/net/P_SSLNetProcessor.h b/iocore/net/P_SSLNetProcessor.h index b90266bfce9..4b15f001424 100644 --- a/iocore/net/P_SSLNetProcessor.h +++ b/iocore/net/P_SSLNetProcessor.h @@ -42,7 +42,6 @@ #include "P_Net.h" #include "P_SSLConfig.h" #include -#include "tscore/Map.h" class UnixNetVConnection; struct NetAccept; diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index 05318a71915..c6545aba3b7 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -60,13 +60,13 @@ struct SNIConfigParams : public ConfigInfo { NextHopPropertyTable next_hop_table; NextHopPropertyTable wild_next_hop_table; YamlSNIConfig Y_sni; - NextHopProperty *getPropertyConfig(cchar *servername) const; + NextHopProperty *getPropertyConfig(const char *servername) const; SNIConfigParams(); ~SNIConfigParams() override; void cleanup(); int Initialize(); void loadSNIConfig(); - actionVector *get(cchar *servername) const; + actionVector *get(const char *servername) const; void printSNImap() const; }; diff --git a/iocore/net/SNIActionPerformer.cc b/iocore/net/SNIActionPerformer.cc index fa5116ee361..842345e34b9 100644 --- a/iocore/net/SNIActionPerformer.cc +++ b/iocore/net/SNIActionPerformer.cc @@ -38,7 +38,7 @@ extern std::unordered_map snpsMap; int -SNIActionPerformer::PerformAction(Continuation *cont, cchar *servername) +SNIActionPerformer::PerformAction(Continuation *cont, const char *servername) { SNIConfig::scoped_config params; auto actionvec = params->get(servername); diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 4948a2d2822..ba03ba4a32a 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -473,7 +473,7 @@ SSLConfigParams::initialize() // creates a new context attaching the provided certificate SSL_CTX * -SSLConfigParams::getNewCTX(cchar *client_cert, cchar *client_key) const +SSLConfigParams::getNewCTX(const char *client_cert, const char *client_key) const { SSL_CTX *nclient_ctx = nullptr; nclient_ctx = SSLInitClientContext(this); diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 8790883bf58..c16d52daac9 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -999,7 +999,7 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) // Making the check here instead of later, so we only // do this setting immediately after we create the SSL object SNIConfig::scoped_config sniParam; - cchar *serverKey = this->options.sni_servername; + const char *serverKey = this->options.sni_servername; if (!serverKey) { ats_ip_ntop(this->get_remote_addr(), buff, INET6_ADDRSTRLEN); serverKey = buff; diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index aa963a3a971..ad988c8fdf6 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -43,7 +43,7 @@ extern TunnelHashMap TunnelMap; NextHopProperty::NextHopProperty() {} NextHopProperty * -SNIConfigParams::getPropertyConfig(cchar *servername) const +SNIConfigParams::getPropertyConfig(const char *servername) const { if (auto it = next_hop_table.find(servername); it != next_hop_table.end()) { return it->second; @@ -60,7 +60,7 @@ SNIConfigParams::loadSNIConfig() for (const auto &item : Y_sni.items) { actionVector *aiVec = new actionVector(); Debug("ssl", "name: %s", item.fqdn.data()); - cchar *servername = item.fqdn.data(); + const char *servername = item.fqdn.data(); ats_wildcard_matcher w_Matcher; auto wildcard = w_Matcher.match(servername); @@ -87,9 +87,9 @@ SNIConfigParams::loadSNIConfig() aiVec->push_back(ai3); // set the next hop properties SSLConfig::scoped_config params; - auto clientCTX = params->getClientSSL_CTX(); - cchar *certFile = item.client_cert.data(); - cchar *keyFile = item.client_key.data(); + auto clientCTX = params->getClientSSL_CTX(); + const char *certFile = item.client_cert.data(); + const char *keyFile = item.client_key.data(); if (certFile) { clientCTX = params->getNewCTX(certFile, keyFile); } @@ -114,7 +114,7 @@ int SNIConfig::configid = 0; SNIConfigParams::SNIConfigParams() {} actionVector * -SNIConfigParams::get(cchar *servername) const +SNIConfigParams::get(const char *servername) const { auto action_it = sni_action_map.find(servername); if (action_it != sni_action_map.end()) { diff --git a/iocore/net/SSLSessionCache.h b/iocore/net/SSLSessionCache.h index 548359bc92f..f1ccbe395b1 100644 --- a/iocore/net/SSLSessionCache.h +++ b/iocore/net/SSLSessionCache.h @@ -21,7 +21,6 @@ #pragma once -#include "tscore/Map.h" #include "tscore/List.h" #include "tscore/ink_mutex.h" #include "P_EventSystem.h" diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index 0ab50153561..d3a312c5f22 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -170,7 +170,6 @@ libtscore_la_SOURCES = \ List.h \ llqueue.cc \ lockfile.cc \ - Map.h \ MatcherUtils.cc \ MatcherUtils.h \ MemSpan.h \ @@ -257,7 +256,6 @@ test_tscore_SOURCES = \ unit_tests/test_IntrusivePtr.cc \ unit_tests/test_IpMap.cc \ unit_tests/test_layout.cc \ - unit_tests/test_Map.cc \ unit_tests/test_List.cc \ unit_tests/test_MemArena.cc \ unit_tests/test_MT_hashtable.cc \ @@ -266,8 +264,7 @@ test_tscore_SOURCES = \ unit_tests/test_Regex.cc \ unit_tests/test_Scalar.cc \ unit_tests/test_scoped_resource.cc \ - unit_tests/test_ts_file.cc \ - unit_tests/test_Vec.cc + unit_tests/test_ts_file.cc CompileParseRules_SOURCES = CompileParseRules.cc diff --git a/src/tscore/suppression.txt b/src/tscore/suppression.txt index d76f93e1824..7e7e361b85d 100644 --- a/src/tscore/suppression.txt +++ b/src/tscore/suppression.txt @@ -1,4 +1,3 @@ leak:CRYPTO_malloc leak:CRYPTO_realloc leak:ConsCell -leak:Vec, DefaultAlloc, 2>::set_expand diff --git a/src/tscore/unit_tests/test_Map.cc b/src/tscore/unit_tests/test_Map.cc deleted file mode 100644 index a0a98860774..00000000000 --- a/src/tscore/unit_tests/test_Map.cc +++ /dev/null @@ -1,215 +0,0 @@ -/** @file - - Test code for the Map templates. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "catch.hpp" - -#include -#include "tscore/Map.h" -#include - -using cchar = const char; - -struct Item { - LINK(Item, m_link); - struct Hash { - using ID = uint32_t; - using Key = uint32_t; - using Value = Item; - using ListHead = DLL; - - static ID - hash(Key key) - { - return key; - } - static Key key(Value *); - static bool equal(Key lhs, Key rhs); - }; - - uint32_t _key; - uint32_t _value; - - Item(uint32_t x) : _key(x), _value(x) {} - Item(uint32_t key, uint32_t value) : _key(key), _value(value) {} -}; - -uint32_t -Item::Hash::key(Value *v) -{ - return v->_key; -} -bool -Item::Hash::equal(Key lhs, Key rhs) -{ - return lhs == rhs; -} - -using Table = TSHashTable; - -class testHashMap -{ -private: - HashMap testsh; - -public: - int - get(cchar *ch) const - { - return testsh.get(ch); - } - - void - put(cchar *key, int v) - { - testsh.put(key, v); - } -}; - -TEST_CASE("test Map", "[libts][Map]") -{ - typedef Map SSMap; - typedef MapElem SSMapElem; - testHashMap testsh; -#define form_SSMap(_p, _v) form_Map(SSMapElem, _p, _v) - SSMap ssm; - ssm.put("a", "A"); - ssm.put("b", "B"); - ssm.put("c", "C"); - ssm.put("d", "D"); - form_SSMap(x, ssm) - { /* nop */ - } - - /* - if ((ssm).n) - for (SSMapElem *qq__x = (SSMapElem*)0, *x = &(ssm).v[0]; - ((intptr_t)(qq__x) < (ssm).n) && ((x = &(ssm).v[(intptr_t)qq__x]) || 1); - qq__x = (SSMapElem*)(((intptr_t)qq__x) + 1)) - if ((x)->key) { - // nop - } - */ - - cchar *hi = "hi", *ho = "ho", *hum = "hum", *hhi = "hhi"; - - ++hhi; - HashMap sh; - sh.put(hi, 1); - sh.put(ho, 2); - sh.put(hum, 3); - sh.put(hhi, 4); - REQUIRE(sh.get(hi) == 4); - REQUIRE(sh.get(ho) == 2); - REQUIRE(sh.get(hum) == 3); - sh.put("aa", 5); - sh.put("ab", 6); - sh.put("ac", 7); - sh.put("ad", 8); - sh.put("ae", 9); - sh.put("af", 10); - REQUIRE(sh.get(hi) == 4); - REQUIRE(sh.get(ho) == 2); - REQUIRE(sh.get(hum) == 3); - REQUIRE(sh.get("af") == 10); - REQUIRE(sh.get("ac") == 7); - - HashMap sh2(-99); // return -99 if key not found - sh2.put("aa", 15); - sh2.put("ab", 16); - testsh.put("aa", 15); - testsh.put("ab", 16); - REQUIRE(sh2.get("aa") == 15); - REQUIRE(sh2.get("ac") == -99); - REQUIRE(testsh.get("aa") == 15); - - // test_TSHashTable - static uint32_t const N = 270; - Table t; - Item *item = nullptr; - Table::Location loc; - std::list to_delete; - - for (uint32_t i = 1; i <= N; ++i) { - item = new Item(i); - t.insert(item); - to_delete.push_back(item); - } - - int failures = 0; - for (uint32_t i = 1; i <= N; ++i) { - Table::Location l = t.find(i); - if (!l.isValid() || i != l->_value) { - failures++; - } - } - REQUIRE(failures == 0); - - REQUIRE(!(t.find(N * 2).isValid())); - - loc = t.find(N / 2 | 1); - if (loc) { - t.remove(loc); - } else { - REQUIRE(!"Did not find expected value"); - } - - if (!loc) { - ; // compiler check. - } - - REQUIRE(!(t.find(N / 2 | 1).isValid())); - - for (uint32_t i = 1; i <= N; i += 2) { - t.remove(i); - } - - failures = 0; - for (uint32_t i = 1; i <= N; ++i) { - Table::Location l = t.find(i); - if (1 & i) { - if (l.isValid()) { - failures++; - } - } else { - if (!l.isValid()) { - failures++; - } - } - } - REQUIRE(failures == 0); - - int n = 0; - failures = 0; - for (Table::iterator spot = t.begin(), limit = t.end(); spot != limit; ++spot) { - ++n; - if ((spot->_value & 1) != 0) { - failures++; - } - } - REQUIRE(failures == 0); - REQUIRE(n == N / 2); - - for (auto it : to_delete) { - delete it; - } -} diff --git a/src/tscore/unit_tests/test_Vec.cc b/src/tscore/unit_tests/test_Vec.cc deleted file mode 100644 index 0ae6af65b98..00000000000 --- a/src/tscore/unit_tests/test_Vec.cc +++ /dev/null @@ -1,388 +0,0 @@ -/* -*-Mode: c++;-*- - Various vector related code. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* UnionFind after Tarjan */ - -#include "catch.hpp" - -#include -#include -#include "tscore/ink_assert.h" -#include "tscore/Map.h" - -// Intervals store sets in interval format (e.g. [1..10][12..12]). -// Inclusion test is by binary search on intervals. -// Deletion is not supported -class Intervals : public Vec -{ -public: - void insert(int n); - bool in(int n) const; -}; - -// UnionFind supports fast unify and finding of -// 'representitive elements'. -// Elements are numbered from 0 to N-1. -class UnionFind : public Vec -{ -public: - // set number of elements, initialized to singletons, may be called repeatedly to increase size - void size(int n); - // return representitive element - int find(int n); - // unify the sets containing the two elements - void unify(int n, int m); -}; - -// binary search over intervals -static int -i_find(const Intervals *i, int x) -{ - ink_assert(i->n); - int l = 0, h = i->n; -Lrecurse: - if (h <= l + 2) { - if (h <= l) { - return -(l + 1); - } - if (x < i->v[l] || x > i->v[l + 1]) { - return -(l + 1); - } - return h; - } - int m = (((h - l) / 4) * 2) + l; - if (x > i->v[m + 1]) { - l = m; - goto Lrecurse; - } - if (x < i->v[m]) { - h = m; - goto Lrecurse; - } - return (l + 1); -} - -bool -Intervals::in(int x) const -{ - if (!n) { - return false; - } - if (i_find(this, x) > 0) { - return true; - } - return false; -} - -// insert into interval with merge -void -Intervals::insert(int x) -{ - if (!n) { - add(x); - add(x); - return; - } - int l = i_find(this, x); - if (l > 0) { - return; - } - l = -l - 1; - - if (x > v[l + 1]) { - if (x == v[l + 1] + 1) { - v[l + 1]++; - goto Lmerge; - } - l += 2; - if (l < (int)n) { - if (x == v[l] - 1) { - v[l]--; - goto Lmerge; - } - } - goto Lmore; - } else { - ink_assert(x < v[l]); - if (x == v[l] - 1) { - v[l]--; - goto Lmerge; - } - if (!l) { - goto Lmore; - } - l -= 2; - if (x == v[l + 1] + 1) { - v[l + 1]++; - goto Lmerge; - } - } -Lmore: - fill(n + 2); - if (n - 2 - l > 0) { - memmove(v + l + 2, v + l, sizeof(int) * (n - 2 - l)); - } - v[l] = x; - v[l + 1] = x; - return; -Lmerge: - if (l) { - if (v[l] - v[l - 1] < 2) { - l -= 2; - goto Ldomerge; - } - } - if (l < (int)(n - 2)) { - if (v[l + 2] - v[l + 1] < 2) { - goto Ldomerge; - } - } - return; -Ldomerge: - memmove(v + l + 1, v + l + 3, sizeof(int) * (n - 3 - l)); - n -= 2; - goto Lmerge; -} - -void -UnionFind::size(int s) -{ - size_t nn = n; - fill(s); - for (size_t i = nn; i < n; i++) { - v[i] = -1; - } -} - -int -UnionFind::find(int n) -{ - int i, t; - for (i = n; v[i] >= 0; i = v[i]) { - ; - } - while (v[n] >= 0) { - t = n; - n = v[n]; - v[t] = i; - } - return i; -} - -void -UnionFind::unify(int n, int m) -{ - n = find(n); - m = find(m); - if (n != m) { - if (v[m] < v[n]) { - v[m] += (v[n] - 1); - v[n] = m; - } else { - v[n] += (v[m] - 1); - v[m] = n; - } - } -} - -TEST_CASE("test append", "[Vec]") -{ - static const char value[] = "this is a string"; - unsigned int len = (int)sizeof(value) - 1; - - Vec str; - - str.append(value, 0); - REQUIRE(str.length() == 0); - - str.append(value, len); - REQUIRE(memcmp(&str[0], value, len) == 0); - REQUIRE(str.length() == len); - - str.clear(); - REQUIRE(str.length() == 0); - - int failures = 0; - for (unsigned i = 0; i < 1000; ++i) { - str.append(value, len); - if (memcmp(&str[i * len], value, len) != 0) { - failures++; - } - } - REQUIRE(failures == 0); - - REQUIRE(str.length() == 1000 * len); -} - -TEST_CASE("test basic", "[libts][Vec]") -{ - Vec v, vv, vvv; - int tt = 99 * 50, t = 0; - - for (size_t i = 0; i < 100; i++) { - v.add((void *)(intptr_t)i); - } - for (size_t i = 0; i < 100; i++) { - t += (int)(intptr_t)v.v[i]; - } - REQUIRE(t == tt); - - t = 0; - for (size_t i = 1; i < 100; i++) { - vv.set_add((void *)(intptr_t)i); - } - for (size_t i = 1; i < 100; i++) { - vvv.set_add((void *)(intptr_t)i); - } - for (size_t i = 1; i < 100; i++) { - vvv.set_add((void *)(intptr_t)(i * 1000)); - } - vv.set_union(vvv); - for (size_t i = 0; i < vv.n; i++) { - if (vv.v[i]) { - t += (int)(intptr_t)vv.v[i]; - } - } - REQUIRE(t == tt + 1000 * tt); - - v.clear(); - v.reserve(1000); - t = 0; - for (size_t i = 0; i < 1000; i++) { - v.add((void *)(intptr_t)i); - } - for (size_t i = 0; i < 1000; i++) { - t += (int)(intptr_t)v.v[i]; - } - REQUIRE(t == 999 * 500); - - Intervals in; - in.insert(1); - REQUIRE(in.n == 2); - in.insert(2); - REQUIRE(in.n == 2); - in.insert(6); - REQUIRE(in.n == 4); - in.insert(7); - REQUIRE(in.n == 4); - in.insert(9); - REQUIRE(in.n == 6); - in.insert(4); - REQUIRE(in.n == 8); - in.insert(5); - REQUIRE(in.n == 6); - in.insert(3); - REQUIRE(in.n == 4); - in.insert(8); - REQUIRE(in.n == 2); - - UnionFind uf; - uf.size(4); - uf.unify(0, 1); - uf.unify(2, 3); - REQUIRE(uf.find(2) == uf.find(3)); - REQUIRE(uf.find(0) == uf.find(1)); - REQUIRE(uf.find(0) != uf.find(3)); - REQUIRE(uf.find(1) != uf.find(3)); - REQUIRE(uf.find(1) != uf.find(2)); - REQUIRE(uf.find(0) != uf.find(2)); - uf.unify(1, 2); - REQUIRE(uf.find(0) == uf.find(3)); - REQUIRE(uf.find(1) == uf.find(3)); -} - -static bool -compare(void *a, void *b) -{ - return a < b; -} - -TEST_CASE("test sort", "[libts][Vec]") -{ - Vec v; - for (long i = 1; i <= 1000; ++i) { - v.add(reinterpret_cast(static_cast(((i * 149) % 1000) + 1))); - } - v.qsort(&compare); - int failures = 0; - for (int i = 0; i < 1000; ++i) { - if (reinterpret_cast(static_cast(i + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); - - v.clear(); - for (long i = 1; i <= 1000000; ++i) { - v.add(reinterpret_cast(static_cast(((i * 51511) % 1000000) + 1))); - } - v.qsort(&compare); - failures = 0; - for (long i = 0; i < 1000000; ++i) { - if (reinterpret_cast(static_cast(i + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); - - v.clear(); - for (long i = 1; i <= 1000000; ++i) { - // This should be every number 1..500000 twice. - v.add(reinterpret_cast(static_cast(((i * 199999) % 500000) + 1))); - } - v.qsort(&compare); - failures = 0; - for (long i = 0; i < 1000000; ++i) { - if (reinterpret_cast(static_cast((i / 2) + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); - - // Very long array, already sorted. This is what broke before. - v.clear(); - for (long i = 1; i <= 10000000; ++i) { - v.add(reinterpret_cast(static_cast(i))); - } - v.qsort(&compare); - failures = 0; - for (long i = 0; i < 10000000; ++i) { - if (reinterpret_cast(static_cast(i + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); - - // very long, reverse sorted. - v.clear(); - for (long i = 10000000; i >= 1; --i) { - v.add(reinterpret_cast(static_cast(i))); - } - v.qsort(&compare); - failures = 0; - for (long i = 0; i < 10000000; ++i) { - if (reinterpret_cast(static_cast(i + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); -} From 63ea263fc3e9dc5905a9782bbd98f30d1756b24d Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Fri, 2 Nov 2018 11:54:34 -0500 Subject: [PATCH 045/526] TCL: Remove TCL from our code base --- README | 6 - build/tcl.m4 | 3307 ----------------- ci/docker/yum/Dockerfile | 2 +- ci/jenkins/jobs.yaml | 2 +- ci/regression | 1 - configure.ac | 20 - contrib/install_trafficserver.sh | 3 - contrib/vagrant-setup.sh | 8 +- doc/Doxyfile | 6 - doc/admin-guide/installation/index.en.rst | 1 - doc/getting-started/index.en.rst | 1 - .../admin-guide/installation/index.en.po | 4 - .../ja/LC_MESSAGES/getting-started.en.po | 4 - .../LC_MESSAGES/getting-started/index.en.po | 4 - include/tscore/ink_hash_table.h | 145 - iocore/aio/Makefile.am | 2 +- iocore/eventsystem/Makefile.am | 2 +- iocore/hostdb/Makefile.am | 2 +- iocore/net/Makefile.am | 2 +- mgmt/api/Makefile.am | 2 +- proxy/hdrs/Makefile.am | 5 +- proxy/http/Makefile.am | 2 +- proxy/http2/Makefile.am | 1 - src/traffic_cache_tool/Makefile.inc | 2 +- src/traffic_crashlog/Makefile.inc | 2 +- src/traffic_ctl/Makefile.inc | 2 +- src/traffic_layout/Makefile.inc | 2 +- src/traffic_logcat/Makefile.inc | 2 +- src/traffic_logstats/Makefile.inc | 2 +- src/traffic_manager/Makefile.inc | 2 +- src/traffic_server/Makefile.inc | 1 - src/traffic_top/Makefile.inc | 2 +- src/tscore/Makefile.am | 11 +- src/tscore/ink_hash_table.cc | 425 --- tools/package/trafficserver.spec | 4 +- 35 files changed, 26 insertions(+), 3963 deletions(-) delete mode 100644 build/tcl.m4 delete mode 100644 include/tscore/ink_hash_table.h delete mode 100644 src/tscore/ink_hash_table.cc diff --git a/README b/README index 3f88a04bb06..1a5f5457f3b 100644 --- a/README +++ b/README @@ -88,7 +88,6 @@ plugins to build large scale web applications. perl-ExtUtils-MakeMaker gcc/g++ or clang/clang++ openssl-devel - tcl-devel pcre-devel ncurses-devel and libcurl-devel(optional, needed for traffic_top) libcap-devel (optional, highly recommended) @@ -103,7 +102,6 @@ plugins to build large scale web applications. libmodule-install-perl gcc/g++ or clang/clang++ libssl-dev - tcl-dev libpcre3-dev libcap-dev (optional, highly recommended) libhwloc-dev (optional, highly recommended) @@ -119,7 +117,6 @@ plugins to build large scale web applications. autoconf automake libtool - tcl-dev linux-headers OSX (we recommend HomeBrew): @@ -127,7 +124,6 @@ plugins to build large scale web applications. automake pkg-config libtool - tcl-tk (or tcl from Homebrew-Cask) openssl pcre @@ -137,7 +133,6 @@ plugins to build large scale web applications. devel/automake devel/pkgconf devel/libtool - lang/tcl85 security/openssl devel/pcre textproc/flex (optional, install newer version from ports, fix PATH) @@ -149,7 +144,6 @@ plugins to build large scale web applications. developer/build/autoconf developer/build/automake-111 developer/build/libtool - omniti/runtime/tcl-8 library/security/openssl library/pcre diff --git a/build/tcl.m4 b/build/tcl.m4 deleted file mode 100644 index 61b223eb41c..00000000000 --- a/build/tcl.m4 +++ /dev/null @@ -1,3307 +0,0 @@ -#------------------------------------------------------------------------ -# SC_PATH_TCLCONFIG -- -# -# Locate the tclConfig.sh file and perform a sanity check on -# the Tcl compile flags -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-tcl=... -# -# Defines the following vars: -# TCL_BIN_DIR Full path to the directory containing -# the tclConfig.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([SC_PATH_TCLCONFIG], [ - # - # Ok, lets find the tcl configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-tcl - # - - if test x"${no_tcl}" = x ; then - # we reset no_tcl in case something fails here - no_tcl=true - AC_ARG_WITH(tcl, - AC_HELP_STRING([--with-tcl], - [directory containing tcl configuration (tclConfig.sh)]), - with_tclconfig="${withval}") - AC_MSG_CHECKING([for Tcl configuration]) - AC_CACHE_VAL(ac_cv_c_tclconfig,[ - - # First check to see if --with-tcl was specified. - if test x"${with_tclconfig}" != x ; then - case "${with_tclconfig}" in - */tclConfig.sh ) - if test -f "${with_tclconfig}"; then - AC_MSG_WARN([--with-tcl argument should refer to directory containing tclConfig.sh, not to tclConfig.sh itself]) - with_tclconfig="`echo "${with_tclconfig}" | sed 's!/tclConfig\.sh$!!'`" - fi ;; - esac - if test -f "${with_tclconfig}/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd "${with_tclconfig}"; pwd)`" - else - AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh]) - fi - fi - - # then check for a private Tcl installation - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ../tcl \ - `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tcl \ - `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tcl \ - `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - - # on Darwin, check in Framework installation locations - if test "`uname -s`" = "Darwin" -a x"${ac_cv_c_tclconfig}" = x ; then - for i in "`xcrun --show-sdk-path 2>/dev/null`/usr/lib" \ - `xcrun --show-sdk-path 2>/dev/null`/System/Library/Frameworks \ - `ls -d ~/Library/Frameworks 2>/dev/null` \ - `ls -d /Library/Frameworks 2>/dev/null` \ - `ls -d /Network/Library/Frameworks 2>/dev/null` \ - `ls -d /System/Library/Frameworks 2>/dev/null` \ - ; do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i; pwd)`" - break - elif test -f "$i/Tcl.framework/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/Tcl.framework; pwd)`" - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -dr ${exec_prefix}/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${exec_prefix}/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${exec_prefix}/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -dr ${prefix}/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${prefix}/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${prefix}/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -dr /usr/local/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr /usr/local/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr /usr/local/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d /usr/lib64 2>/dev/null` \ - `ls -dr /usr/lib64/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr /usr/lib64/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr /usr/lib64/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -dr /usr/contrib/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr /usr/contrib/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr /usr/contrib/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - `ls -dr /usr/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr /usr/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr /usr/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ; do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ${srcdir}/../tcl \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tclconfig}" = x ; then - TCL_BIN_DIR="# no Tcl configs found" - AC_MSG_ERROR([Can't find Tcl configuration, install the TCL dev package]) - else - no_tcl= - TCL_BIN_DIR="${ac_cv_c_tclconfig}" - AC_MSG_RESULT([found ${TCL_BIN_DIR}/tclConfig.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# SC_PATH_TKCONFIG -- -# -# Locate the tkConfig.sh file -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-tk=... -# -# Defines the following vars: -# TK_BIN_DIR Full path to the directory containing -# the tkConfig.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([SC_PATH_TKCONFIG], [ - # - # Ok, lets find the tk configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-tk - # - - if test x"${no_tk}" = x ; then - # we reset no_tk in case something fails here - no_tk=true - AC_ARG_WITH(tk, - AC_HELP_STRING([--with-tk], - [directory containing tk configuration (tkConfig.sh)]), - with_tkconfig="${withval}") - AC_MSG_CHECKING([for Tk configuration]) - AC_CACHE_VAL(ac_cv_c_tkconfig,[ - - # First check to see if --with-tkconfig was specified. - if test x"${with_tkconfig}" != x ; then - case "${with_tkconfig}" in - */tkConfig.sh ) - if test -f "${with_tkconfig}"; then - AC_MSG_WARN([--with-tk argument should refer to directory containing tkConfig.sh, not to tkConfig.sh itself]) - with_tkconfig="`echo "${with_tkconfig}" | sed 's!/tkConfig\.sh$!!'`" - fi ;; - esac - if test -f "${with_tkconfig}/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd "${with_tkconfig}"; pwd)`" - else - AC_MSG_ERROR([${with_tkconfig} directory doesn't contain tkConfig.sh]) - fi - fi - - # then check for a private Tk library - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in \ - ../tk \ - `ls -dr ../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tk[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tk \ - `ls -dr ../../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tk[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tk \ - `ls -dr ../../../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tk[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test -f "$i/unix/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - - # on Darwin, check in Framework installation locations - if test "`uname -s`" = "Darwin" -a x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d ~/Library/Frameworks 2>/dev/null` \ - `ls -d /Library/Frameworks 2>/dev/null` \ - `ls -d /Network/Library/Frameworks 2>/dev/null` \ - `ls -d /System/Library/Frameworks 2>/dev/null` \ - ; do - if test -f "$i/Tk.framework/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/Tk.framework; pwd)`" - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - ; do - if test -f "$i/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in \ - ${srcdir}/../tk \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test -f "$i/unix/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tkconfig}" = x ; then - TK_BIN_DIR="# no Tk configs found" - AC_MSG_WARN([Can't find Tk configuration definitions]) - exit 0 - else - no_tk= - TK_BIN_DIR="${ac_cv_c_tkconfig}" - AC_MSG_RESULT([found ${TK_BIN_DIR}/tkConfig.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# SC_LOAD_TCLCONFIG -- -# -# Load the tclConfig.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# TCL_BIN_DIR -# -# Results: -# -# Subst the following vars: -# TCL_BIN_DIR -# TCL_SRC_DIR -# TCL_LIB_FILE -# -#------------------------------------------------------------------------ - -AC_DEFUN([SC_LOAD_TCLCONFIG], [ - AC_MSG_CHECKING([for existence of ${TCL_BIN_DIR}/tclConfig.sh]) - - if test -f "${TCL_BIN_DIR}/tclConfig.sh" ; then - AC_MSG_RESULT([loading]) - . "${TCL_BIN_DIR}/tclConfig.sh" - else - AC_MSG_RESULT([could not find ${TCL_BIN_DIR}/tclConfig.sh]) - fi - - # eval is required to do the TCL_DBGX substitution - eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\"" - eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\"" - - # If the TCL_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TCL_LIB_SPEC will be set to the value - # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC - # instead of TCL_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - if test -f "${TCL_BIN_DIR}/Makefile" ; then - TCL_LIB_SPEC="${TCL_BUILD_LIB_SPEC}" - TCL_STUB_LIB_SPEC="${TCL_BUILD_STUB_LIB_SPEC}" - TCL_STUB_LIB_PATH="${TCL_BUILD_STUB_LIB_PATH}" - elif test "`uname -s`" = "Darwin"; then - # If Tcl was built as a framework, attempt to use the libraries - # from the framework at the given location so that linking works - # against Tcl.framework installed in an arbitary location. - case ${TCL_DEFS} in - *TCL_FRAMEWORK*) - if test -f "${TCL_BIN_DIR}/${TCL_LIB_FILE}"; then - for i in "`cd "${TCL_BIN_DIR}"; pwd`" \ - "`cd "${TCL_BIN_DIR}"/../..; pwd`"; do - if test "`basename "$i"`" = "${TCL_LIB_FILE}.framework"; then - TCL_LIB_SPEC="-F`dirname "$i" | sed -e 's/ /\\\\ /g'` -framework ${TCL_LIB_FILE}" - break - fi - done - fi - if test -f "${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}"; then - TCL_STUB_LIB_SPEC="-L`echo "${TCL_BIN_DIR}" | sed -e 's/ /\\\\ /g'` ${TCL_STUB_LIB_FLAG}" - TCL_STUB_LIB_PATH="${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}" - fi - ;; - esac - fi - - # eval is required to do the TCL_DBGX substitution - eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\"" - eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\"" - eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\"" - eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\"" - - AC_SUBST(TCL_VERSION) - AC_SUBST(TCL_PATCH_LEVEL) - AC_SUBST(TCL_BIN_DIR) - AC_SUBST(TCL_SRC_DIR) - - AC_SUBST(TCL_LIB_FILE) - AC_SUBST(TCL_LIB_FLAG) - AC_SUBST(TCL_LIB_SPEC) - - AC_SUBST(TCL_STUB_LIB_FILE) - AC_SUBST(TCL_STUB_LIB_FLAG) - AC_SUBST(TCL_STUB_LIB_SPEC) -]) - -#------------------------------------------------------------------------ -# SC_LOAD_TKCONFIG -- -# -# Load the tkConfig.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# TK_BIN_DIR -# -# Results: -# -# Sets the following vars that should be in tkConfig.sh: -# TK_BIN_DIR -#------------------------------------------------------------------------ - -AC_DEFUN([SC_LOAD_TKCONFIG], [ - AC_MSG_CHECKING([for existence of ${TK_BIN_DIR}/tkConfig.sh]) - - if test -f "${TK_BIN_DIR}/tkConfig.sh" ; then - AC_MSG_RESULT([loading]) - . "${TK_BIN_DIR}/tkConfig.sh" - else - AC_MSG_RESULT([could not find ${TK_BIN_DIR}/tkConfig.sh]) - fi - - # eval is required to do the TK_DBGX substitution - eval "TK_LIB_FILE=\"${TK_LIB_FILE}\"" - eval "TK_STUB_LIB_FILE=\"${TK_STUB_LIB_FILE}\"" - - # If the TK_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TK_LIB_SPEC will be set to the value - # of TK_BUILD_LIB_SPEC. An extension should make use of TK_LIB_SPEC - # instead of TK_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - if test -f "${TK_BIN_DIR}/Makefile" ; then - TK_LIB_SPEC="${TK_BUILD_LIB_SPEC}" - TK_STUB_LIB_SPEC="${TK_BUILD_STUB_LIB_SPEC}" - TK_STUB_LIB_PATH="${TK_BUILD_STUB_LIB_PATH}" - elif test "`uname -s`" = "Darwin"; then - # If Tk was built as a framework, attempt to use the libraries - # from the framework at the given location so that linking works - # against Tk.framework installed in an arbitary location. - case ${TK_DEFS} in - *TK_FRAMEWORK*) - if test -f "${TK_BIN_DIR}/${TK_LIB_FILE}"; then - for i in "`cd "${TK_BIN_DIR}"; pwd`" \ - "`cd "${TK_BIN_DIR}"/../..; pwd`"; do - if test "`basename "$i"`" = "${TK_LIB_FILE}.framework"; then - TK_LIB_SPEC="-F`dirname "$i" | sed -e 's/ /\\\\ /g'` -framework ${TK_LIB_FILE}" - break - fi - done - fi - if test -f "${TK_BIN_DIR}/${TK_STUB_LIB_FILE}"; then - TK_STUB_LIB_SPEC="-L` echo "${TK_BIN_DIR}" | sed -e 's/ /\\\\ /g'` ${TK_STUB_LIB_FLAG}" - TK_STUB_LIB_PATH="${TK_BIN_DIR}/${TK_STUB_LIB_FILE}" - fi - ;; - esac - fi - - # eval is required to do the TK_DBGX substitution - eval "TK_LIB_FLAG=\"${TK_LIB_FLAG}\"" - eval "TK_LIB_SPEC=\"${TK_LIB_SPEC}\"" - eval "TK_STUB_LIB_FLAG=\"${TK_STUB_LIB_FLAG}\"" - eval "TK_STUB_LIB_SPEC=\"${TK_STUB_LIB_SPEC}\"" - - AC_SUBST(TK_VERSION) - AC_SUBST(TK_BIN_DIR) - AC_SUBST(TK_SRC_DIR) - - AC_SUBST(TK_LIB_FILE) - AC_SUBST(TK_LIB_FLAG) - AC_SUBST(TK_LIB_SPEC) - - AC_SUBST(TK_STUB_LIB_FILE) - AC_SUBST(TK_STUB_LIB_FLAG) - AC_SUBST(TK_STUB_LIB_SPEC) -]) - -#------------------------------------------------------------------------ -# SC_PROG_TCLSH -# Locate a tclsh shell installed on the system path. This macro -# will only find a Tcl shell that already exists on the system. -# It will not find a Tcl shell in the Tcl build directory or -# a Tcl shell that has been installed from the Tcl build directory. -# If a Tcl shell can't be located on the PATH, then TCLSH_PROG will -# be set to "". Extensions should take care not to create Makefile -# rules that are run by default and depend on TCLSH_PROG. An -# extension can't assume that an executable Tcl shell exists at -# build time. -# -# Arguments -# none -# -# Results -# Subst's the following values: -# TCLSH_PROG -#------------------------------------------------------------------------ - -AC_DEFUN([SC_PROG_TCLSH], [ - AC_MSG_CHECKING([for tclsh]) - AC_CACHE_VAL(ac_cv_path_tclsh, [ - search_path=`echo ${PATH} | sed -e 's/:/ /g'` - for dir in $search_path ; do - for j in `ls -r $dir/tclsh[[8-9]]* 2> /dev/null` \ - `ls -r $dir/tclsh* 2> /dev/null` ; do - if test x"$ac_cv_path_tclsh" = x ; then - if test -f "$j" ; then - ac_cv_path_tclsh=$j - break - fi - fi - done - done - ]) - - if test -f "$ac_cv_path_tclsh" ; then - TCLSH_PROG="$ac_cv_path_tclsh" - AC_MSG_RESULT([$TCLSH_PROG]) - else - # It is not an error if an installed version of Tcl can't be located. - TCLSH_PROG="" - AC_MSG_RESULT([No tclsh found on PATH]) - fi - AC_SUBST(TCLSH_PROG) -]) - -#------------------------------------------------------------------------ -# SC_BUILD_TCLSH -# Determine the fully qualified path name of the tclsh executable -# in the Tcl build directory. This macro will correctly determine -# the name of the tclsh executable even if tclsh has not yet -# been built in the build directory. The build tclsh must be used -# when running tests from an extension build directory. It is not -# correct to use the TCLSH_PROG in cases like this. -# -# Arguments -# none -# -# Results -# Subst's the following values: -# BUILD_TCLSH -#------------------------------------------------------------------------ - -AC_DEFUN([SC_BUILD_TCLSH], [ - AC_MSG_CHECKING([for tclsh in Tcl build directory]) - BUILD_TCLSH="${TCL_BIN_DIR}"/tclsh - AC_MSG_RESULT([$BUILD_TCLSH]) - AC_SUBST(BUILD_TCLSH) -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_SHARED -- -# -# Allows the building of shared libraries -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-shared=yes|no -# -# Defines the following vars: -# STATIC_BUILD Used for building import/export libraries -# on Windows. -# -# Sets the following vars: -# SHARED_BUILD Value of 1 or 0 -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_SHARED], [ - AC_MSG_CHECKING([how to build libraries]) - AC_ARG_ENABLE(shared, - AC_HELP_STRING([--enable-shared], - [build and link with shared libraries (default: on)]), - [tcl_ok=$enableval], [tcl_ok=yes]) - - if test "${enable_shared+set}" = set; then - enableval="$enable_shared" - tcl_ok=$enableval - else - tcl_ok=yes - fi - - if test "$tcl_ok" = "yes" ; then - AC_MSG_RESULT([shared]) - SHARED_BUILD=1 - else - AC_MSG_RESULT([static]) - SHARED_BUILD=0 - AC_DEFINE(STATIC_BUILD, 1, [Is this a static build?]) - fi -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_FRAMEWORK -- -# -# Allows the building of shared libraries into frameworks -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-framework=yes|no -# -# Sets the following vars: -# FRAMEWORK_BUILD Value of 1 or 0 -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_FRAMEWORK], [ - if test "`uname -s`" = "Darwin" ; then - AC_MSG_CHECKING([how to package libraries]) - AC_ARG_ENABLE(framework, - AC_HELP_STRING([--enable-framework], - [package shared libraries in MacOSX frameworks (default: off)]), - [enable_framework=$enableval], [enable_framework=no]) - if test $enable_framework = yes; then - if test $SHARED_BUILD = 0; then - AC_MSG_WARN([Frameworks can only be built if --enable-shared is yes]) - enable_framework=no - fi - if test $tcl_corefoundation = no; then - AC_MSG_WARN([Frameworks can only be used when CoreFoundation is available]) - enable_framework=no - fi - fi - if test $enable_framework = yes; then - AC_MSG_RESULT([framework]) - FRAMEWORK_BUILD=1 - else - if test $SHARED_BUILD = 1; then - AC_MSG_RESULT([shared library]) - else - AC_MSG_RESULT([static library]) - fi - FRAMEWORK_BUILD=0 - fi - fi -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_THREADS -- -# -# Specify if thread support should be enabled -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-threads -# -# Sets the following vars: -# THREADS_LIBS Thread library(s) -# -# Defines the following vars: -# TCL_THREADS -# _REENTRANT -# _THREAD_SAFE -# -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_THREADS], [ - AC_ARG_ENABLE(threads, - AC_HELP_STRING([--enable-threads], - [build with threads (default: off)]), - [tcl_ok=$enableval], [tcl_ok=no]) - - if test "${TCL_THREADS}" = 1; then - tcl_threaded_core=1; - fi - - if test "$tcl_ok" = "yes" -o "${TCL_THREADS}" = 1; then - TCL_THREADS=1 - # USE_THREAD_ALLOC tells us to try the special thread-based - # allocator that significantly reduces lock contention - AC_DEFINE(USE_THREAD_ALLOC, 1, - [Do we want to use the threaded memory allocator?]) - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - if test "`uname -s`" = "SunOS" ; then - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - fi - AC_DEFINE(_THREAD_SAFE, 1, [Do we want the thread-safe OS API?]) - AC_CHECK_LIB(pthread,pthread_mutex_init,tcl_ok=yes,tcl_ok=no) - if test "$tcl_ok" = "no"; then - # Check a little harder for __pthread_mutex_init in the same - # library, as some systems hide it there until pthread.h is - # defined. We could alternatively do an AC_TRY_COMPILE with - # pthread.h, but that will work with libpthread really doesn't - # exist, like AIX 4.2. [Bug: 4359] - AC_CHECK_LIB(pthread, __pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - fi - - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -lpthread" - else - AC_CHECK_LIB(pthreads, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -lpthreads" - else - AC_CHECK_LIB(c, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "no"; then - AC_CHECK_LIB(c_r, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -pthread" - else - TCL_THREADS=0 - AC_MSG_WARN([Don't know how to find pthread lib on your system - you must disable thread support or edit the LIBS in the Makefile...]) - fi - fi - fi - fi - - # Does the pthread-implementation provide - # 'pthread_attr_setstacksize' ? - - ac_saved_libs=$LIBS - LIBS="$LIBS $THREADS_LIBS" - AC_CHECK_FUNCS(pthread_attr_setstacksize) - AC_CHECK_FUNC(pthread_attr_get_np,tcl_ok=yes,tcl_ok=no) - if test $tcl_ok = yes ; then - AC_DEFINE(HAVE_PTHREAD_ATTR_GET_NP, 1, - [Do we want a BSD-like thread-attribute interface?]) - AC_CACHE_CHECK([for pthread_attr_get_np declaration], - tcl_cv_grep_pthread_attr_get_np, [ - AC_EGREP_HEADER(pthread_attr_get_np, pthread.h, - tcl_cv_grep_pthread_attr_get_np=present, - tcl_cv_grep_pthread_attr_get_np=missing)]) - if test $tcl_cv_grep_pthread_attr_get_np = missing ; then - AC_DEFINE(ATTRGETNP_NOT_DECLARED, 1, - [Is pthread_attr_get_np() declared in ?]) - fi - else - AC_CHECK_FUNC(pthread_getattr_np,tcl_ok=yes,tcl_ok=no) - if test $tcl_ok = yes ; then - AC_DEFINE(HAVE_PTHREAD_GETATTR_NP, 1, - [Do we want a Linux-like thread-attribute interface?]) - AC_CACHE_CHECK([for pthread_getattr_np declaration], - tcl_cv_grep_pthread_getattr_np, [ - AC_EGREP_HEADER(pthread_getattr_np, pthread.h, - tcl_cv_grep_pthread_getattr_np=present, - tcl_cv_grep_pthread_getattr_np=missing)]) - if test $tcl_cv_grep_pthread_getattr_np = missing ; then - AC_DEFINE(GETATTRNP_NOT_DECLARED, 1, - [Is pthread_getattr_np declared in ?]) - fi - fi - fi - if test $tcl_ok = no; then - # Darwin thread stacksize API - AC_CHECK_FUNCS(pthread_get_stacksize_np) - fi - LIBS=$ac_saved_libs - else - TCL_THREADS=0 - fi - # Do checking message here to not mess up interleaved configure output - AC_MSG_CHECKING([for building with threads]) - if test "${TCL_THREADS}" = 1; then - AC_DEFINE(TCL_THREADS, 1, [Are we building with threads enabled?]) - if test "${tcl_threaded_core}" = 1; then - AC_MSG_RESULT([yes (threaded core)]) - else - AC_MSG_RESULT([yes]) - fi - else - AC_MSG_RESULT([no (default)]) - fi - - AC_SUBST(TCL_THREADS) -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_SYMBOLS -- -# -# Specify if debugging symbols should be used. -# Memory (TCL_MEM_DEBUG) and compile (TCL_COMPILE_DEBUG) debugging -# can also be enabled. -# -# Arguments: -# none -# -# Requires the following vars to be set in the Makefile: -# CFLAGS_DEBUG -# CFLAGS_OPTIMIZE -# LDFLAGS_DEBUG -# LDFLAGS_OPTIMIZE -# -# Results: -# -# Adds the following arguments to configure: -# --enable-symbols -# -# Defines the following vars: -# CFLAGS_DEFAULT Sets to $(CFLAGS_DEBUG) if true -# Sets to $(CFLAGS_OPTIMIZE) if false -# LDFLAGS_DEFAULT Sets to $(LDFLAGS_DEBUG) if true -# Sets to $(LDFLAGS_OPTIMIZE) if false -# DBGX Formerly used as debug library extension; -# always blank now. -# -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_SYMBOLS], [ - AC_MSG_CHECKING([for build with symbols]) - AC_ARG_ENABLE(symbols, - AC_HELP_STRING([--enable-symbols], - [build with debugging symbols (default: off)]), - [tcl_ok=$enableval], [tcl_ok=no]) -# FIXME: Currently, LDFLAGS_DEFAULT is not used, it should work like CFLAGS_DEFAULT. - DBGX="" - if test "$tcl_ok" = "no"; then - CFLAGS_DEFAULT='$(CFLAGS_OPTIMIZE)' - LDFLAGS_DEFAULT='$(LDFLAGS_OPTIMIZE)' - AC_MSG_RESULT([no]) - AC_DEFINE(TCL_CFG_OPTIMIZED, 1, [Is this an optimized build?]) - else - CFLAGS_DEFAULT='$(CFLAGS_DEBUG)' - LDFLAGS_DEFAULT='$(LDFLAGS_DEBUG)' - if test "$tcl_ok" = "yes"; then - AC_MSG_RESULT([yes (standard debugging)]) - fi - fi - AC_SUBST(CFLAGS_DEFAULT) - AC_SUBST(LDFLAGS_DEFAULT) - ### FIXME: Surely TCL_CFG_DEBUG should be set to whether we're debugging? - AC_DEFINE(TCL_CFG_DEBUG, 1, [Is debugging enabled?]) - - if test "$tcl_ok" = "mem" -o "$tcl_ok" = "all"; then - AC_DEFINE(TCL_MEM_DEBUG, 1, [Is memory debugging enabled?]) - fi - - ifelse($1,bccdebug,dnl Only enable 'compile' for the Tcl core itself - if test "$tcl_ok" = "compile" -o "$tcl_ok" = "all"; then - AC_DEFINE(TCL_COMPILE_DEBUG, 1, [Is bytecode debugging enabled?]) - AC_DEFINE(TCL_COMPILE_STATS, 1, [Are bytecode statistics enabled?]) - fi) - - if test "$tcl_ok" != "yes" -a "$tcl_ok" != "no"; then - if test "$tcl_ok" = "all"; then - AC_MSG_RESULT([enabled symbols mem ]ifelse($1,bccdebug,[compile ])[debugging]) - else - AC_MSG_RESULT([enabled $tcl_ok debugging]) - fi - fi -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_LANGINFO -- -# -# Allows use of modern nl_langinfo check for better l10n. -# This is only relevant for Unix. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-langinfo=yes|no (default is yes) -# -# Defines the following vars: -# HAVE_LANGINFO Triggers use of nl_langinfo if defined. -# -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_LANGINFO], [ - AC_ARG_ENABLE(langinfo, - AC_HELP_STRING([--enable-langinfo], - [use nl_langinfo if possible to determine encoding at startup, otherwise use old heuristic (default: on)]), - [langinfo_ok=$enableval], [langinfo_ok=yes]) - - HAVE_LANGINFO=0 - if test "$langinfo_ok" = "yes"; then - AC_CHECK_HEADER(langinfo.h,[langinfo_ok=yes],[langinfo_ok=no]) - fi - AC_MSG_CHECKING([whether to use nl_langinfo]) - if test "$langinfo_ok" = "yes"; then - AC_CACHE_VAL(tcl_cv_langinfo_h, [ - AC_TRY_COMPILE([#include ], [nl_langinfo(CODESET);], - [tcl_cv_langinfo_h=yes],[tcl_cv_langinfo_h=no])]) - AC_MSG_RESULT([$tcl_cv_langinfo_h]) - if test $tcl_cv_langinfo_h = yes; then - AC_DEFINE(HAVE_LANGINFO, 1, [Do we have nl_langinfo()?]) - fi - else - AC_MSG_RESULT([$langinfo_ok]) - fi -]) - -#-------------------------------------------------------------------- -# SC_CONFIG_MANPAGES -# -# Decide whether to use symlinks for linking the manpages, -# whether to compress the manpages after installation, and -# whether to add a package name suffix to the installed -# manpages to avoidfile name clashes. -# If compression is enabled also find out what file name suffix -# the given compression program is using. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-man-symlinks -# --enable-man-compression=PROG -# --enable-man-suffix[=STRING] -# -# Defines the following variable: -# -# MAN_FLAGS - The apropriate flags for installManPage -# according to the user's selection. -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_CONFIG_MANPAGES], [ - AC_MSG_CHECKING([whether to use symlinks for manpages]) - AC_ARG_ENABLE(man-symlinks, - AC_HELP_STRING([--enable-man-symlinks], - [use symlinks for the manpages (default: off)]), - test "$enableval" != "no" && MAN_FLAGS="$MAN_FLAGS --symlinks", - enableval="no") - AC_MSG_RESULT([$enableval]) - - AC_MSG_CHECKING([whether to compress the manpages]) - AC_ARG_ENABLE(man-compression, - AC_HELP_STRING([--enable-man-compression=PROG], - [compress the manpages with PROG (default: off)]), - [case $enableval in - yes) AC_MSG_ERROR([missing argument to --enable-man-compression]);; - no) ;; - *) MAN_FLAGS="$MAN_FLAGS --compress $enableval";; - esac], - enableval="no") - AC_MSG_RESULT([$enableval]) - if test "$enableval" != "no"; then - AC_MSG_CHECKING([for compressed file suffix]) - touch TeST - $enableval TeST - Z=`ls TeST* | sed 's/^....//'` - rm -f TeST* - MAN_FLAGS="$MAN_FLAGS --extension $Z" - AC_MSG_RESULT([$Z]) - fi - - AC_MSG_CHECKING([whether to add a package name suffix for the manpages]) - AC_ARG_ENABLE(man-suffix, - AC_HELP_STRING([--enable-man-suffix=STRING], - [use STRING as a suffix to manpage file names (default: no, AC_PACKAGE_NAME if enabled without specifying STRING)]), - [case $enableval in - yes) enableval="AC_PACKAGE_NAME" MAN_FLAGS="$MAN_FLAGS --suffix $enableval";; - no) ;; - *) MAN_FLAGS="$MAN_FLAGS --suffix $enableval";; - esac], - enableval="no") - AC_MSG_RESULT([$enableval]) - - AC_SUBST(MAN_FLAGS) -]) - -#-------------------------------------------------------------------- -# SC_CONFIG_SYSTEM -# -# Determine what the system is (some things cannot be easily checked -# on a feature-driven basis, alas). This can usually be done via the -# "uname" command, but there are a few systems, like Next, where -# this doesn't work. -# -# Arguments: -# none -# -# Results: -# Defines the following var: -# -# system - System/platform/version identification code. -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_CONFIG_SYSTEM], [ - AC_CACHE_CHECK([system version], tcl_cv_sys_version, [ - if test -f /usr/lib/NextStep/software_version; then - tcl_cv_sys_version=NEXTSTEP-`awk '/3/,/3/' /usr/lib/NextStep/software_version` - else - tcl_cv_sys_version=`uname -s`-`uname -r` - if test "$?" -ne 0 ; then - AC_MSG_WARN([can't find uname command]) - tcl_cv_sys_version=unknown - else - # Special check for weird MP-RAS system (uname returns weird - # results, and the version is kept in special file). - - if test -r /etc/.relid -a "X`uname -n`" = "X`uname -s`" ; then - tcl_cv_sys_version=MP-RAS-`awk '{print $[3]}' /etc/.relid` - fi - if test "`uname -s`" = "AIX" ; then - tcl_cv_sys_version=AIX-`uname -v`.`uname -r` - fi - fi - fi - ]) - system=$tcl_cv_sys_version -]) - -#-------------------------------------------------------------------- -# SC_CONFIG_CFLAGS -# -# Try to determine the proper flags to pass to the compiler -# for building shared libraries and other such nonsense. -# -# Arguments: -# none -# -# Results: -# -# Defines and substitutes the following vars: -# -# DL_OBJS - Name of the object file that implements dynamic -# loading for Tcl on this system. -# DL_LIBS - Library file(s) to include in tclsh and other base -# applications in order for the "load" command to work. -# LDFLAGS - Flags to pass to the compiler when linking object -# files into an executable application binary such -# as tclsh. -# LD_SEARCH_FLAGS-Flags to pass to ld, such as "-R /usr/local/tcl/lib", -# that tell the run-time dynamic linker where to look -# for shared libraries such as libtcl.so. Depends on -# the variable LIB_RUNTIME_DIR in the Makefile. Could -# be the same as CC_SEARCH_FLAGS if ${CC} is used to link. -# CC_SEARCH_FLAGS-Flags to pass to ${CC}, such as "-Wl,-rpath,/usr/local/tcl/lib", -# that tell the run-time dynamic linker where to look -# for shared libraries such as libtcl.so. Depends on -# the variable LIB_RUNTIME_DIR in the Makefile. -# MAKE_LIB - Command to execute to build the a library; -# differs when building shared or static. -# MAKE_STUB_LIB - -# Command to execute to build a stub library. -# INSTALL_LIB - Command to execute to install a library; -# differs when building shared or static. -# INSTALL_STUB_LIB - -# Command to execute to install a stub library. -# STLIB_LD - Base command to use for combining object files -# into a static library. -# SHLIB_CFLAGS - Flags to pass to cc when compiling the components -# of a shared library (may request position-independent -# code, among other things). -# SHLIB_LD - Base command to use for combining object files -# into a shared library. -# SHLIB_LD_LIBS - Dependent libraries for the linker to scan when -# creating shared libraries. This symbol typically -# goes at the end of the "ld" commands that build -# shared libraries. The value of the symbol is -# "${LIBS}" if all of the dependent libraries should -# be specified when creating a shared library. If -# dependent libraries should not be specified (as on -# SunOS 4.x, where they cause the link to fail, or in -# general if Tcl and Tk aren't themselves shared -# libraries), then this symbol has an empty string -# as its value. -# SHLIB_SUFFIX - Suffix to use for the names of dynamically loadable -# extensions. An empty string means we don't know how -# to use shared libraries on this platform. -# TCL_SHLIB_LD_EXTRAS - Additional element which are added to SHLIB_LD_LIBS -# TK_SHLIB_LD_EXTRAS for the build of Tcl and Tk, but not recorded in the -# tclConfig.sh, since they are only used for the build -# of Tcl and Tk. -# Examples: MacOS X records the library version and -# compatibility version in the shared library. But -# of course the Tcl version of this is only used for Tcl. -# LIB_SUFFIX - Specifies everything that comes after the "libfoo" -# in a static or shared library name, using the $VERSION variable -# to put the version in the right place. This is used -# by platforms that need non-standard library names. -# Examples: ${VERSION}.so.1.1 on NetBSD, since it needs -# to have a version after the .so, and ${VERSION}.a -# on AIX, since a shared library needs to have -# a .a extension whereas shared objects for loadable -# extensions have a .so extension. Defaults to -# ${VERSION}${SHLIB_SUFFIX}. -# TCL_NEEDS_EXP_FILE - -# 1 means that an export file is needed to link to a -# shared library. -# TCL_EXP_FILE - The name of the installed export / import file which -# should be used to link to the Tcl shared library. -# Empty if Tcl is unshared. -# TCL_BUILD_EXP_FILE - -# The name of the built export / import file which -# should be used to link to the Tcl shared library. -# Empty if Tcl is unshared. -# TCL_LIBS - -# Libs to use when linking Tcl shell or some other -# shell that includes Tcl libs. -# CFLAGS_DEBUG - -# Flags used when running the compiler in debug mode -# CFLAGS_OPTIMIZE - -# Flags used when running the compiler in optimize mode -# CFLAGS - Additional CFLAGS added as necessary (usually 64-bit) -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_CONFIG_CFLAGS], [ - - # Step 0.a: Enable 64 bit support? - - AC_MSG_CHECKING([if 64bit support is requested]) - AC_ARG_ENABLE(64bit, - AC_HELP_STRING([--enable-64bit], - [enable 64bit support (default: off)]), - [do64bit=$enableval], [do64bit=no]) - AC_MSG_RESULT([$do64bit]) - - # Step 0.b: Enable Solaris 64 bit VIS support? - - AC_MSG_CHECKING([if 64bit Sparc VIS support is requested]) - AC_ARG_ENABLE(64bit-vis, - AC_HELP_STRING([--enable-64bit-vis], - [enable 64bit Sparc VIS support (default: off)]), - [do64bitVIS=$enableval], [do64bitVIS=no]) - AC_MSG_RESULT([$do64bitVIS]) - # Force 64bit on with VIS - AS_IF([test "$do64bitVIS" = "yes"], [do64bit=yes]) - - # Step 0.c: Check if visibility support is available. Do this here so - # that platform specific alternatives can be used below if this fails. - - AC_CACHE_CHECK([if compiler supports visibility "hidden"], - tcl_cv_cc_visibility_hidden, [ - hold_cflags=$CFLAGS; CFLAGS="$CFLAGS -Werror" - AC_TRY_LINK([ - extern __attribute__((__visibility__("hidden"))) void f(void); - void f(void) {}], [f();], tcl_cv_cc_visibility_hidden=yes, - tcl_cv_cc_visibility_hidden=no) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_visibility_hidden = yes], [ - AC_DEFINE(MODULE_SCOPE, - [extern __attribute__((__visibility__("hidden")))], - [Compiler support for module scope symbols]) - ]) - - # Step 0.d: Disable -rpath support? - - AC_MSG_CHECKING([if rpath support is requested]) - AC_ARG_ENABLE(rpath, - AC_HELP_STRING([--disable-rpath], - [disable rpath support (default: on)]), - [doRpath=$enableval], [doRpath=yes]) - AC_MSG_RESULT([$doRpath]) - - # Step 1: set the variable "system" to hold the name and version number - # for the system. - - SC_CONFIG_SYSTEM - - # Step 2: check for existence of -ldl library. This is needed because - # Linux can use either -ldl or -ldld for dynamic loading. - - AC_CHECK_LIB(dl, dlopen, have_dl=yes, have_dl=no) - - # Require ranlib early so we can override it in special cases below. - LDAIX_SRC="" - AS_IF([test x"${SHLIB_VERSION}" = x], [SHLIB_VERSION="1.0"]) - - AC_REQUIRE([AC_PROG_RANLIB]) - - # Step 3: set configuration options based on system name and version. - - do64bit_ok=no - LDFLAGS_ORIG="$LDFLAGS" - # When ld needs options to work in 64-bit mode, put them in - # LDFLAGS_ARCH so they eventually end up in LDFLAGS even if [load] - # is disabled by the user. [Bug 1016796] - LDFLAGS_ARCH="" - TCL_EXPORT_FILE_SUFFIX="" - UNSHARED_LIB_SUFFIX="" - TCL_TRIM_DOTS='`echo ${VERSION} | tr -d .`' - ECHO_VERSION='`echo ${VERSION}`' - TCL_LIB_VERSIONS_OK=ok - CFLAGS_DEBUG=-g - CFLAGS_OPTIMIZE=-O - AS_IF([test "$GCC" = yes], [ - CFLAGS_WARNING="-Wall" - ], [CFLAGS_WARNING=""]) - TCL_NEEDS_EXP_FILE=0 - TCL_BUILD_EXP_FILE="" - TCL_EXP_FILE="" -dnl FIXME: Replace AC_CHECK_PROG with AC_CHECK_TOOL once cross compiling is fixed. -dnl AC_CHECK_TOOL(AR, ar) - AC_CHECK_PROG(AR, ar, ar) - AS_IF([test "${AR}" = ""], [ - AC_MSG_ERROR([Required archive tool 'ar' not found on PATH.]) - ]) - STLIB_LD='${AR} cr' - LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH" - PLAT_OBJS="" - PLAT_SRCS="" - case $system in - AIX-*) - AS_IF([test "${TCL_THREADS}" = "1" -a "$GCC" != "yes"], [ - # AIX requires the _r compiler when gcc isn't being used - case "${CC}" in - *_r) - # ok ... - ;; - *) - CC=${CC}_r - ;; - esac - AC_MSG_RESULT([Using $CC for compiling with threads]) - ]) - LIBS="$LIBS -lc" - SHLIB_CFLAGS="" - # Note: need the LIBS below, otherwise Tk won't find Tcl's - # symbols when dynamically loaded into tclsh. - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - - DL_OBJS="tclLoadDl.o" - LD_LIBRARY_PATH_VAR="LIBPATH" - - # Check to enable 64-bit flags for compiler/linker on AIX 4+ - LDAIX_SRC='$(UNIX_DIR)/ldAix' - AS_IF([test "$do64bit" = yes -a "`uname -v`" -gt 3], [ - AS_IF([test "$GCC" = yes], [ - AC_MSG_WARN([64bit mode not supported with GCC on $system]) - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS -q64" - LDFLAGS_ARCH="-q64" - RANLIB="${RANLIB} -X64" - AR="${AR} -X64" - SHLIB_LD_FLAGS="-b64" - ]) - ]) - - AS_IF([test "`uname -m`" = ia64], [ - # AIX-5 uses ELF style dynamic libraries on IA-64, but not PPC - SHLIB_LD="/usr/ccs/bin/ld -G -z text" - # AIX-5 has dl* in libc.so - DL_LIBS="" - AS_IF([test "$GCC" = yes], [ - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - ], [ - CC_SEARCH_FLAGS='-R${LIB_RUNTIME_DIR}' - ]) - LD_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' - ], [ - AS_IF([test "$GCC" = yes], [SHLIB_LD='${CC} -shared'], [ - SHLIB_LD="/bin/ld -bhalt:4 -bM:SRE -bE:lib.exp -H512 -T512 -bnoentry" - ]) - SHLIB_LD="${TCL_SRC_DIR}/unix/ldAix ${SHLIB_LD} ${SHLIB_LD_FLAGS}" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS='-L${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - TCL_NEEDS_EXP_FILE=1 - TCL_EXPORT_FILE_SUFFIX='${VERSION}.exp' - ]) - - # AIX v<=4.1 has some different flags than 4.2+ - AS_IF([test "$system" = "AIX-4.1" -o "`uname -v`" -lt 4], [ - AC_LIBOBJ([tclLoadAix]) - DL_LIBS="-lld" - ]) - - # On AIX <=v4 systems, libbsd.a has to be linked in to support - # non-blocking file IO. This library has to be linked in after - # the MATH_LIBS or it breaks the pow() function. The way to - # insure proper sequencing, is to add it to the tail of MATH_LIBS. - # This library also supplies gettimeofday. - # - # AIX does not have a timezone field in struct tm. When the AIX - # bsd library is used, the timezone global and the gettimeofday - # methods are to be avoided for timezone deduction instead, we - # deduce the timezone by comparing the localtime result on a - # known GMT value. - - AC_CHECK_LIB(bsd, gettimeofday, libbsd=yes, libbsd=no) - AS_IF([test $libbsd = yes], [ - MATH_LIBS="$MATH_LIBS -lbsd" - AC_DEFINE(USE_DELTA_FOR_TZ, 1, [Do we need a special AIX hack for timezones?]) - ]) - ;; - BeOS*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -nostart' - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - - #----------------------------------------------------------- - # Check for inet_ntoa in -lbind, for BeOS (which also needs - # -lsocket, even if the network functions are in -lnet which - # is always linked to, for compatibility. - #----------------------------------------------------------- - AC_CHECK_LIB(bind, inet_ntoa, [LIBS="$LIBS -lbind -lsocket"]) - ;; - BSD/OS-2.1*|BSD/OS-3*) - SHLIB_CFLAGS="" - SHLIB_LD="shlicc -r" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - BSD/OS-4.*) - SHLIB_CFLAGS="-export-dynamic -fPIC" - SHLIB_LD='${CC} -shared' - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - LDFLAGS="$LDFLAGS -export-dynamic" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - dgux*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - HP-UX-*.11.*) - # Use updated header definitions where possible - AC_DEFINE(_XOPEN_SOURCE_EXTENDED, 1, [Do we want to use the XOPEN network library?]) - AC_DEFINE(_XOPEN_SOURCE, 1, [Do we want to use the XOPEN network library?]) - LIBS="$LIBS -lxnet" # Use the XOPEN network library - - AS_IF([test "`uname -m`" = ia64], [ - SHLIB_SUFFIX=".so" - ], [ - SHLIB_SUFFIX=".sl" - ]) - AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) - AS_IF([test "$tcl_ok" = yes], [ - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" - SHLIB_LD_LIBS='${LIBS}' - DL_OBJS="tclLoadShl.o" - DL_LIBS="-ldld" - LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='-Wl,+s,+b,${LIB_RUNTIME_DIR}:.' - LD_SEARCH_FLAGS='+s +b ${LIB_RUNTIME_DIR}:.' - LD_LIBRARY_PATH_VAR="SHLIB_PATH" - ]) - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - SHLIB_LD_LIBS='${LIBS}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ]) - - # Users may want PA-RISC 1.1/2.0 portable code - needs HP cc - #CFLAGS="$CFLAGS +DAportable" - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = "yes"], [ - AS_IF([test "$GCC" = yes], [ - case `${CC} -dumpmachine` in - hppa64*) - # 64-bit gcc in use. Fix flags for GNU ld. - do64bit_ok=yes - SHLIB_LD='${CC} -shared' - SHLIB_LD_LIBS='${LIBS}' - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ;; - *) - AC_MSG_WARN([64bit mode not supported with GCC on $system]) - ;; - esac - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS +DD64" - LDFLAGS_ARCH="+DD64" - ]) - ]) ;; - HP-UX-*.08.*|HP-UX-*.09.*|HP-UX-*.10.*) - SHLIB_SUFFIX=".sl" - AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) - AS_IF([test "$tcl_ok" = yes], [ - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" - SHLIB_LD_LIBS="" - DL_OBJS="tclLoadShl.o" - DL_LIBS="-ldld" - LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='-Wl,+s,+b,${LIB_RUNTIME_DIR}:.' - LD_SEARCH_FLAGS='+s +b ${LIB_RUNTIME_DIR}:.' - LD_LIBRARY_PATH_VAR="SHLIB_PATH" - ]) ;; - IRIX-5.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -shared -rdata_shared" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - ;; - IRIX-6.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -n32 -shared -rdata_shared" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - AS_IF([test "$GCC" = yes], [ - CFLAGS="$CFLAGS -mabi=n32" - LDFLAGS="$LDFLAGS -mabi=n32" - ], [ - case $system in - IRIX-6.3) - # Use to build 6.2 compatible binaries on 6.3. - CFLAGS="$CFLAGS -n32 -D_OLD_TERMIOS" - ;; - *) - CFLAGS="$CFLAGS -n32" - ;; - esac - LDFLAGS="$LDFLAGS -n32" - ]) - ;; - IRIX64-6.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -n32 -shared -rdata_shared" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - - # Check to enable 64-bit flags for compiler/linker - - AS_IF([test "$do64bit" = yes], [ - AS_IF([test "$GCC" = yes], [ - AC_MSG_WARN([64bit mode not supported by gcc]) - ], [ - do64bit_ok=yes - SHLIB_LD="ld -64 -shared -rdata_shared" - CFLAGS="$CFLAGS -64" - LDFLAGS_ARCH="-64" - ]) - ]) - ;; - Linux*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - - CFLAGS_OPTIMIZE="-O2" - # egcs-2.91.66 on Redhat Linux 6.0 generates lots of warnings - # when you inline the string and math operations. Turn this off to - # get rid of the warnings. - #CFLAGS_OPTIMIZE="${CFLAGS_OPTIMIZE} -D__NO_STRING_INLINES -D__NO_MATH_INLINES" - - SHLIB_LD='${CC} -shared ${CFLAGS} ${LDFLAGS}' - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - LDFLAGS="$LDFLAGS -Wl,--export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "`uname -m`" = "alpha"], [CFLAGS="$CFLAGS -mieee"]) - AS_IF([test $do64bit = yes], [ - AC_CACHE_CHECK([if compiler accepts -m64 flag], tcl_cv_cc_m64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -m64" - AC_TRY_LINK(,, tcl_cv_cc_m64=yes, tcl_cv_cc_m64=no) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_m64 = yes], [ - CFLAGS="$CFLAGS -m64" - do64bit_ok=yes - ]) - ]) - - # The combo of gcc + glibc has a bug related to inlining of - # functions like strtod(). The -fno-builtin flag should address - # this problem but it does not work. The -fno-inline flag is kind - # of overkill but it works. Disable inlining only when one of the - # files in compat/*.c is being linked in. - - AS_IF([test x"${USE_COMPAT}" != x],[CFLAGS="$CFLAGS -fno-inline"]) - ;; - GNU*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - - SHLIB_LD='${CC} -shared' - DL_OBJS="" - DL_LIBS="-ldl" - LDFLAGS="$LDFLAGS -Wl,--export-dynamic" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - AS_IF([test "`uname -m`" = "alpha"], [CFLAGS="$CFLAGS -mieee"]) - ;; - Lynx*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - CFLAGS_OPTIMIZE=-02 - SHLIB_LD='${CC} -shared' - DL_OBJS="tclLoadDl.o" - DL_LIBS="-mshared -ldl" - LD_FLAGS="-Wl,--export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - ;; - MP-RAS-02*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - MP-RAS-*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - LDFLAGS="$LDFLAGS -Wl,-Bexport" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - NetBSD-1.*|FreeBSD-[[1-2]].*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD="ld -Bshareable -x" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - AC_CACHE_CHECK([for ELF], tcl_cv_ld_elf, [ - AC_EGREP_CPP(yes, [ -#ifdef __ELF__ - yes -#endif - ], tcl_cv_ld_elf=yes, tcl_cv_ld_elf=no)]) - AS_IF([test $tcl_cv_ld_elf = yes], [ - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so' - ], [ - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so.${SHLIB_VERSION}' - ]) - - # Ancient FreeBSD doesn't handle version numbers with dots. - - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - TCL_LIB_VERSIONS_OK=nodots - ;; - OpenBSD-*) - CFLAGS_OPTIMIZE='-O2' - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -shared ${SHLIB_CFLAGS}' - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so.${SHLIB_VERSION}' - AC_CACHE_CHECK([for ELF], tcl_cv_ld_elf, [ - AC_EGREP_CPP(yes, [ -#ifdef __ELF__ - yes -#endif - ], tcl_cv_ld_elf=yes, tcl_cv_ld_elf=no)]) - AS_IF([test $tcl_cv_ld_elf = yes], [ - LDFLAGS=-Wl,-export-dynamic - ], [LDFLAGS=""]) - - # OpenBSD doesn't do version numbers with dots. - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - TCL_LIB_VERSIONS_OK=nodots - ;; - NetBSD-*|FreeBSD-*) - # FreeBSD 3.* and greater have ELF. - # NetBSD 2.* has ELF and can use 'cc -shared' to build shared libs - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -shared ${SHLIB_CFLAGS}' - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - LDFLAGS="$LDFLAGS -export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "${TCL_THREADS}" = "1"], [ - # The -pthread needs to go in the CFLAGS, not LIBS - LIBS=`echo $LIBS | sed s/-pthread//` - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ]) - case $system in - FreeBSD-3.*) - # FreeBSD-3 doesn't handle version numbers with dots. - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so' - TCL_LIB_VERSIONS_OK=nodots - ;; - esac - ;; - Darwin-*) - CFLAGS_OPTIMIZE="-Os" - SHLIB_CFLAGS="-fno-common" - # To avoid discrepancies between what headers configure sees during - # preprocessing tests and compiling tests, move any -isysroot and - # -mmacosx-version-min flags from CFLAGS to CPPFLAGS: - CPPFLAGS="${CPPFLAGS} `echo " ${CFLAGS}" | \ - awk 'BEGIN {FS=" +-";ORS=" "}; {for (i=2;i<=NF;i++) \ - if ([$]i~/^(isysroot|mmacosx-version-min)/) print "-"[$]i}'`" - CFLAGS="`echo " ${CFLAGS}" | \ - awk 'BEGIN {FS=" +-";ORS=" "}; {for (i=2;i<=NF;i++) \ - if (!([$]i~/^(isysroot|mmacosx-version-min)/)) print "-"[$]i}'`" - AS_IF([test $do64bit = yes], [ - case `arch` in - ppc) - AC_CACHE_CHECK([if compiler accepts -arch ppc64 flag], - tcl_cv_cc_arch_ppc64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - AC_TRY_LINK(,, tcl_cv_cc_arch_ppc64=yes, - tcl_cv_cc_arch_ppc64=no) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_arch_ppc64 = yes], [ - CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - do64bit_ok=yes - ]);; - i386) - AC_CACHE_CHECK([if compiler accepts -arch x86_64 flag], - tcl_cv_cc_arch_x86_64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -arch x86_64" - AC_TRY_LINK(,, tcl_cv_cc_arch_x86_64=yes, - tcl_cv_cc_arch_x86_64=no) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_arch_x86_64 = yes], [ - CFLAGS="$CFLAGS -arch x86_64" - do64bit_ok=yes - ]);; - *) - AC_MSG_WARN([Don't know how enable 64-bit on architecture `arch`]);; - esac - ], [ - # Check for combined 32-bit and 64-bit fat build - AS_IF([echo "$CFLAGS " |grep -E -q -- '-arch (ppc64|x86_64) ' \ - && echo "$CFLAGS " |grep -E -q -- '-arch (ppc|i386) '], [ - fat_32_64=yes]) - ]) - SHLIB_LD='${CC} -dynamiclib ${CFLAGS} ${LDFLAGS}' - AC_CACHE_CHECK([if ld accepts -single_module flag], tcl_cv_ld_single_module, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -dynamiclib -Wl,-single_module" - AC_TRY_LINK(, [int i;], tcl_cv_ld_single_module=yes, tcl_cv_ld_single_module=no) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_single_module = yes], [ - SHLIB_LD="${SHLIB_LD} -Wl,-single_module" - ]) - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".dylib" - DL_OBJS="tclLoadDyld.o" - DL_LIBS="" - # Don't use -prebind when building for Mac OS X 10.4 or later only: - AS_IF([test "`echo "${MACOSX_DEPLOYMENT_TARGET}" | awk -F '10\\.' '{print int([$]2)}'`" -lt 4 -a \ - "`echo "${CPPFLAGS}" | awk -F '-mmacosx-version-min=10\\.' '{print int([$]2)}'`" -lt 4], [ - LDFLAGS="$LDFLAGS -prebind"]) - LDFLAGS="$LDFLAGS -headerpad_max_install_names" - AC_CACHE_CHECK([if ld accepts -search_paths_first flag], - tcl_cv_ld_search_paths_first, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - AC_TRY_LINK(, [int i;], tcl_cv_ld_search_paths_first=yes, - tcl_cv_ld_search_paths_first=no) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_search_paths_first = yes], [ - LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - ]) - AS_IF([test "$tcl_cv_cc_visibility_hidden" != yes], [ - AC_DEFINE(MODULE_SCOPE, [__private_extern__], - [Compiler support for module scope symbols]) - ]) - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - LD_LIBRARY_PATH_VAR="DYLD_LIBRARY_PATH" - AC_DEFINE(MAC_OSX_TCL, 1, [Is this a Mac I see before me?]) - PLAT_OBJS='${MAC_OSX_OBJS}' - PLAT_SRCS='${MAC_OSX_SRCS}' - AC_MSG_CHECKING([whether to use CoreFoundation]) - AC_ARG_ENABLE(corefoundation, - AC_HELP_STRING([--enable-corefoundation], - [use CoreFoundation API on MacOSX (default: on)]), - [tcl_corefoundation=$enableval], [tcl_corefoundation=yes]) - AC_MSG_RESULT([$tcl_corefoundation]) - AS_IF([test $tcl_corefoundation = yes], [ - AC_CACHE_CHECK([for CoreFoundation.framework], - tcl_cv_lib_corefoundation, [ - hold_libs=$LIBS - AS_IF([test "$fat_32_64" = yes], [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - # On Tiger there is no 64-bit CF, so remove 64-bit - # archs from CFLAGS et al. while testing for - # presence of CF. 64-bit CF is disabled in - # tclUnixPort.h if necessary. - eval 'hold_'$v'="$'$v'";'$v'="`echo "$'$v' "|sed -e "s/-arch ppc64 / /g" -e "s/-arch x86_64 / /g"`"' - done]) - LIBS="$LIBS -framework CoreFoundation" - AC_TRY_LINK([#include ], - [CFBundleRef b = CFBundleGetMainBundle();], - tcl_cv_lib_corefoundation=yes, - tcl_cv_lib_corefoundation=no) - AS_IF([test "$fat_32_64" = yes], [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="$hold_'$v'"' - done]) - LIBS=$hold_libs]) - AS_IF([test $tcl_cv_lib_corefoundation = yes], [ - LIBS="$LIBS -framework CoreFoundation" - AC_DEFINE(HAVE_COREFOUNDATION, 1, - [Do we have access to Darwin CoreFoundation.framework?]) - ], [tcl_corefoundation=no]) - AS_IF([test "$fat_32_64" = yes -a $tcl_corefoundation = yes],[ - AC_CACHE_CHECK([for 64-bit CoreFoundation], - tcl_cv_lib_corefoundation_64, [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval 'hold_'$v'="$'$v'";'$v'="`echo "$'$v' "|sed -e "s/-arch ppc / /g" -e "s/-arch i386 / /g"`"' - done - AC_TRY_LINK([#include ], - [CFBundleRef b = CFBundleGetMainBundle();], - tcl_cv_lib_corefoundation_64=yes, - tcl_cv_lib_corefoundation_64=no) - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="$hold_'$v'"' - done]) - AS_IF([test $tcl_cv_lib_corefoundation_64 = no], [ - AC_DEFINE(NO_COREFOUNDATION_64, 1, - [Is Darwin CoreFoundation unavailable for 64-bit?]) - LDFLAGS="$LDFLAGS -Wl,-no_arch_warnings" - ]) - ]) - ]) - ;; - NEXTSTEP-*) - SHLIB_CFLAGS="" - SHLIB_LD='${CC} -nostdlib -r' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadNext.o" - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - OS/390-*) - CFLAGS_OPTIMIZE="" # Optimizer is buggy - AC_DEFINE(_OE_SOCKETS, 1, # needed in sys/socket.h - [Should OS/390 do the right thing with sockets?]) - ;; - OSF1-1.0|OSF1-1.1|OSF1-1.2) - # OSF/1 1.[012] from OSF, and derivatives, including Paragon OSF/1 - SHLIB_CFLAGS="" - # Hack: make package name same as library name - SHLIB_LD='ld -R -export $@:' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadOSF.o" - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - OSF1-1.*) - # OSF/1 1.3 from OSF using ELF, and derivatives, including AD2 - SHLIB_CFLAGS="-fPIC" - AS_IF([test "$SHARED_BUILD" = 1], [SHLIB_LD="ld -shared"], [ - SHLIB_LD="ld -non_shared" - ]) - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - OSF1-V*) - # Digital OSF/1 - SHLIB_CFLAGS="" - AS_IF([test "$SHARED_BUILD" = 1], [ - SHLIB_LD='ld -shared -expect_unresolved "*"' - ], [ - SHLIB_LD='ld -non_shared -expect_unresolved "*"' - ]) - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - AS_IF([test "$GCC" = yes], [CFLAGS="$CFLAGS -mieee"], [ - CFLAGS="$CFLAGS -DHAVE_TZSET -std1 -ieee"]) - # see pthread_intro(3) for pthread support on osf1, k.furukawa - AS_IF([test "${TCL_THREADS}" = 1], [ - CFLAGS="$CFLAGS -DHAVE_PTHREAD_ATTR_SETSTACKSIZE" - CFLAGS="$CFLAGS -DTCL_THREAD_STACK_MIN=PTHREAD_STACK_MIN*64" - LIBS=`echo $LIBS | sed s/-lpthreads//` - AS_IF([test "$GCC" = yes], [ - LIBS="$LIBS -lpthread -lmach -lexc" - ], [ - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ]) - ]) - ;; - QNX-6*) - # QNX RTP - # This may work for all QNX, but it was only reported for v6. - SHLIB_CFLAGS="-fPIC" - SHLIB_LD="ld -Bshareable -x" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - # dlopen is in -lc on QNX - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SCO_SV-3.2*) - # Note, dlopen is available only on SCO 3.2.5 and greater. However, - # this test works, since "uname -s" was non-standard in 3.2.4 and - # below. - AS_IF([test "$GCC" = yes], [ - SHLIB_CFLAGS="-fPIC -melf" - LDFLAGS="$LDFLAGS -melf -Wl,-Bexport" - ], [ - SHLIB_CFLAGS="-Kpic -belf" - LDFLAGS="$LDFLAGS -belf -Wl,-Bexport" - ]) - SHLIB_LD="ld -G" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SINIX*5.4*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SunOS-4*) - SHLIB_CFLAGS="-PIC" - SHLIB_LD="ld" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS='-L${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - - # SunOS can't handle version numbers with dots in them in library - # specs, like -ltcl7.5, so use -ltcl75 instead. Also, it - # requires an extra version number at the end of .so file names. - # So, the library has to have a name like libtcl75.so.1.0 - - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so.${SHLIB_VERSION}' - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - TCL_LIB_VERSIONS_OK=nodots - ;; - SunOS-5.[[0-6]]) - # Careful to not let 5.10+ fall into this case - - # Note: If _REENTRANT isn't defined, then Solaris - # won't define thread-safe library routines. - - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - - SHLIB_CFLAGS="-KPIC" - - # Note: need the LIBS below, otherwise Tk won't find Tcl's - # symbols when dynamically loaded into tclsh. - - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ], [ - SHLIB_LD="/usr/ccs/bin/ld -G -z text" - CC_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ]) - ;; - SunOS-5*) - # Note: If _REENTRANT isn't defined, then Solaris - # won't define thread-safe library routines. - - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - - SHLIB_CFLAGS="-KPIC" - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = yes], [ - arch=`isainfo` - AS_IF([test "$arch" = "sparcv9 sparc"], [ - AS_IF([test "$GCC" = yes], [ - AS_IF([test "`${CC} -dumpversion | awk -F. '{print [$]1}'`" -lt 3], [ - AC_MSG_WARN([64bit mode not supported with GCC < 3.2 on $system]) - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS -m64 -mcpu=v9" - LDFLAGS="$LDFLAGS -m64 -mcpu=v9" - SHLIB_CFLAGS="-fPIC" - ]) - ], [ - do64bit_ok=yes - AS_IF([test "$do64bitVIS" = yes], [ - CFLAGS="$CFLAGS -xarch=v9a" - LDFLAGS_ARCH="-xarch=v9a" - ], [ - CFLAGS="$CFLAGS -xarch=v9" - LDFLAGS_ARCH="-xarch=v9" - ]) - # Solaris 64 uses this as well - #LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH_64" - ]) - ], [AS_IF([test "$arch" = "amd64 i386"], [ - AS_IF([test "$GCC" = yes], [ - case $system in - SunOS-5.1[[1-9]]*|SunOS-5.[[2-9]][[0-9]]*) - do64bit_ok=yes - CFLAGS="$CFLAGS -m64" - LDFLAGS="$LDFLAGS -m64";; - *) - AC_MSG_WARN([64bit mode not supported with GCC on $system]);; - esac - ], [ - do64bit_ok=yes - case $system in - SunOS-5.1[[1-9]]*|SunOS-5.[[2-9]][[0-9]]*) - CFLAGS="$CFLAGS -m64" - LDFLAGS="$LDFLAGS -m64";; - *) - CFLAGS="$CFLAGS -xarch=amd64" - LDFLAGS="$LDFLAGS -xarch=amd64";; - esac - ]) - ], [AC_MSG_WARN([64bit mode not supported for $arch])])]) - ]) - - #-------------------------------------------------------------------- - # On Solaris 5.x i386 with the sunpro compiler we need to link - # with sunmath to get floating point rounding control - #-------------------------------------------------------------------- - AS_IF([test "$GCC" = yes],[use_sunmath=no],[ - arch=`isainfo` - AC_MSG_CHECKING([whether to use -lsunmath for fp rounding control]) - AS_IF([test "$arch" = "amd64 i386"], [ - AC_MSG_RESULT([yes]) - MATH_LIBS="-lsunmath $MATH_LIBS" - AC_CHECK_HEADER(sunmath.h) - use_sunmath=yes - ], [ - AC_MSG_RESULT([no]) - use_sunmath=no - ]) - ]) - - # Note: need the LIBS below, otherwise Tk won't find Tcl's - # symbols when dynamically loaded into tclsh. - - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "$do64bit_ok" = yes], [ - AS_IF([test "$arch" = "sparcv9 sparc"], [ - # We need to specify -static-libgcc or we need to - # add the path to the sparv9 libgcc. - SHLIB_LD="$SHLIB_LD -m64 -mcpu=v9 -static-libgcc" - # for finding sparcv9 libgcc, get the regular libgcc - # path, remove so name and append 'sparcv9' - #v9gcclibdir="`gcc -print-file-name=libgcc_s.so` | ..." - #CC_SEARCH_FLAGS="${CC_SEARCH_FLAGS},-R,$v9gcclibdir" - ], [AS_IF([test "$arch" = "amd64 i386"], [ - SHLIB_LD="$SHLIB_LD -m64 -static-libgcc" - ])]) - ]) - ], [ - AS_IF([test "$use_sunmath" = yes], [textmode=textoff],[textmode=text]) - case $system in - SunOS-5.[[1-9]][[0-9]]*) - SHLIB_LD="\${CC} -G -z $textmode \${LDFLAGS}";; - *) - SHLIB_LD="/usr/ccs/bin/ld -G -z $textmode";; - esac - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' - ]) - ;; - UNIX_SV* | UnixWare-5*) - SHLIB_CFLAGS="-KPIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - # Some UNIX_SV* systems (unixware 1.1.2 for example) have linkers - # that don't grok the -Bexport option. Test that it does. - AC_CACHE_CHECK([for ld accepts -Bexport flag], tcl_cv_ld_Bexport, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-Bexport" - AC_TRY_LINK(, [int i;], tcl_cv_ld_Bexport=yes, tcl_cv_ld_Bexport=no) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_Bexport = yes], [ - LDFLAGS="$LDFLAGS -Wl,-Bexport" - ]) - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - esac - - AS_IF([test "$do64bit" = yes -a "$do64bit_ok" = no], [ - AC_MSG_WARN([64bit support being disabled -- don't know magic for this platform]) - ]) - - AS_IF([test "$do64bit" = yes -a "$do64bit_ok" = yes], [ - AC_DEFINE(TCL_CFG_DO64BIT, 1, [Is this a 64-bit build?]) - ]) - -dnl # Add any CPPFLAGS set in the environment to our CFLAGS, but delay doing so -dnl # until the end of configure, as configure's compile and link tests use -dnl # both CPPFLAGS and CFLAGS (unlike our compile and link) but configure's -dnl # preprocessing tests use only CPPFLAGS. - AC_CONFIG_COMMANDS_PRE([CFLAGS="${CFLAGS} ${CPPFLAGS}"; CPPFLAGS=""]) - - # Step 4: disable dynamic loading if requested via a command-line switch. - - AC_ARG_ENABLE(load, - AC_HELP_STRING([--enable-load], - [allow dynamic loading and "load" command (default: on)]), - [tcl_ok=$enableval], [tcl_ok=yes]) - AS_IF([test "$tcl_ok" = no], [DL_OBJS=""]) - - AS_IF([test "x$DL_OBJS" != x], [BUILD_DLTEST="\$(DLTEST_TARGETS)"], [ - AC_MSG_WARN([Can't figure out how to do dynamic loading or shared libraries on this system.]) - SHLIB_CFLAGS="" - SHLIB_LD="" - SHLIB_SUFFIX="" - DL_OBJS="tclLoadNone.o" - DL_LIBS="" - LDFLAGS="$LDFLAGS_ORIG" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - BUILD_DLTEST="" - ]) - LDFLAGS="$LDFLAGS $LDFLAGS_ARCH" - - # If we're running gcc, then change the C flags for compiling shared - # libraries to the right flags for gcc, instead of those for the - # standard manufacturer compiler. - - AS_IF([test "$DL_OBJS" != "tclLoadNone.o" -a "$GCC" = yes], [ - case $system in - AIX-*) ;; - BSD/OS*) ;; - IRIX*) ;; - NetBSD-*|FreeBSD-*) ;; - Darwin-*) ;; - SCO_SV-3.2*) ;; - *) SHLIB_CFLAGS="-fPIC" ;; - esac]) - - AS_IF([test "$SHARED_LIB_SUFFIX" = ""], [ - SHARED_LIB_SUFFIX='${VERSION}${SHLIB_SUFFIX}']) - AS_IF([test "$UNSHARED_LIB_SUFFIX" = ""], [ - UNSHARED_LIB_SUFFIX='${VERSION}.a']) - - AS_IF([test "${SHARED_BUILD}" = 1 -a "${SHLIB_SUFFIX}" != ""], [ - LIB_SUFFIX=${SHARED_LIB_SUFFIX} - MAKE_LIB='${SHLIB_LD} -o [$]@ ${OBJS} ${SHLIB_LD_LIBS} ${TCL_SHLIB_LD_EXTRAS} ${TK_SHLIB_LD_EXTRAS} ${LD_SEARCH_FLAGS}' - INSTALL_LIB='$(INSTALL_LIBRARY) $(LIB_FILE) "$(LIB_INSTALL_DIR)"/$(LIB_FILE)' - ], [ - LIB_SUFFIX=${UNSHARED_LIB_SUFFIX} - - AS_IF([test "$RANLIB" = ""], [ - MAKE_LIB='$(STLIB_LD) [$]@ ${OBJS}' - INSTALL_LIB='$(INSTALL_LIBRARY) $(LIB_FILE) "$(LIB_INSTALL_DIR)"/$(LIB_FILE)' - ], [ - MAKE_LIB='${STLIB_LD} [$]@ ${OBJS} ; ${RANLIB} [$]@' - INSTALL_LIB='$(INSTALL_LIBRARY) $(LIB_FILE) "$(LIB_INSTALL_DIR)"/$(LIB_FILE) ; (cd "$(LIB_INSTALL_DIR)" ; $(RANLIB) $(LIB_FILE))' - ]) - ]) - - # Stub lib does not depend on shared/static configuration - AS_IF([test "$RANLIB" = ""], [ - MAKE_STUB_LIB='${STLIB_LD} [$]@ ${STUB_LIB_OBJS}' - INSTALL_STUB_LIB='$(INSTALL_LIBRARY) $(STUB_LIB_FILE) "$(LIB_INSTALL_DIR)/$(STUB_LIB_FILE)"' - ], [ - MAKE_STUB_LIB='${STLIB_LD} [$]@ ${STUB_LIB_OBJS} ; ${RANLIB} [$]@' - INSTALL_STUB_LIB='$(INSTALL_LIBRARY) $(STUB_LIB_FILE) "$(LIB_INSTALL_DIR)"/$(STUB_LIB_FILE) ; (cd "$(LIB_INSTALL_DIR)" ; $(RANLIB) $(STUB_LIB_FILE))' - ]) - - # Define TCL_LIBS now that we know what DL_LIBS is. - # The trick here is that we don't want to change the value of TCL_LIBS if - # it is already set when tclConfig.sh had been loaded by Tk. - AS_IF([test "x${TCL_LIBS}" = x], [ - TCL_LIBS="${DL_LIBS} ${LIBS} ${MATH_LIBS}"]) - AC_SUBST(TCL_LIBS) - - # FIXME: This subst was left in only because the TCL_DL_LIBS - # entry in tclConfig.sh uses it. It is not clear why someone - # would use TCL_DL_LIBS instead of TCL_LIBS. - AC_SUBST(DL_LIBS) - - AC_SUBST(DL_OBJS) - AC_SUBST(PLAT_OBJS) - AC_SUBST(PLAT_SRCS) - AC_SUBST(LDAIX_SRC) - AC_SUBST(CFLAGS) - AC_SUBST(CFLAGS_DEBUG) - AC_SUBST(CFLAGS_OPTIMIZE) - AC_SUBST(CFLAGS_WARNING) - - AC_SUBST(LDFLAGS) - AC_SUBST(LDFLAGS_DEBUG) - AC_SUBST(LDFLAGS_OPTIMIZE) - AC_SUBST(CC_SEARCH_FLAGS) - AC_SUBST(LD_SEARCH_FLAGS) - - AC_SUBST(STLIB_LD) - AC_SUBST(SHLIB_LD) - AC_SUBST(TCL_SHLIB_LD_EXTRAS) - AC_SUBST(TK_SHLIB_LD_EXTRAS) - AC_SUBST(SHLIB_LD_LIBS) - AC_SUBST(SHLIB_CFLAGS) - AC_SUBST(SHLIB_SUFFIX) - AC_DEFINE_UNQUOTED(TCL_SHLIB_EXT,"${SHLIB_SUFFIX}", - [What is the default extension for shared libraries?]) - - AC_SUBST(MAKE_LIB) - AC_SUBST(MAKE_STUB_LIB) - AC_SUBST(INSTALL_LIB) - AC_SUBST(INSTALL_STUB_LIB) - AC_SUBST(RANLIB) -]) - -#-------------------------------------------------------------------- -# SC_SERIAL_PORT -# -# Determine which interface to use to talk to the serial port. -# Note that #include lines must begin in leftmost column for -# some compilers to recognize them as preprocessor directives, -# and some build environments have stdin not pointing at a -# pseudo-terminal (usually /dev/null instead.) -# -# Arguments: -# none -# -# Results: -# -# Defines only one of the following vars: -# HAVE_SYS_MODEM_H -# USE_TERMIOS -# USE_TERMIO -# USE_SGTTY -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_SERIAL_PORT], [ - AC_CHECK_HEADERS(sys/modem.h) - AC_CACHE_CHECK([termios vs. termio vs. sgtty], tcl_cv_api_serial, [ - AC_TRY_RUN([ -#include - -int main() { - struct termios t; - if (tcgetattr(0, &t) == 0) { - cfsetospeed(&t, 0); - t.c_cflag |= PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}], tcl_cv_api_serial=termios, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ -#include - -int main() { - struct termio t; - if (ioctl(0, TCGETA, &t) == 0) { - t.c_cflag |= CBAUD | PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}], tcl_cv_api_serial=termio, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - fi - if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ -#include - -int main() { - struct sgttyb t; - if (ioctl(0, TIOCGETP, &t) == 0) { - t.sg_ospeed = 0; - t.sg_flags |= ODDP | EVENP | RAW; - return 0; - } - return 1; -}], tcl_cv_api_serial=sgtty, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - fi - if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ -#include -#include - -int main() { - struct termios t; - if (tcgetattr(0, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - cfsetospeed(&t, 0); - t.c_cflag |= PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}], tcl_cv_api_serial=termios, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - fi - if test $tcl_cv_api_serial = no; then - AC_TRY_RUN([ -#include -#include - -int main() { - struct termio t; - if (ioctl(0, TCGETA, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - t.c_cflag |= CBAUD | PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; - }], tcl_cv_api_serial=termio, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - fi - if test $tcl_cv_api_serial = no; then - AC_TRY_RUN([ -#include -#include - -int main() { - struct sgttyb t; - if (ioctl(0, TIOCGETP, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - t.sg_ospeed = 0; - t.sg_flags |= ODDP | EVENP | RAW; - return 0; - } - return 1; -}], tcl_cv_api_serial=sgtty, tcl_cv_api_serial=none, tcl_cv_api_serial=none) - fi]) - case $tcl_cv_api_serial in - termios) AC_DEFINE(USE_TERMIOS, 1, [Use the termios API for serial lines]);; - termio) AC_DEFINE(USE_TERMIO, 1, [Use the termio API for serial lines]);; - sgtty) AC_DEFINE(USE_SGTTY, 1, [Use the sgtty API for serial lines]);; - esac -]) - -#-------------------------------------------------------------------- -# SC_MISSING_POSIX_HEADERS -# -# Supply substitutes for missing POSIX header files. Special -# notes: -# - stdlib.h doesn't define strtol, strtoul, or -# strtod insome versions of SunOS -# - some versions of string.h don't declare procedures such -# as strstr -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# NO_DIRENT_H -# NO_VALUES_H -# HAVE_LIMITS_H or NO_LIMITS_H -# NO_STDLIB_H -# NO_STRING_H -# NO_SYS_WAIT_H -# NO_DLFCN_H -# HAVE_SYS_PARAM_H -# -# HAVE_STRING_H ? -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_MISSING_POSIX_HEADERS], [ - AC_CACHE_CHECK([dirent.h], tcl_cv_dirent_h, [ - AC_TRY_LINK([#include -#include ], [ -#ifndef _POSIX_SOURCE -# ifdef __Lynx__ - /* - * Generate compilation error to make the test fail: Lynx headers - * are only valid if really in the POSIX environment. - */ - - missing_procedure(); -# endif -#endif -DIR *d; -struct dirent *entryPtr; -char *p; -d = opendir("foobar"); -entryPtr = readdir(d); -p = entryPtr->d_name; -closedir(d); -], tcl_cv_dirent_h=yes, tcl_cv_dirent_h=no)]) - - if test $tcl_cv_dirent_h = no; then - AC_DEFINE(NO_DIRENT_H, 1, [Do we have ?]) - fi - - AC_CHECK_HEADER(float.h, , [AC_DEFINE(NO_FLOAT_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(values.h, , [AC_DEFINE(NO_VALUES_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(limits.h, - [AC_DEFINE(HAVE_LIMITS_H, 1, [Do we have ?])], - [AC_DEFINE(NO_LIMITS_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(stdlib.h, tcl_ok=1, tcl_ok=0) - AC_EGREP_HEADER(strtol, stdlib.h, , tcl_ok=0) - AC_EGREP_HEADER(strtoul, stdlib.h, , tcl_ok=0) - AC_EGREP_HEADER(strtod, stdlib.h, , tcl_ok=0) - if test $tcl_ok = 0; then - AC_DEFINE(NO_STDLIB_H, 1, [Do we have ?]) - fi - AC_CHECK_HEADER(string.h, tcl_ok=1, tcl_ok=0) - AC_EGREP_HEADER(strstr, string.h, , tcl_ok=0) - AC_EGREP_HEADER(strerror, string.h, , tcl_ok=0) - - # See also memmove check below for a place where NO_STRING_H can be - # set and why. - - if test $tcl_ok = 0; then - AC_DEFINE(NO_STRING_H, 1, [Do we have ?]) - fi - - AC_CHECK_HEADER(sys/wait.h, , [AC_DEFINE(NO_SYS_WAIT_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(dlfcn.h, , [AC_DEFINE(NO_DLFCN_H, 1, [Do we have ?])]) - - # OS/390 lacks sys/param.h (and doesn't need it, by chance). - AC_HAVE_HEADERS(sys/param.h) -]) - -#-------------------------------------------------------------------- -# SC_PATH_X -# -# Locate the X11 header files and the X11 library archive. Try -# the ac_path_x macro first, but if it doesn't find the X stuff -# (e.g. because there's no xmkmf program) then check through -# a list of possible directories. Under some conditions the -# autoconf macro will return an include directory that contains -# no include files, so double-check its result just to be safe. -# -# Arguments: -# none -# -# Results: -# -# Sets the the following vars: -# XINCLUDES -# XLIBSW -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_PATH_X], [ - AC_PATH_X - not_really_there="" - if test "$no_x" = ""; then - if test "$x_includes" = ""; then - AC_TRY_CPP([#include ], , not_really_there="yes") - else - if test ! -r $x_includes/X11/Intrinsic.h; then - not_really_there="yes" - fi - fi - fi - if test "$no_x" = "yes" -o "$not_really_there" = "yes"; then - AC_MSG_CHECKING([for X11 header files]) - found_xincludes="no" - AC_TRY_CPP([#include ], found_xincludes="yes", found_xincludes="no") - if test "$found_xincludes" = "no"; then - dirs="/usr/unsupported/include /usr/local/include /usr/X386/include /usr/X11R6/include /usr/X11R5/include /usr/include/X11R5 /usr/include/X11R4 /usr/openwin/include /usr/X11/include /usr/sww/include" - for i in $dirs ; do - if test -r $i/X11/Intrinsic.h; then - AC_MSG_RESULT([$i]) - XINCLUDES=" -I$i" - found_xincludes="yes" - break - fi - done - fi - else - if test "$x_includes" != ""; then - XINCLUDES="-I$x_includes" - found_xincludes="yes" - fi - fi - if test found_xincludes = "no"; then - AC_MSG_RESULT([couldn't find any!]) - fi - - if test "$no_x" = yes; then - AC_MSG_CHECKING([for X11 libraries]) - XLIBSW=nope - dirs="/usr/unsupported/lib /usr/local/lib /usr/X386/lib /usr/X11R6/lib /usr/X11R5/lib /usr/lib/X11R5 /usr/lib/X11R4 /usr/openwin/lib /usr/X11/lib /usr/sww/X11/lib" - for i in $dirs ; do - if test -r $i/libX11.a -o -r $i/libX11.so -o -r $i/libX11.sl -o -r $i/libX11.dylib; then - AC_MSG_RESULT([$i]) - XLIBSW="-L$i -lX11" - x_libraries="$i" - break - fi - done - else - if test "$x_libraries" = ""; then - XLIBSW=-lX11 - else - XLIBSW="-L$x_libraries -lX11" - fi - fi - if test "$XLIBSW" = nope ; then - AC_CHECK_LIB(Xwindow, XCreateWindow, XLIBSW=-lXwindow) - fi - if test "$XLIBSW" = nope ; then - AC_MSG_RESULT([could not find any! Using -lX11.]) - XLIBSW=-lX11 - fi -]) - -#-------------------------------------------------------------------- -# SC_BLOCKING_STYLE -# -# The statements below check for systems where POSIX-style -# non-blocking I/O (O_NONBLOCK) doesn't work or is unimplemented. -# On these systems (mostly older ones), use the old BSD-style -# FIONBIO approach instead. -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# HAVE_SYS_IOCTL_H -# HAVE_SYS_FILIO_H -# USE_FIONBIO -# O_NONBLOCK -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_BLOCKING_STYLE], [ - AC_CHECK_HEADERS(sys/ioctl.h) - AC_CHECK_HEADERS(sys/filio.h) - SC_CONFIG_SYSTEM - AC_MSG_CHECKING([FIONBIO vs. O_NONBLOCK for nonblocking I/O]) - case $system in - # There used to be code here to use FIONBIO under AIX. However, it - # was reported that FIONBIO doesn't work under AIX 3.2.5. Since - # using O_NONBLOCK seems fine under AIX 4.*, I removed the FIONBIO - # code (JO, 5/31/97). - - OSF*) - AC_DEFINE(USE_FIONBIO, 1, [Should we use FIONBIO?]) - AC_MSG_RESULT([FIONBIO]) - ;; - SunOS-4*) - AC_DEFINE(USE_FIONBIO, 1, [Should we use FIONBIO?]) - AC_MSG_RESULT([FIONBIO]) - ;; - *) - AC_MSG_RESULT([O_NONBLOCK]) - ;; - esac -]) - -#-------------------------------------------------------------------- -# SC_TIME_HANLDER -# -# Checks how the system deals with time.h, what time structures -# are used on the system, and what fields the structures have. -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# USE_DELTA_FOR_TZ -# HAVE_TM_GMTOFF -# HAVE_TM_TZADJ -# HAVE_TIMEZONE_VAR -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TIME_HANDLER], [ - AC_CHECK_HEADERS(sys/time.h) - AC_HEADER_TIME - AC_STRUCT_TIMEZONE - - AC_CHECK_FUNCS(gmtime_r localtime_r mktime) - - AC_CACHE_CHECK([tm_tzadj in struct tm], tcl_cv_member_tm_tzadj, [ - AC_TRY_COMPILE([#include ], [struct tm tm; tm.tm_tzadj;], - tcl_cv_member_tm_tzadj=yes, tcl_cv_member_tm_tzadj=no)]) - if test $tcl_cv_member_tm_tzadj = yes ; then - AC_DEFINE(HAVE_TM_TZADJ, 1, [Should we use the tm_tzadj field of struct tm?]) - fi - - AC_CACHE_CHECK([tm_gmtoff in struct tm], tcl_cv_member_tm_gmtoff, [ - AC_TRY_COMPILE([#include ], [struct tm tm; tm.tm_gmtoff;], - tcl_cv_member_tm_gmtoff=yes, tcl_cv_member_tm_gmtoff=no)]) - if test $tcl_cv_member_tm_gmtoff = yes ; then - AC_DEFINE(HAVE_TM_GMTOFF, 1, [Should we use the tm_gmtoff field of struct tm?]) - fi - - # - # Its important to include time.h in this check, as some systems - # (like convex) have timezone functions, etc. - # - AC_CACHE_CHECK([long timezone variable], tcl_cv_timezone_long, [ - AC_TRY_COMPILE([#include ], - [extern long timezone; - timezone += 1; - exit (0);], - tcl_cv_timezone_long=yes, tcl_cv_timezone_long=no)]) - if test $tcl_cv_timezone_long = yes ; then - AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) - else - # - # On some systems (eg IRIX 6.2), timezone is a time_t and not a long. - # - AC_CACHE_CHECK([time_t timezone variable], tcl_cv_timezone_time, [ - AC_TRY_COMPILE([#include ], - [extern time_t timezone; - timezone += 1; - exit (0);], - tcl_cv_timezone_time=yes, tcl_cv_timezone_time=no)]) - if test $tcl_cv_timezone_time = yes ; then - AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) - fi - fi -]) - -#-------------------------------------------------------------------- -# SC_BUGGY_STRTOD -# -# Under Solaris 2.4, strtod returns the wrong value for the -# terminating character under some conditions. Check for this -# and if the problem exists use a substitute procedure -# "fixstrtod" (provided by Tcl) that corrects the error. -# Also, on Compaq's Tru64 Unix 5.0, -# strtod(" ") returns 0.0 instead of a failure to convert. -# -# Arguments: -# none -# -# Results: -# -# Might defines some of the following vars: -# strtod (=fixstrtod) -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_BUGGY_STRTOD], [ - AC_CHECK_FUNC(strtod, tcl_strtod=1, tcl_strtod=0) - if test "$tcl_strtod" = 1; then - AC_CACHE_CHECK([for Solaris2.4/Tru64 strtod bugs], tcl_cv_strtod_buggy,[ - AC_TRY_RUN([ - extern double strtod(); - int main() { - char *infString="Inf", *nanString="NaN", *spaceString=" "; - char *term; - double value; - value = strtod(infString, &term); - if ((term != infString) && (term[-1] == 0)) { - exit(1); - } - value = strtod(nanString, &term); - if ((term != nanString) && (term[-1] == 0)) { - exit(1); - } - value = strtod(spaceString, &term); - if (term == (spaceString+1)) { - exit(1); - } - exit(0); - }], tcl_cv_strtod_buggy=ok, tcl_cv_strtod_buggy=buggy, - tcl_cv_strtod_buggy=buggy)]) - if test "$tcl_cv_strtod_buggy" = buggy; then - AC_LIBOBJ([fixstrtod]) - USE_COMPAT=1 - AC_DEFINE(strtod, fixstrtod, [Do we want to use the strtod() in compat?]) - fi - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_LINK_LIBS -# -# Search for the libraries needed to link the Tcl shell. -# Things like the math library (-lm) and socket stuff (-lsocket vs. -# -lnsl) are dealt with here. -# -# Arguments: -# None. -# -# Results: -# -# Might append to the following vars: -# LIBS -# MATH_LIBS -# -# Might define the following vars: -# HAVE_NET_ERRNO_H -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_LINK_LIBS], [ - #-------------------------------------------------------------------- - # On a few very rare systems, all of the libm.a stuff is - # already in libc.a. Set compiler flags accordingly. - # Also, Linux requires the "ieee" library for math to work - # right (and it must appear before "-lm"). - #-------------------------------------------------------------------- - - AC_CHECK_FUNC(sin, MATH_LIBS="", MATH_LIBS="-lm") - AC_CHECK_LIB(ieee, main, [MATH_LIBS="-lieee $MATH_LIBS"]) - - #-------------------------------------------------------------------- - # Interactive UNIX requires -linet instead of -lsocket, plus it - # needs net/errno.h to define the socket-related error codes. - #-------------------------------------------------------------------- - - AC_CHECK_LIB(inet, main, [LIBS="$LIBS -linet"]) - AC_CHECK_HEADER(net/errno.h, [ - AC_DEFINE(HAVE_NET_ERRNO_H, 1, [Do we have ?])]) - - #-------------------------------------------------------------------- - # Check for the existence of the -lsocket and -lnsl libraries. - # The order here is important, so that they end up in the right - # order in the command line generated by make. Here are some - # special considerations: - # 1. Use "connect" and "accept" to check for -lsocket, and - # "gethostbyname" to check for -lnsl. - # 2. Use each function name only once: can't redo a check because - # autoconf caches the results of the last check and won't redo it. - # 3. Use -lnsl and -lsocket only if they supply procedures that - # aren't already present in the normal libraries. This is because - # IRIX 5.2 has libraries, but they aren't needed and they're - # bogus: they goof up name resolution if used. - # 4. On some SVR4 systems, can't use -lsocket without -lnsl too. - # To get around this problem, check for both libraries together - # if -lsocket doesn't work by itself. - #-------------------------------------------------------------------- - - tcl_checkBoth=0 - AC_CHECK_FUNC(connect, tcl_checkSocket=0, tcl_checkSocket=1) - if test "$tcl_checkSocket" = 1; then - AC_CHECK_FUNC(setsockopt, , [AC_CHECK_LIB(socket, setsockopt, - LIBS="$LIBS -lsocket", tcl_checkBoth=1)]) - fi - if test "$tcl_checkBoth" = 1; then - tk_oldLibs=$LIBS - LIBS="$LIBS -lsocket -lnsl" - AC_CHECK_FUNC(accept, tcl_checkNsl=0, [LIBS=$tk_oldLibs]) - fi - AC_CHECK_FUNC(gethostbyname, , [AC_CHECK_LIB(nsl, gethostbyname, - [LIBS="$LIBS -lnsl"])]) -]) - -#-------------------------------------------------------------------- -# SC_TCL_EARLY_FLAGS -# -# Check for what flags are needed to be passed so the correct OS -# features are available. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# _ISOC99_SOURCE -# _LARGEFILE64_SOURCE -# _LARGEFILE_SOURCE64 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_EARLY_FLAG],[ - AC_CACHE_VAL([tcl_cv_flag_]translit($1,[A-Z],[a-z]), - AC_TRY_COMPILE([$2], $3, [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no, - AC_TRY_COMPILE([[#define ]$1[ 1 -]$2], $3, - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=yes, - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no))) - if test ["x${tcl_cv_flag_]translit($1,[A-Z],[a-z])[}" = "xyes"] ; then - AC_DEFINE($1, 1, [Add the ]$1[ flag when building]) - tcl_flags="$tcl_flags $1" - fi -]) - -AC_DEFUN([SC_TCL_EARLY_FLAGS],[ - AC_MSG_CHECKING([for required early compiler flags]) - tcl_flags="" - SC_TCL_EARLY_FLAG(_ISOC99_SOURCE,[#include ], - [char *p = (char *)strtoll; char *q = (char *)strtoull;]) - SC_TCL_EARLY_FLAG(_LARGEFILE64_SOURCE,[#include ], - [struct stat64 buf; int i = stat64("/", &buf);]) - SC_TCL_EARLY_FLAG(_LARGEFILE_SOURCE64,[#include ], - [char *p = (char *)open64;]) - if test "x${tcl_flags}" = "x" ; then - AC_MSG_RESULT([none]) - else - AC_MSG_RESULT([${tcl_flags}]) - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_64BIT_FLAGS -# -# Check for what is defined in the way of 64-bit features. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# TCL_WIDE_INT_IS_LONG -# TCL_WIDE_INT_TYPE -# HAVE_STRUCT_DIRENT64 -# HAVE_STRUCT_STAT64 -# HAVE_TYPE_OFF64_T -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_64BIT_FLAGS], [ - AC_MSG_CHECKING([for 64-bit integer type]) - AC_CACHE_VAL(tcl_cv_type_64bit,[ - tcl_cv_type_64bit=none - # See if the compiler knows natively about __int64 - AC_TRY_COMPILE(,[__int64 value = (__int64) 0;], - tcl_type_64bit=__int64, tcl_type_64bit="long long") - # See if we should use long anyway Note that we substitute in the - # type that is our current guess for a 64-bit type inside this check - # program, so it should be modified only carefully... - AC_TRY_COMPILE(,[switch (0) { - case 1: case (sizeof(]${tcl_type_64bit}[)==sizeof(long)): ; - }],tcl_cv_type_64bit=${tcl_type_64bit})]) - if test "${tcl_cv_type_64bit}" = none ; then - AC_DEFINE(TCL_WIDE_INT_IS_LONG, 1, [Are wide integers to be implemented with C 'long's?]) - AC_MSG_RESULT([using long]) - else - AC_DEFINE_UNQUOTED(TCL_WIDE_INT_TYPE,${tcl_cv_type_64bit}, - [What type should be used to define wide integers?]) - AC_MSG_RESULT([${tcl_cv_type_64bit}]) - - # Now check for auxiliary declarations - AC_CACHE_CHECK([for struct dirent64], tcl_cv_struct_dirent64,[ - AC_TRY_COMPILE([#include -#include ],[struct dirent64 p;], - tcl_cv_struct_dirent64=yes,tcl_cv_struct_dirent64=no)]) - if test "x${tcl_cv_struct_dirent64}" = "xyes" ; then - AC_DEFINE(HAVE_STRUCT_DIRENT64, 1, [Is 'struct dirent64' in ?]) - fi - - AC_CACHE_CHECK([for struct stat64], tcl_cv_struct_stat64,[ - AC_TRY_COMPILE([#include ],[struct stat64 p; -], - tcl_cv_struct_stat64=yes,tcl_cv_struct_stat64=no)]) - if test "x${tcl_cv_struct_stat64}" = "xyes" ; then - AC_DEFINE(HAVE_STRUCT_STAT64, 1, [Is 'struct stat64' in ?]) - fi - - AC_CHECK_FUNCS(open64 lseek64) - AC_MSG_CHECKING([for off64_t]) - AC_CACHE_VAL(tcl_cv_type_off64_t,[ - AC_TRY_COMPILE([#include ],[off64_t offset; -], - tcl_cv_type_off64_t=yes,tcl_cv_type_off64_t=no)]) - dnl Define HAVE_TYPE_OFF64_T only when the off64_t type and the - dnl functions lseek64 and open64 are defined. - if test "x${tcl_cv_type_off64_t}" = "xyes" && \ - test "x${ac_cv_func_lseek64}" = "xyes" && \ - test "x${ac_cv_func_open64}" = "xyes" ; then - AC_DEFINE(HAVE_TYPE_OFF64_T, 1, [Is off64_t in ?]) - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - fi - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_CFG_ENCODING TIP #59 -# -# Declare the encoding to use for embedded configuration information. -# -# Arguments: -# None. -# -# Results: -# Might append to the following vars: -# DEFS (implicit) -# -# Will define the following vars: -# TCL_CFGVAL_ENCODING -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_CFG_ENCODING], [ - AC_ARG_WITH(encoding, - AC_HELP_STRING([--with-encoding], - [encoding for configuration values (default: iso8859-1)]), - with_tcencoding=${withval}) - - if test x"${with_tcencoding}" != x ; then - AC_DEFINE_UNQUOTED(TCL_CFGVAL_ENCODING,"${with_tcencoding}", - [What encoding should be used for embedded configuration info?]) - else - AC_DEFINE(TCL_CFGVAL_ENCODING,"iso8859-1", - [What encoding should be used for embedded configuration info?]) - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_CHECK_BROKEN_FUNC -# -# Check for broken function. -# -# Arguments: -# funcName - function to test for -# advancedTest - the advanced test to run if the function is present -# -# Results: -# Might cause compatability versions of the function to be used. -# Might affect the following vars: -# USE_COMPAT (implicit) -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_CHECK_BROKEN_FUNC],[ - AC_CHECK_FUNC($1, tcl_ok=1, tcl_ok=0) - if test ["$tcl_ok"] = 1; then - AC_CACHE_CHECK([proper ]$1[ implementation], [tcl_cv_]$1[_unbroken], - AC_TRY_RUN([[int main() {]$2[}]],[tcl_cv_]$1[_unbroken]=ok, - [tcl_cv_]$1[_unbroken]=broken,[tcl_cv_]$1[_unbroken]=unknown)) - if test ["$tcl_cv_]$1[_unbroken"] = "ok"; then - tcl_ok=1 - else - tcl_ok=0 - fi - fi - if test ["$tcl_ok"] = 0; then - AC_LIBOBJ($1) - USE_COMPAT=1 - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_GETHOSTBYADDR_R -# -# Check if we have MT-safe variant of gethostbyaddr(). -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETHOSTBYADDR_R -# HAVE_GETHOSTBYADDR_R_7 -# HAVE_GETHOSTBYADDR_R_8 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETHOSTBYADDR_R], [AC_CHECK_FUNC(gethostbyaddr_r, [ - AC_CACHE_CHECK([for gethostbyaddr_r with 7 args], tcl_cv_api_gethostbyaddr_r_7, [ - AC_TRY_COMPILE([ - #include - ], [ - char *addr; - int length; - int type; - struct hostent *result; - char buffer[2048]; - int buflen = 2048; - int h_errnop; - - (void) gethostbyaddr_r(addr, length, type, result, buffer, buflen, - &h_errnop); - ], tcl_cv_api_gethostbyaddr_r_7=yes, tcl_cv_api_gethostbyaddr_r_7=no)]) - tcl_ok=$tcl_cv_api_gethostbyaddr_r_7 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYADDR_R_7, 1, - [Define to 1 if gethostbyaddr_r takes 7 args.]) - else - AC_CACHE_CHECK([for gethostbyaddr_r with 8 args], tcl_cv_api_gethostbyaddr_r_8, [ - AC_TRY_COMPILE([ - #include - ], [ - char *addr; - int length; - int type; - struct hostent *result, *resultp; - char buffer[2048]; - int buflen = 2048; - int h_errnop; - - (void) gethostbyaddr_r(addr, length, type, result, buffer, buflen, - &resultp, &h_errnop); - ], tcl_cv_api_gethostbyaddr_r_8=yes, tcl_cv_api_gethostbyaddr_r_8=no)]) - tcl_ok=$tcl_cv_api_gethostbyaddr_r_8 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYADDR_R_8, 1, - [Define to 1 if gethostbyaddr_r takes 8 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYADDR_R, 1, - [Define to 1 if gethostbyaddr_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETHOSTBYNAME_R -# -# Check to see what variant of gethostbyname_r() we have. -# Based on David Arnold's example from the comp.programming.threads -# FAQ Q213 -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETHOSTBYADDR_R -# HAVE_GETHOSTBYADDR_R_3 -# HAVE_GETHOSTBYADDR_R_5 -# HAVE_GETHOSTBYADDR_R_6 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETHOSTBYNAME_R], [AC_CHECK_FUNC(gethostbyname_r, [ - AC_CACHE_CHECK([for gethostbyname_r with 6 args], tcl_cv_api_gethostbyname_r_6, [ - AC_TRY_COMPILE([ - #include - ], [ - char *name; - struct hostent *he, *res; - char buffer[2048]; - int buflen = 2048; - int h_errnop; - - (void) gethostbyname_r(name, he, buffer, buflen, &res, &h_errnop); - ], tcl_cv_api_gethostbyname_r_6=yes, tcl_cv_api_gethostbyname_r_6=no)]) - tcl_ok=$tcl_cv_api_gethostbyname_r_6 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYNAME_R_6, 1, - [Define to 1 if gethostbyname_r takes 6 args.]) - else - AC_CACHE_CHECK([for gethostbyname_r with 5 args], tcl_cv_api_gethostbyname_r_5, [ - AC_TRY_COMPILE([ - #include - ], [ - char *name; - struct hostent *he; - char buffer[2048]; - int buflen = 2048; - int h_errnop; - - (void) gethostbyname_r(name, he, buffer, buflen, &h_errnop); - ], tcl_cv_api_gethostbyname_r_5=yes, tcl_cv_api_gethostbyname_r_5=no)]) - tcl_ok=$tcl_cv_api_gethostbyname_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYNAME_R_5, 1, - [Define to 1 if gethostbyname_r takes 5 args.]) - else - AC_CACHE_CHECK([for gethostbyname_r with 3 args], tcl_cv_api_gethostbyname_r_3, [ - AC_TRY_COMPILE([ - #include - ], [ - char *name; - struct hostent *he; - struct hostent_data data; - - (void) gethostbyname_r(name, he, &data); - ], tcl_cv_api_gethostbyname_r_3=yes, tcl_cv_api_gethostbyname_r_3=no)]) - tcl_ok=$tcl_cv_api_gethostbyname_r_3 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYNAME_R_3, 1, - [Define to 1 if gethostbyname_r takes 3 args.]) - fi - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYNAME_R, 1, - [Define to 1 if gethostbyname_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETADDRINFO -# -# Check if we have 'getaddrinfo' -# -# Arguments: -# None -# -# Results: -# Might define the following vars: -# HAVE_GETADDRINFO -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETADDRINFO], [AC_CHECK_FUNC(getaddrinfo, [ - AC_CACHE_CHECK([for working getaddrinfo], tcl_cv_api_getaddrinfo, [ - AC_TRY_COMPILE([ - #include - ], [ - const char *name, *port; - struct addrinfo *aiPtr, hints; - (void)getaddrinfo(name,port, &hints, &aiPtr); - (void)freeaddrinfo(aiPtr); - ], tcl_cv_api_getaddrinfo=yes, tcl_cv_getaddrinfo=no)]) - tcl_ok=$tcl_cv_api_getaddrinfo - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETADDRINFO, 1, - [Define to 1 if getaddrinfo is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETPWUID_R -# -# Check if we have MT-safe variant of getpwuid() and if yes, -# which one exactly. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETPWUID_R -# HAVE_GETPWUID_R_4 -# HAVE_GETPWUID_R_5 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETPWUID_R], [AC_CHECK_FUNC(getpwuid_r, [ - AC_CACHE_CHECK([for getpwuid_r with 5 args], tcl_cv_api_getpwuid_r_5, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - uid_t uid; - struct passwd pw, *pwp; - char buf[512]; - int buflen = 512; - - (void) getpwuid_r(uid, &pw, buf, buflen, &pwp); - ], tcl_cv_api_getpwuid_r_5=yes, tcl_cv_api_getpwuid_r_5=no)]) - tcl_ok=$tcl_cv_api_getpwuid_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWUID_R_5, 1, - [Define to 1 if getpwuid_r takes 5 args.]) - else - AC_CACHE_CHECK([for getpwuid_r with 4 args], tcl_cv_api_getpwuid_r_4, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - uid_t uid; - struct passwd pw; - char buf[512]; - int buflen = 512; - - (void)getpwnam_r(uid, &pw, buf, buflen); - ], tcl_cv_api_getpwuid_r_4=yes, tcl_cv_api_getpwuid_r_4=no)]) - tcl_ok=$tcl_cv_api_getpwuid_r_4 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWUID_R_4, 1, - [Define to 1 if getpwuid_r takes 4 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWUID_R, 1, - [Define to 1 if getpwuid_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETPWNAM_R -# -# Check if we have MT-safe variant of getpwnam() and if yes, -# which one exactly. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETPWNAM_R -# HAVE_GETPWNAM_R_4 -# HAVE_GETPWNAM_R_5 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETPWNAM_R], [AC_CHECK_FUNC(getpwnam_r, [ - AC_CACHE_CHECK([for getpwnam_r with 5 args], tcl_cv_api_getpwnam_r_5, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - char *name; - struct passwd pw, *pwp; - char buf[512]; - int buflen = 512; - - (void) getpwnam_r(name, &pw, buf, buflen, &pwp); - ], tcl_cv_api_getpwnam_r_5=yes, tcl_cv_api_getpwnam_r_5=no)]) - tcl_ok=$tcl_cv_api_getpwnam_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWNAM_R_5, 1, - [Define to 1 if getpwnam_r takes 5 args.]) - else - AC_CACHE_CHECK([for getpwnam_r with 4 args], tcl_cv_api_getpwnam_r_4, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - char *name; - struct passwd pw; - char buf[512]; - int buflen = 512; - - (void)getpwnam_r(name, &pw, buf, buflen); - ], tcl_cv_api_getpwnam_r_4=yes, tcl_cv_api_getpwnam_r_4=no)]) - tcl_ok=$tcl_cv_api_getpwnam_r_4 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWNAM_R_4, 1, - [Define to 1 if getpwnam_r takes 4 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWNAM_R, 1, - [Define to 1 if getpwnam_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETGRGID_R -# -# Check if we have MT-safe variant of getgrgid() and if yes, -# which one exactly. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETGRGID_R -# HAVE_GETGRGID_R_4 -# HAVE_GETGRGID_R_5 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETGRGID_R], [AC_CHECK_FUNC(getgrgid_r, [ - AC_CACHE_CHECK([for getgrgid_r with 5 args], tcl_cv_api_getgrgid_r_5, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - gid_t gid; - struct group gr, *grp; - char buf[512]; - int buflen = 512; - - (void) getgrgid_r(gid, &gr, buf, buflen, &grp); - ], tcl_cv_api_getgrgid_r_5=yes, tcl_cv_api_getgrgid_r_5=no)]) - tcl_ok=$tcl_cv_api_getgrgid_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRGID_R_5, 1, - [Define to 1 if getgrgid_r takes 5 args.]) - else - AC_CACHE_CHECK([for getgrgid_r with 4 args], tcl_cv_api_getgrgid_r_4, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - gid_t gid; - struct group gr; - char buf[512]; - int buflen = 512; - - (void)getgrgid_r(gid, &gr, buf, buflen); - ], tcl_cv_api_getgrgid_r_4=yes, tcl_cv_api_getgrgid_r_4=no)]) - tcl_ok=$tcl_cv_api_getgrgid_r_4 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRGID_R_4, 1, - [Define to 1 if getgrgid_r takes 4 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRGID_R, 1, - [Define to 1 if getgrgid_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETGRNAM_R -# -# Check if we have MT-safe variant of getgrnam() and if yes, -# which one exactly. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETGRNAM_R -# HAVE_GETGRNAM_R_4 -# HAVE_GETGRNAM_R_5 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETGRNAM_R], [AC_CHECK_FUNC(getgrnam_r, [ - AC_CACHE_CHECK([for getgrnam_r with 5 args], tcl_cv_api_getgrnam_r_5, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - char *name; - struct group gr, *grp; - char buf[512]; - int buflen = 512; - - (void) getgrnam_r(name, &gr, buf, buflen, &grp); - ], tcl_cv_api_getgrnam_r_5=yes, tcl_cv_api_getgrnam_r_5=no)]) - tcl_ok=$tcl_cv_api_getgrnam_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRNAM_R_5, 1, - [Define to 1 if getgrnam_r takes 5 args.]) - else - AC_CACHE_CHECK([for getgrnam_r with 4 args], tcl_cv_api_getgrnam_r_4, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - char *name; - struct group gr; - char buf[512]; - int buflen = 512; - - (void)getgrnam_r(name, &gr, buf, buflen); - ], tcl_cv_api_getgrnam_r_4=yes, tcl_cv_api_getgrnam_r_4=no)]) - tcl_ok=$tcl_cv_api_getgrnam_r_4 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRNAM_R_4, 1, - [Define to 1 if getgrnam_r takes 4 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRNAM_R, 1, - [Define to 1 if getgrnam_r is available.]) - fi -])]) - -# Local Variables: -# mode: autoconf -# End: diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 885c70ff398..82248c9e264 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -50,7 +50,7 @@ RUN yum -y update; \ # Various other tools git rpm-build distcc-server file; \ # Devel packages that ATS needs - yum -y install openssl-devel tcl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ + yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel kyotocabinet-devel luajit-devel \ brotli-devel ImageMagick-devel ImageMagick-c++-devel perl-ExtUtils-MakeMaker perl-Digest-SHA; \ # This is for autest stuff diff --git a/ci/jenkins/jobs.yaml b/ci/jenkins/jobs.yaml index 3fd1023fc42..24c802628a8 100644 --- a/ci/jenkins/jobs.yaml +++ b/ci/jenkins/jobs.yaml @@ -119,7 +119,7 @@ export CXXFLAGS=-m64 export CPPFLAGS=-I/opt/omni/include export LDFLAGS="-L/opt/omni/lib/amd64 -R/opt/omni/lib/amd64" - "${WORKSPACE}"/src/configure --prefix="${WORKSPACE}/install/${JOB_NAME}.${BUILD_NUMBER}" --enable-werror --with-tcl=/opt/omni/lib/amd64 --enable-experimental-plugins --enable-example-plugins --enable-test-tools $ENABLE_DEBUG $ENABLE_EXPERIMENTAL + "${WORKSPACE}"/src/configure --prefix="${WORKSPACE}/install/${JOB_NAME}.${BUILD_NUMBER}" --enable-werror --enable-experimental-plugins --enable-example-plugins --enable-test-tools $ENABLE_DEBUG $ENABLE_EXPERIMENTAL gmake -j2 V=1 gmake check gmake install diff --git a/ci/regression b/ci/regression index f8adfcffed9..1f83f6eeae3 100755 --- a/ci/regression +++ b/ci/regression @@ -68,7 +68,6 @@ prog() { extras() { case $(uname -v) in - omnios*) echo --with-tcl=/opt/omni/lib/amd64 ;; Darwin*) echo --with-openssl=$(brew --prefix openssl) ;; esac } diff --git a/configure.ac b/configure.ac index 5a8245d3b8f..9600c4f04b1 100644 --- a/configure.ac +++ b/configure.ac @@ -1252,26 +1252,6 @@ TS_CHECK_ZLIB # Check for lzma presence and usability TS_CHECK_LZMA -# -# Tcl macros provided by build/tcl.m4 -# -# this will error out if tclConfig.sh is not found -SC_PATH_TCLCONFIG - -# if tclConfig.sh loads properly, assume libraries are there and working -SC_LOAD_TCLCONFIG - -# expect tclConfig.sh to populate TCL_LIB_FLAG and TCL_INCLUDE_SPEC -if test "$host_os_def" == "darwin"; then - TCL_LIB_SPEC="-ltcl" # OSX fails to populate this variable -fi -AC_SUBST([LIBTCL],[$TCL_LIB_SPEC]) - - -if test "x${TCL_INCLUDE_SPEC}" != "x-I/usr/include"; then - TS_ADDTO(TS_INCLUDES, [$TCL_INCLUDE_SPEC]) -fi - AC_CHECK_FUNCS([clock_gettime kqueue epoll_ctl posix_fadvise posix_madvise posix_fallocate inotify_init]) AC_CHECK_FUNCS([lrand48_r srand48_r port_create strlcpy strlcat sysconf sysctlbyname getpagesize]) AC_CHECK_FUNCS([getreuid getresuid getresgid setreuid setresuid getpeereid getpeerucred]) diff --git a/contrib/install_trafficserver.sh b/contrib/install_trafficserver.sh index 30daf2ce514..e8269434ffe 100644 --- a/contrib/install_trafficserver.sh +++ b/contrib/install_trafficserver.sh @@ -77,7 +77,6 @@ function updateInstall() { make \ libtool \ libssl-dev \ - tcl-dev \ libpcre3-dev \ curl apt-get install -y subversion git git-svn @@ -95,7 +94,6 @@ function updateInstall() { gcc-c++ \ glibc-devel \ openssl-devel \ - tcl-devel \ db4-devel \ pcre \ pcre-devel @@ -108,7 +106,6 @@ function updateInstall() { gcc-c++ \ glibc-devel \ openssl-devel \ - tcl-devel \ pcre \ pcre-devel fi diff --git a/contrib/vagrant-setup.sh b/contrib/vagrant-setup.sh index 8b62a4172b2..3bb255a144d 100644 --- a/contrib/vagrant-setup.sh +++ b/contrib/vagrant-setup.sh @@ -36,8 +36,7 @@ trusty*|jessie*) libssl-dev \ m4 \ ncurses-dev \ - git \ - tcl-dev + git ;; centos*) @@ -57,8 +56,7 @@ centos*) ncurses-devel \ openssl-devel \ pcre-devel \ - git \ - tcl-devel + git ;; fedora*) @@ -78,7 +76,6 @@ fedora*) ncurses-devel \ openssl-devel \ pcre-devel \ - tcl-devel \ git \ make ;; @@ -106,7 +103,6 @@ omnios) developer/versioning/git \ library/idnkit \ library/idnkit/header-idnkit \ - omniti/runtime/tcl-8 \ omniti/system/hwloc \ system/header \ system/library/math \ diff --git a/doc/Doxyfile b/doc/Doxyfile index 5eb7b83d797..526d7447e8a 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -230,12 +230,6 @@ TAB_SIZE = 4 ALIASES = -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. - -TCL_SUBST = - # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For # instance, some of the names that are used will be different. The list of all diff --git a/doc/admin-guide/installation/index.en.rst b/doc/admin-guide/installation/index.en.rst index 48cbd90348a..07caf4c12f0 100644 --- a/doc/admin-guide/installation/index.en.rst +++ b/doc/admin-guide/installation/index.en.rst @@ -105,7 +105,6 @@ development tools and libraries installed: - gcc (>= 4.3 or clang > 3.0) - GNU make - openssl (libssl-dev for Ubuntu 16.04) -- tcl (tcl-8.6-dev for Ubuntu 16.04) - pcre (libpcre3-dev for Ubuntu 16.04) - libcap - flex (for TPROXY) diff --git a/doc/getting-started/index.en.rst b/doc/getting-started/index.en.rst index e65ad67ab2d..1713768fdef 100644 --- a/doc/getting-started/index.en.rst +++ b/doc/getting-started/index.en.rst @@ -154,7 +154,6 @@ libraries on the machine used to build |TS|: - gcc (>= 4.3 or clang > 3.0) - GNU make - openssl -- tcl - pcre - libcap - flex (for TPROXY) diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/installation/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/installation/index.en.po index 1b8840e5bad..107ca84ef96 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/installation/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/installation/index.en.po @@ -143,10 +143,6 @@ msgstr "" msgid "openssl" msgstr "openssl" -#: ../../../admin-guide/installation/index.en.rst:108 -msgid "tcl" -msgstr "tcl" - #: ../../../admin-guide/installation/index.en.rst:109 msgid "expat" msgstr "expat" diff --git a/doc/locale/ja/LC_MESSAGES/getting-started.en.po b/doc/locale/ja/LC_MESSAGES/getting-started.en.po index 191a40fffbb..eb659724e91 100644 --- a/doc/locale/ja/LC_MESSAGES/getting-started.en.po +++ b/doc/locale/ja/LC_MESSAGES/getting-started.en.po @@ -236,10 +236,6 @@ msgstr "" msgid "openssl" msgstr "" -#: ../../getting-started.en.rst:157 -msgid "tcl" -msgstr "" - #: ../../getting-started.en.rst:158 msgid "expat" msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/getting-started/index.en.po b/doc/locale/ja/LC_MESSAGES/getting-started/index.en.po index 9b3e4d64629..329cce2e91e 100644 --- a/doc/locale/ja/LC_MESSAGES/getting-started/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/getting-started/index.en.po @@ -238,10 +238,6 @@ msgstr "" msgid "openssl" msgstr "openssl" -#: ../../../getting-started/index.en.rst:157 -msgid "tcl" -msgstr "tcl" - #: ../../../getting-started/index.en.rst:158 msgid "expat" msgstr "expat" diff --git a/include/tscore/ink_hash_table.h b/include/tscore/ink_hash_table.h deleted file mode 100644 index e85df7a124e..00000000000 --- a/include/tscore/ink_hash_table.h +++ /dev/null @@ -1,145 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - ink_hash_table.h - - This file implements hash tables. This allows us to provide alternative - implementations of hash tables. - - ****************************************************************************/ - -#pragma once - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ -#include "tscore/ink_apidefs.h" - -/*===========================================================================* - - This is the Tcl implementation of InkHashTable - - *===========================================================================*/ - -#include - -typedef Tcl_HashTable InkHashTable; -typedef Tcl_HashEntry InkHashTableEntry; -typedef char *InkHashTableKey; -typedef ClientData InkHashTableValue; -typedef Tcl_HashSearch InkHashTableIteratorState; - -typedef int (*InkHashTableEntryFunction)(InkHashTable *ht, InkHashTableEntry *entry); - -/* Types of InkHashTable Keys */ - -typedef enum { - InkHashTableKeyType_String, - InkHashTableKeyType_Word, -} InkHashTableKeyType; - -/*===========================================================================* - - Function Prototypes - - *===========================================================================*/ - -InkHashTable *ink_hash_table_create(InkHashTableKeyType key_type); -InkHashTable *ink_hash_table_destroy(InkHashTable *ht_ptr); -InkHashTable *ink_hash_table_destroy_and_free_values(InkHashTable *ht_ptr); -inkcoreapi int ink_hash_table_isbound(InkHashTable *ht_ptr, const char *key); -inkcoreapi int ink_hash_table_lookup(InkHashTable *ht_ptr, const char *key, InkHashTableValue *value_ptr); -inkcoreapi int ink_hash_table_delete(InkHashTable *ht_ptr, const char *key); -InkHashTableEntry *ink_hash_table_lookup_entry(InkHashTable *ht_ptr, const char *key); -InkHashTableEntry *ink_hash_table_get_entry(InkHashTable *ht_ptr, const char *key, int *new_value); -void ink_hash_table_set_entry(InkHashTable *ht_ptr, InkHashTableEntry *he_ptr, InkHashTableValue value); -inkcoreapi void ink_hash_table_insert(InkHashTable *ht_ptr, const char *key, InkHashTableValue value); -void ink_hash_table_map(InkHashTable *ht_ptr, InkHashTableEntryFunction map); -InkHashTableKey ink_hash_table_entry_key(InkHashTable *ht_ptr, InkHashTableEntry *entry_ptr); -inkcoreapi InkHashTableValue ink_hash_table_entry_value(InkHashTable *ht_ptr, InkHashTableEntry *entry_ptr); -void ink_hash_table_dump_strings(InkHashTable *ht_ptr); -void ink_hash_table_replace_string(InkHashTable *ht_ptr, InkHashTableKey key, char *str); - -/* inline functions declarations */ - -/*---------------------------------------------------------------------------* - - InkHashTableEntry *ink_hash_table_iterator_first - (InkHashTable *ht_ptr, InkHashTableIteratorState *state_ptr) - - This routine takes a hash table , creates some iterator state - stored through , and returns a pointer to the first hash table - entry. The iterator state is used by InkHashTableIteratorNext() to proceed - through the rest of the entries. - - *---------------------------------------------------------------------------*/ - -static inline InkHashTableEntry * -ink_hash_table_iterator_first(InkHashTable *ht_ptr, InkHashTableIteratorState *state_ptr) -{ - Tcl_HashTable *tcl_ht_ptr; - Tcl_HashSearch *tcl_search_state_ptr; - Tcl_HashEntry *tcl_he_ptr; - InkHashTableEntry *he_ptr; - - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - tcl_search_state_ptr = (Tcl_HashSearch *)state_ptr; - - tcl_he_ptr = Tcl_FirstHashEntry(tcl_ht_ptr, tcl_search_state_ptr); - he_ptr = (InkHashTableEntry *)tcl_he_ptr; - - return (he_ptr); -} /* End ink_hash_table_iterator_first */ - -/*---------------------------------------------------------------------------* - - InkHashTableEntry *ink_hash_table_iterator_next(InkHashTable *ht_ptr, - InkHashTableIteratorState *state_ptr) - - This routine takes a hash table and a pointer to iterator state - initialized by a previous call to InkHashTableIteratorFirst(), and returns - a pointer to the next InkHashTableEntry, or nullptr if no entries remain. - - *---------------------------------------------------------------------------*/ - -static inline InkHashTableEntry * -ink_hash_table_iterator_next(InkHashTable *ht_ptr, InkHashTableIteratorState *state_ptr) -{ - (void)ht_ptr; - Tcl_HashSearch *tcl_search_state_ptr; - Tcl_HashEntry *tcl_he_ptr; - InkHashTableEntry *he_ptr; - - tcl_search_state_ptr = (Tcl_HashSearch *)state_ptr; - - tcl_he_ptr = Tcl_NextHashEntry(tcl_search_state_ptr); - he_ptr = (InkHashTableEntry *)tcl_he_ptr; - - return (he_ptr); -} /* End ink_hash_table_iterator_next */ - -#ifdef __cplusplus -} -#endif /* __cplusplus */ diff --git a/iocore/aio/Makefile.am b/iocore/aio/Makefile.am index 8815fda4b6e..999f80de6c5 100644 --- a/iocore/aio/Makefile.am +++ b/iocore/aio/Makefile.am @@ -59,7 +59,7 @@ test_AIO_LDADD = \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ include $(top_srcdir)/build/tidy.mk diff --git a/iocore/eventsystem/Makefile.am b/iocore/eventsystem/Makefile.am index e1b83484485..d481f40b82b 100644 --- a/iocore/eventsystem/Makefile.am +++ b/iocore/eventsystem/Makefile.am @@ -95,7 +95,7 @@ test_LD_ADD = \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ $(top_builddir)/src/tscore/libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ test_Buffer_SOURCES = \ test_Buffer.cc diff --git a/iocore/hostdb/Makefile.am b/iocore/hostdb/Makefile.am index 1ae57e313b0..8f49b12d686 100644 --- a/iocore/hostdb/Makefile.am +++ b/iocore/hostdb/Makefile.am @@ -74,7 +74,7 @@ test_LD_ADD = \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ test_RefCountCache_CPPFLAGS = $(test_CPP_FLAGS) diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index ed407751ec0..4248b38e82a 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -76,7 +76,7 @@ test_UDPNet_LDADD = \ $(top_builddir)/src/tscore/libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/ParentSelectionStrategy.o \ $(top_builddir)/lib/tsconfig/libtsconfig.la \ - @LIBTCL@ @HWLOC_LIBS@ @OPENSSL_LIBS@ @YAMLCPP_LIBS@ + @HWLOC_LIBS@ @OPENSSL_LIBS@ @YAMLCPP_LIBS@ test_UDPNet_SOURCES = \ test_I_UDPNet.cc diff --git a/mgmt/api/Makefile.am b/mgmt/api/Makefile.am index ae9fa950e7c..0ddd113b49d 100644 --- a/mgmt/api/Makefile.am +++ b/mgmt/api/Makefile.am @@ -83,7 +83,7 @@ traffic_api_cli_remote_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ @OPENSSL_LIBS@ + @OPENSSL_LIBS@ clang-tidy-local: $(DIST_SOURCES) $(CXX_Clang_Tidy) diff --git a/proxy/hdrs/Makefile.am b/proxy/hdrs/Makefile.am index 42dad696a94..0f08679ed40 100644 --- a/proxy/hdrs/Makefile.am +++ b/proxy/hdrs/Makefile.am @@ -59,8 +59,7 @@ load_http_hdr_SOURCES = \ load_http_hdr_LDADD = -L. -lhdrs \ $(top_builddir)/src/tscore/libtscore.la \ - $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ + $(top_builddir)/src/tscpp/util/libtscpputil.la check_PROGRAMS = test_mime @@ -74,7 +73,7 @@ test_mime_LDADD = -L. -lhdrs \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ @HWLOC_LIBS@ \ - @LIBTCL@ @LIBCAP@ + @LIBCAP@ test_mime_SOURCES = test_mime.cc diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am index 0706127c8f8..6f5733229b0 100644 --- a/proxy/http/Makefile.am +++ b/proxy/http/Makefile.am @@ -107,7 +107,7 @@ test_proxy_http_LDADD = \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/iocore/utils/libinkutils.a \ @HWLOC_LIBS@ \ - @LIBTCL@ @LIBCAP@ + @LIBCAP@ clang-tidy-local: $(libhttp_a_SOURCES) $(noinst_HEADERS) $(CXX_Clang_Tidy) diff --git a/proxy/http2/Makefile.am b/proxy/http2/Makefile.am index ad08f7e7318..47cbb8e7577 100644 --- a/proxy/http2/Makefile.am +++ b/proxy/http2/Makefile.am @@ -95,7 +95,6 @@ test_HPACK_LDADD = \ $(top_builddir)/lib/records/librecords_p.a \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @LIBTCL@ \ @HWLOC_LIBS@ test_HPACK_SOURCES = \ diff --git a/src/traffic_cache_tool/Makefile.inc b/src/traffic_cache_tool/Makefile.inc index 6cb9430f8ce..8af4112533c 100644 --- a/src/traffic_cache_tool/Makefile.inc +++ b/src/traffic_cache_tool/Makefile.inc @@ -53,4 +53,4 @@ traffic_cache_tool_traffic_cache_tool_LDADD = \ $(top_builddir)/src/tscore/.libs/ink_args.o \ $(top_builddir)/src/tscore/.libs/ParseRules.o \ $(top_builddir)/src/tscore/.libs/SourceLocation.o \ - @OPENSSL_LIBS@ @LIBPCRE@ @LIBTCL@ + @OPENSSL_LIBS@ @LIBPCRE@ diff --git a/src/traffic_crashlog/Makefile.inc b/src/traffic_crashlog/Makefile.inc index b7a408b5499..f665ab63d06 100644 --- a/src/traffic_crashlog/Makefile.inc +++ b/src/traffic_crashlog/Makefile.inc @@ -44,4 +44,4 @@ traffic_crashlog_traffic_crashlog_LDADD = \ $(top_builddir)/mgmt/api/libtsmgmt.la \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ diff --git a/src/traffic_ctl/Makefile.inc b/src/traffic_ctl/Makefile.inc index 1efd85e6ab9..446e56539f9 100644 --- a/src/traffic_ctl/Makefile.inc +++ b/src/traffic_ctl/Makefile.inc @@ -47,4 +47,4 @@ traffic_ctl_traffic_ctl_LDADD = \ $(top_builddir)/mgmt/api/libtsmgmt.la \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ diff --git a/src/traffic_layout/Makefile.inc b/src/traffic_layout/Makefile.inc index e22f3dec0b3..7664fe3d790 100644 --- a/src/traffic_layout/Makefile.inc +++ b/src/traffic_layout/Makefile.inc @@ -47,4 +47,4 @@ traffic_layout_traffic_layout_LDADD = \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ @HWLOC_LIBS@ @YAMLCPP_LIBS@ + @HWLOC_LIBS@ @YAMLCPP_LIBS@ diff --git a/src/traffic_logcat/Makefile.inc b/src/traffic_logcat/Makefile.inc index 4c962424051..fe77562a4c6 100644 --- a/src/traffic_logcat/Makefile.inc +++ b/src/traffic_logcat/Makefile.inc @@ -50,6 +50,6 @@ traffic_logcat_traffic_logcat_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la traffic_logcat_traffic_logcat_LDADD += \ - @LIBTCL@ @HWLOC_LIBS@ \ + @HWLOC_LIBS@ \ @YAMLCPP_LIBS@ \ @LIBPROFILER@ -lm diff --git a/src/traffic_logstats/Makefile.inc b/src/traffic_logstats/Makefile.inc index 89260840107..12f8335692f 100644 --- a/src/traffic_logstats/Makefile.inc +++ b/src/traffic_logstats/Makefile.inc @@ -54,6 +54,6 @@ traffic_logstats_traffic_logstats_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la traffic_logstats_traffic_logstats_LDADD += \ - @LIBTCL@ @HWLOC_LIBS@ \ + @HWLOC_LIBS@ \ @YAMLCPP_LIBS@ \ @LIBPROFILER@ -lm diff --git a/src/traffic_manager/Makefile.inc b/src/traffic_manager/Makefile.inc index de2d62584ad..19dfb927568 100644 --- a/src/traffic_manager/Makefile.inc +++ b/src/traffic_manager/Makefile.inc @@ -49,7 +49,7 @@ traffic_manager_traffic_manager_LDADD = \ $(top_builddir)/lib/records/librecords_lm.a \ $(top_builddir)/proxy/shared/libdiagsconfig.a \ $(LIBUNWIND_LIBS) \ - @LIBPCRE@ @LIBTCL@ @LIBCAP@ @HWLOC_LIBS@ \ + @LIBPCRE@ @LIBCAP@ @HWLOC_LIBS@ \ @YAMLCPP_LIBS@ -lm diff --git a/src/traffic_server/Makefile.inc b/src/traffic_server/Makefile.inc index 98966c383e8..4657128de0f 100644 --- a/src/traffic_server/Makefile.inc +++ b/src/traffic_server/Makefile.inc @@ -83,7 +83,6 @@ traffic_server_traffic_server_LDADD = \ $(top_builddir)/lib/tsconfig/libtsconfig.la \ @HWLOC_LIBS@ \ @LIBPCRE@ \ - @LIBTCL@ \ @LIBRESOLV@ \ @LIBZ@ \ @LIBLZMA@ \ diff --git a/src/traffic_top/Makefile.inc b/src/traffic_top/Makefile.inc index d3bced9e5d8..7d0a820fb95 100644 --- a/src/traffic_top/Makefile.inc +++ b/src/traffic_top/Makefile.inc @@ -49,6 +49,6 @@ traffic_top_traffic_top_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ @CURL_LIBS@ \ @CURSES_LIBS@ \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ endif diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index d3a312c5f22..52047452d0f 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -42,7 +42,6 @@ libtscore_la_LIBADD = \ @LIBOBJS@ \ @LIBPCRE@ \ @OPENSSL_LIBS@ \ - @LIBTCL@ \ @LIBRESOLV@ \ @LIBCAP@ \ @YAMLCPP_LIBS@ \ @@ -112,8 +111,6 @@ libtscore_la_SOURCES = \ ink_exception.h \ ink_file.cc \ ink_file.h \ - ink_hash_table.cc \ - ink_hash_table.h \ ink_hrtime.cc \ ink_hrtime.h \ ink_inet.cc \ @@ -225,16 +222,16 @@ ParseRulesCType: CompileParseRules mkdfa_SOURCES = mkdfa.c test_atomic_SOURCES = test_atomic.cc -test_atomic_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBTCL@ @LIBPCRE@ +test_atomic_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBPCRE@ test_freelist_SOURCES = test_freelist.cc -test_freelist_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBTCL@ @LIBPCRE@ +test_freelist_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBPCRE@ test_geometry_SOURCES = test_geometry.cc -test_geometry_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBTCL@ @LIBPCRE@ +test_geometry_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBPCRE@ test_X509HostnameValidator_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -test_X509HostnameValidator_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBTCL@ @LIBPCRE@ @OPENSSL_LIBS@ +test_X509HostnameValidator_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBPCRE@ @OPENSSL_LIBS@ test_X509HostnameValidator_SOURCES = unit_tests/test_X509HostnameValidator.cc test_tscore_CPPFLAGS = $(AM_CPPFLAGS)\ diff --git a/src/tscore/ink_hash_table.cc b/src/tscore/ink_hash_table.cc deleted file mode 100644 index d4cf6e7eab1..00000000000 --- a/src/tscore/ink_hash_table.cc +++ /dev/null @@ -1,425 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - ink_hash_table.c - - This file implements hash tables. This allows us to provide alternative - implementations of hash tables. - - ****************************************************************************/ - -#include "tscore/ink_error.h" -#include "tscore/ink_hash_table.h" -#include "tscore/ink_memory.h" - -/*===========================================================================* - - This is the Tcl implementation of InkHashTable - - *===========================================================================*/ - -/*---------------------------------------------------------------------------* - - InkHashTable *ink_hash_table_create(InkHashTableKeyType key_type) - - This routine allocates an initializes an empty InkHashTable, and returns a - pointer to the new table. The argument indicates whether keys - are represented as strings, or as words. Legal values are - InkHashTableKeyType_String and InkHashTableKeyType_Word. - - *---------------------------------------------------------------------------*/ - -InkHashTable * -ink_hash_table_create(InkHashTableKeyType key_type) -{ - InkHashTable *ht_ptr; - Tcl_HashTable *tcl_ht_ptr; - int tcl_key_type; - - tcl_ht_ptr = (Tcl_HashTable *)ats_malloc(sizeof(Tcl_HashTable)); - - if (key_type == InkHashTableKeyType_String) { - tcl_key_type = TCL_STRING_KEYS; - } else if (key_type == InkHashTableKeyType_Word) { - tcl_key_type = TCL_ONE_WORD_KEYS; - } else { - ink_fatal("ink_hash_table_create: bad key_type %d", key_type); - } - - Tcl_InitHashTable(tcl_ht_ptr, tcl_key_type); - - ht_ptr = (InkHashTable *)tcl_ht_ptr; - return (ht_ptr); -} /* End ink_hash_table_create */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_destroy(InkHashTable *ht_ptr) - - This routine takes a hash table , and frees its storage. - - *---------------------------------------------------------------------------*/ - -InkHashTable * -ink_hash_table_destroy(InkHashTable *ht_ptr) -{ - Tcl_HashTable *tcl_ht_ptr; - - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - Tcl_DeleteHashTable(tcl_ht_ptr); - ats_free(tcl_ht_ptr); - return (InkHashTable *)nullptr; -} /* End ink_hash_table_destroy */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_destroy_and_free_values(InkHashTable *ht_ptr) - - This routine takes a hash table , and frees its storage, after - first calling ink_free on all the values. You better darn well make sure the - values have been dynamically allocated. - - *---------------------------------------------------------------------------*/ - -static int -_ink_hash_table_free_entry_value(InkHashTable *ht_ptr, InkHashTableEntry *e) -{ - InkHashTableValue value; - - value = ink_hash_table_entry_value(ht_ptr, e); - if (value != nullptr) { - ats_free(value); - } - - return (0); -} /* End _ink_hash_table_free_entry_value */ - -InkHashTable * -ink_hash_table_destroy_and_free_values(InkHashTable *ht_ptr) -{ - ink_hash_table_map(ht_ptr, _ink_hash_table_free_entry_value); - ink_hash_table_destroy(ht_ptr); - return (InkHashTable *)nullptr; -} /* End ink_hash_table_destroy_and_free_values */ - -/*---------------------------------------------------------------------------* - - int ink_hash_table_isbound(InkHashTable *ht_ptr, InkHashTableKey key) - - This routine takes a hash table , a key , and returns 1 - if the value is bound in the hash table, 0 otherwise. - - *---------------------------------------------------------------------------*/ - -int -ink_hash_table_isbound(InkHashTable *ht_ptr, const char *key) -{ - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_lookup_entry(ht_ptr, key); - return ((he_ptr == nullptr) ? 0 : 1); -} /* End ink_hash_table_isbound */ - -/*---------------------------------------------------------------------------* - int ink_hash_table_lookup(InkHashTable *ht_ptr, - InkHashTableKey key, - InkHashTableValue *value_ptr) - - This routine takes a hash table , a key , and stores the - value bound to the key by reference through . If no binding is - found, 0 is returned, else 1 is returned. - - *---------------------------------------------------------------------------*/ - -int -ink_hash_table_lookup(InkHashTable *ht_ptr, const char *key, InkHashTableValue *value_ptr) -{ - InkHashTableEntry *he_ptr; - InkHashTableValue value; - - he_ptr = ink_hash_table_lookup_entry(ht_ptr, key); - if (he_ptr == nullptr) { - return (0); - } - - value = ink_hash_table_entry_value(ht_ptr, he_ptr); - *value_ptr = value; - return (1); -} /* End ink_hash_table_lookup */ - -/*---------------------------------------------------------------------------* - - int ink_hash_table_delete(InkHashTable *ht_ptr, InkHashTableKey key) - - This routine takes a hash table and a key , and deletes the - binding for the in the hash table if it exists. This routine - returns 1 if the key existed, else 0. - - *---------------------------------------------------------------------------*/ - -int -ink_hash_table_delete(InkHashTable *ht_ptr, const char *key) -{ - char *tcl_key; - Tcl_HashTable *tcl_ht_ptr; - Tcl_HashEntry *tcl_he_ptr; - - tcl_key = (char *)key; - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - tcl_he_ptr = Tcl_FindHashEntry(tcl_ht_ptr, tcl_key); - - if (!tcl_he_ptr) { - return (0); - } - Tcl_DeleteHashEntry(tcl_he_ptr); - - return (1); -} /* End ink_hash_table_delete */ - -/*---------------------------------------------------------------------------* - - InkHashTableEntry *ink_hash_table_lookup_entry(InkHashTable *ht_ptr, - InkHashTableKey key) - - This routine takes a hash table and a key , and returns the - entry matching the key, or nullptr otherwise. - - *---------------------------------------------------------------------------*/ - -InkHashTableEntry * -ink_hash_table_lookup_entry(InkHashTable *ht_ptr, const char *key) -{ - Tcl_HashTable *tcl_ht_ptr; - Tcl_HashEntry *tcl_he_ptr; - InkHashTableEntry *he_ptr; - - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - tcl_he_ptr = Tcl_FindHashEntry(tcl_ht_ptr, key); - he_ptr = (InkHashTableEntry *)tcl_he_ptr; - - return (he_ptr); -} /* End ink_hash_table_lookup_entry */ - -/*---------------------------------------------------------------------------* - - InkHashTableEntry *ink_hash_table_get_entry(InkHashTable *ht_ptr, - InkHashTableKey key, - int *new_value) - - This routine takes a hash table and a key , and returns the - entry matching the key, or creates, binds, and returns a new entry. - If the binding already existed, *new is set to 0, else 1. - - *---------------------------------------------------------------------------*/ - -InkHashTableEntry * -ink_hash_table_get_entry(InkHashTable *ht_ptr, const char *key, int *new_value) -{ - Tcl_HashTable *tcl_ht_ptr; - Tcl_HashEntry *tcl_he_ptr; - - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - tcl_he_ptr = Tcl_CreateHashEntry(tcl_ht_ptr, key, new_value); - - if (tcl_he_ptr == nullptr) { - ink_fatal("%s: Tcl_CreateHashEntry returned nullptr", "ink_hash_table_get_entry"); - } - - return ((InkHashTableEntry *)tcl_he_ptr); -} /* End ink_hash_table_get_entry */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_set_entry(InkHashTable *ht_ptr, - InkHashTableEntry *he_ptr, - InkHashTableValue value) - - This routine takes a hash table , a hash table entry , - and changes the value field of the entry to . - - *---------------------------------------------------------------------------*/ - -void -ink_hash_table_set_entry(InkHashTable *ht_ptr, InkHashTableEntry *he_ptr, InkHashTableValue value) -{ - (void)ht_ptr; - ClientData tcl_value; - Tcl_HashEntry *tcl_he_ptr; - - tcl_value = (ClientData)value; - tcl_he_ptr = (Tcl_HashEntry *)he_ptr; - Tcl_SetHashValue(tcl_he_ptr, tcl_value); -} /* End ink_hash_table_set_entry */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_insert(InkHashTable *ht_ptr, - InkHashTableKey key, - InkHashTableValue value) - - This routine takes a hash table , a key , and binds the value - to the key, replacing any previous binding, if any. - - *---------------------------------------------------------------------------*/ - -void -ink_hash_table_insert(InkHashTable *ht_ptr, const char *key, InkHashTableValue value) -{ - int new_value; - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_get_entry(ht_ptr, key, &new_value); - ink_hash_table_set_entry(ht_ptr, he_ptr, value); -} /* End ink_hash_table_insert */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_map(InkHashTable *ht_ptr, InkHashTableEntryFunction map) - - This routine takes a hash table and a function pointer , and - applies the function to each entry in the hash table. The function - should return 0 normally, otherwise the iteration will stop. - - *---------------------------------------------------------------------------*/ - -void -ink_hash_table_map(InkHashTable *ht_ptr, InkHashTableEntryFunction map) -{ - int retcode; - InkHashTableEntry *e; - InkHashTableIteratorState state; - - for (e = ink_hash_table_iterator_first(ht_ptr, &state); e != nullptr; e = ink_hash_table_iterator_next(ht_ptr, &state)) { - retcode = (*map)(ht_ptr, e); - if (retcode != 0) { - break; - } - } -} /* End ink_hash_table_map */ - -/*---------------------------------------------------------------------------* - - InkHashTableKey ink_hash_table_entry_key(InkHashTable *ht_ptr, - InkHashTableEntry *entry_ptr) - - This routine takes a hash table and a pointer to a hash table - entry , and returns the key portion of the entry. - - *---------------------------------------------------------------------------*/ - -InkHashTableKey -ink_hash_table_entry_key(InkHashTable *ht_ptr, InkHashTableEntry *entry_ptr) -{ - char *tcl_key; - - tcl_key = (char *)Tcl_GetHashKey((Tcl_HashTable *)ht_ptr, (Tcl_HashEntry *)entry_ptr); - return ((InkHashTableKey)tcl_key); -} /* End ink_hash_table_entry_key */ - -/*---------------------------------------------------------------------------* - - InkHashTableValue ink_hash_table_entry_value(InkHashTable *ht_ptr, - InkHashTableEntry *entry_ptr) - - This routine takes a hash table and a pointer to a hash table - entry , and returns the value portion of the entry. - - *---------------------------------------------------------------------------*/ - -InkHashTableValue -ink_hash_table_entry_value(InkHashTable *ht_ptr, InkHashTableEntry *entry_ptr) -{ - (void)ht_ptr; - ClientData tcl_value; - - tcl_value = Tcl_GetHashValue((Tcl_HashEntry *)entry_ptr); - return ((InkHashTableValue)tcl_value); -} /* End ink_hash_table_entry_value */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_dump_strings(InkHashTable *ht_ptr) - - This routine takes a hash table of string values, and dumps the keys and - string values to stdout. It is the caller's responsibility to ensure that - both the key and the value are NUL terminated strings. - - *---------------------------------------------------------------------------*/ - -static int -DumpStringEntry(InkHashTable *ht_ptr, InkHashTableEntry *e) -{ - InkHashTableKey key; - InkHashTableValue value; - - key = ink_hash_table_entry_key(ht_ptr, e); - value = ink_hash_table_entry_value(ht_ptr, e); - - fprintf(stderr, "key = '%s', value = '%s'\n", (char *)key, (char *)value); - - return (0); -} - -void -ink_hash_table_dump_strings(InkHashTable *ht_ptr) -{ - ink_hash_table_map(ht_ptr, DumpStringEntry); -} /* End ink_hash_table_dump_strings */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_replace_string(InkHashTable *ht_ptr, - char *string_key, char *string_value) - - This conveninece routine is intended for hash tables with keys of type - InkHashTableKeyType_String, and values being dynamically allocated strings. - This routine binds to a copy of , and any - previous bound value is deallocated. - - *---------------------------------------------------------------------------*/ - -void -ink_hash_table_replace_string(InkHashTable *ht_ptr, char *string_key, char *string_value) -{ - int new_value; - char *old_str; - InkHashTableEntry *he_ptr; - - /* - * The following line will flag a type-conversion warning on the - * DEC Alpha, but that message can be ignored, since we're - * still dealing with pointers, and we aren't loosing any bits. - */ - - he_ptr = ink_hash_table_get_entry(ht_ptr, (InkHashTableKey)string_key, &new_value); - if (new_value == 0) { - old_str = (char *)ink_hash_table_entry_value(ht_ptr, he_ptr); - if (old_str) { - ats_free(old_str); - } - } - - ink_hash_table_set_entry(ht_ptr, he_ptr, (InkHashTableValue)(ats_strdup(string_value))); -} /* End ink_hash_table_replace_string */ diff --git a/tools/package/trafficserver.spec b/tools/package/trafficserver.spec index 86dc4d90d61..3df89b95c0a 100755 --- a/tools/package/trafficserver.spec +++ b/tools/package/trafficserver.spec @@ -34,12 +34,12 @@ URL: https://trafficserver.apache.org/ Source0: http://www.apache.org/dist/%{name}/%{name}-%{version}.tar.bz2 -BuildRequires: expat-devel hwloc-devel openssl-devel pcre-devel tcl-devel zlib-devel xz-devel +BuildRequires: expat-devel hwloc-devel openssl-devel pcre-devel zlib-devel xz-devel BuildRequires: libcurl-devel ncurses-devel BuildRequires: gcc gcc-c++ perl-ExtUtils-MakeMaker BuildRequires: libcap-devel -Requires: expat hwloc openssl pcre tcl zlib xz libcurl ncurses pkgconfig +Requires: expat hwloc openssl pcre zlib xz libcurl ncurses pkgconfig Requires: libcap # Can't seem to use libunwind on RHEL7 or older From 8b40c578c77d8a31ee4fb57a912f269ad9a23aec Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Mon, 12 Nov 2018 10:35:00 -0600 Subject: [PATCH 046/526] Add TS_EVENT_AIO_DONE, deprecate TS_AIO_EVENT_DONE. --- include/ts/apidefs.h.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index ecdc8244402..53d692920a4 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -429,7 +429,8 @@ typedef enum { TS_EVENT_SSL_SESSION_GET = 2000, TS_EVENT_SSL_SESSION_NEW = 2001, TS_EVENT_SSL_SESSION_REMOVE = 2002, - TS_AIO_EVENT_DONE = 3900, + TS_EVENT_AIO_DONE = 3900, + TS_AIO_EVENT_DONE = TS_EVENT_AIO_DONE, // Deprecated, do not use in new code. TS_EVENT_HTTP_CONTINUE = 60000, TS_EVENT_HTTP_ERROR = 60001, TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, From ea65e96db36f549b716e537b22d508f7ebd5996c Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Mon, 12 Nov 2018 11:04:27 -0600 Subject: [PATCH 047/526] Remove deprecated event TS_AIO_EVENT_DONE. --- include/ts/apidefs.h.in | 1 - plugins/experimental/buffer_upload/buffer_upload.cc | 6 +++--- plugins/experimental/traffic_dump/traffic_dump.cc | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 53d692920a4..b8ab9629187 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -430,7 +430,6 @@ typedef enum { TS_EVENT_SSL_SESSION_NEW = 2001, TS_EVENT_SSL_SESSION_REMOVE = 2002, TS_EVENT_AIO_DONE = 3900, - TS_AIO_EVENT_DONE = TS_EVENT_AIO_DONE, // Deprecated, do not use in new code. TS_EVENT_HTTP_CONTINUE = 60000, TS_EVENT_HTTP_ERROR = 60001, TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, diff --git a/plugins/experimental/buffer_upload/buffer_upload.cc b/plugins/experimental/buffer_upload/buffer_upload.cc index fc6c3f601a0..92e64fd1fcd 100644 --- a/plugins/experimental/buffer_upload/buffer_upload.cc +++ b/plugins/experimental/buffer_upload/buffer_upload.cc @@ -543,12 +543,12 @@ pvc_plugin(TSCont contp, TSEvent event, void *edata) pvc_process_n_read(contp, event, my_state); } else if (edata == my_state->n_write_vio) { pvc_process_n_write(contp, event, my_state); - } else if (event == TS_AIO_EVENT_DONE && uconfig->use_disk_buffer) { + } else if (event == TS_EVENT_AIO_DONE && uconfig->use_disk_buffer) { TSMutexLock(my_state->disk_io_mutex); int size = TSAIONBytesGet(callback); char *buf = TSAIOBufGet(callback); if (buf != my_state->chunk_buffer) { - // this TS_AIO_EVENT_DONE event is from TSAIOWrite() + // this TS_EVENT_AIO_DONE event is from TSAIOWrite() TSDebug(DEBUG_TAG, "aio write size: %d", size); my_state->size_written += size; if (buf != nullptr) { @@ -561,7 +561,7 @@ pvc_plugin(TSCont contp, TSEvent event, void *edata) } } } else { - // this TS_AIO_EVENT_DONE event is from TSAIORead() + // this TS_EVENT_AIO_DONE event is from TSAIORead() TSDebug(DEBUG_TAG, "aio read size: %d", size); TSIOBufferWrite(my_state->req_buffer, my_state->chunk_buffer, size); my_state->size_read += size; diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc b/plugins/experimental/traffic_dump/traffic_dump.cc index acc61a39688..fff6e0323bb 100644 --- a/plugins/experimental/traffic_dump/traffic_dump.cc +++ b/plugins/experimental/traffic_dump/traffic_dump.cc @@ -290,7 +290,7 @@ int session_aio_handler(TSCont contp, TSEvent event, void *edata) { switch (event) { - case TS_AIO_EVENT_DONE: { + case TS_EVENT_AIO_DONE: { TSAIOCallback cb = static_cast(edata); SsnData *ssnData = static_cast(TSContDataGet(contp)); if (!ssnData) { From d41bbedc515290969f35cab37abecf826744382c Mon Sep 17 00:00:00 2001 From: Emanuele Rocca Date: Tue, 13 Nov 2018 13:58:31 +0100 Subject: [PATCH 048/526] Update logging.yaml documentation The current logging.yaml documentation still describes the old Lua format in various parts. Update it to reflect the YAML format and remove mentions to wipe filters, broken on 8.x. --- doc/admin-guide/files/logging.yaml.en.rst | 114 ++++++++-------------- 1 file changed, 38 insertions(+), 76 deletions(-) diff --git a/doc/admin-guide/files/logging.yaml.en.rst b/doc/admin-guide/files/logging.yaml.en.rst index 10bc4045596..a301aa5982c 100644 --- a/doc/admin-guide/files/logging.yaml.en.rst +++ b/doc/admin-guide/files/logging.yaml.en.rst @@ -23,14 +23,15 @@ logging.yaml ************** The :file:`logging.yaml` file defines all custom log file formats, filters, -and processing options. The file itself is a Lua script. +and processing options. .. important:: - This configuration file replaces the XML based logs_xml.config from past - |TS| releases. If you are upgrading from a |TS| release which used that - configuration file, and you have created custom log formats, filters, and - destinations, you will need to update those settings to this format. + This configuration file replaces the XML based logs_xml.config, as well as + the Lua based logging.config from past |TS| releases. If you are upgrading + from a |TS| release which used either the XML or the Lua configuration file + format, and you have created custom log formats, filters, and destinations, + you will need to update those settings to this format. .. _admin-custom-logs: @@ -62,11 +63,10 @@ format. Which approach you use is entirely up to you, though it's strongly recommended to create an explicit format object if you intend to reuse the same format for multiple log files. -To create a format object, store the result of the ``format`` function in a -variable. The function takes a table with two attributes: a mandatory string -``Format`` which defines the output format string for every event; and an -optional number ``Interval`` defining the aggregation interval for summary -logs. +Custom formats are defined by choosing a ``name`` to identify the given logging +format, and a ``format`` string, which defines the output format string for +every event. An optional ``interval`` attribute can be specified to define the +aggregation interval for summary logs. .. code:: yaml @@ -89,8 +89,8 @@ desire. Format Specification ~~~~~~~~~~~~~~~~~~~~ -The format specification provided as the required ``Format`` entry of the table -passed to the format function is a simple string, containing whatever mixture +The format specification provided as the required ``format`` attribute of the +objects listed in ``formats`` is a simple string, containing whatever mixture of logging field variables and literal characters meet your needs. Logging fields are discussed in great detail in the :ref:`admin-logging-fields` section. @@ -142,7 +142,7 @@ You will find a complete listing of the available fields in Aggregation Interval ~~~~~~~~~~~~~~~~~~~~ -Every format may be given an optional ``Interval`` value, specified as the +Every format may be given an optional ``interval`` value, specified as the number of seconds over which events destined for a log using the format are aggregated and summarized. Logs which use formats containing an aggregation interval do not behave like regular logs, with a single line for every event. @@ -159,29 +159,26 @@ given one. Filters ------- -Filters may be used, optionally, to accept or reject logging for matching -events, or to scrub the values of individual fields from logging output (while -retaining other information; useful for ensuring that sensitive information -cannot inadvertently make it into log files). +Two different type of filters are available: ``accept`` and ``reject``. They +may be used, optionally, to accept or reject logging for matching events. -Filter objects are created by calling one of the following functions: +Filter objects are created by assigning them a ``name`` to be used later to +refer to the filter, as well as an ``action`` (either ``accept`` or +``reject``). ``Accept`` and ``reject`` filters require a ``condition`` against +which to match all events. The ``condition`` fields must be in the following +format:: -filter.accept(string) - Creates a filter object which accepts events for logging which match the - rule specified in ``string``. Note that you may only have one accept filter. - -filter.reject(string) - Creates a filter object which rejects events for logging which match the - rule specified in ``string``. You may have multiple reject filters. + -filter.wipe(string) - Creates a filter object which clears the values of query parameters listed - in ``string``. +For example, the following snippet defines a filter that matches all POST +requests: -For both ``accept`` and ``wipe`` filters, the string passed defines a rule in -the following format:: +.. code:: yaml - + filters: + - name: postfilter + action: accept + condition: cqhm MATCH POST Filter Fields ~~~~~~~~~~~~~ @@ -245,21 +242,6 @@ supported at this time. expect. If, for example, we had 2 accept log filters, each disjoint from the other, nothing will ever get logged on the given log object. -Wiping Filters -~~~~~~~~~~~~~~ - -Filters created with ``filter.wipe`` function differently than the accept and -reject filters. Instead of a rule, as described above for those filter types, -the wiping filter simply lists the query parameter(s) whose values should be -scrubbed before any logging occurs. This prevents sensitive information from -being logged by fields which include the query string portion of the request -URL. It can also be useful to remove things like cache-busting or -inconsequentially variable parameters that might otherwise obfuscate the -reporting from log analyzers. - -Multiple query parameters may be listed, separated by spaces, though only the -first occurence of each will be wiped from the query string if any individual -parameter appears more than once in the URL. .. _admin-custom-logs-logs: @@ -268,28 +250,16 @@ Logs Up to this point, we've only described what events should be logged and what they should look like in the logging output. Now we define where those logs -should be sent. Three options currently exist for the type of logging output, -and each is selected by invoking the appropriate function. All three functions -take a single Lua table as their argument, with the same set of key/value -pairs. - -log.ascii(table) - Creates an ASCII logging object. +should be sent. -log.binary(table) - Creates a binaryy logging object. +Three options currently exist for the type of logging output: ``ascii``, +``binary``, and ``ascii_pipe``. Which type of logging output you choose +depends largely on how you intend to process the logs with other tools, and a +discussion of the merits of each is covered elsewhere, in +:ref:`admin-logging-ascii-v-binary`. -log.pipe(table) - Creates a logging object that logs to a pipe. - -There is no need to capture the return values of these functions. Which type of -logging output you choose depends largely on how you intend to process the logs -with other tools, and a discussion of the merits of each is covered elsewhere, -in :ref:`admin-logging-ascii-v-binary`. - -The following subsections cover the contents of the table which should be -passed when creating your logging object. Only ``Filename`` and ``Format`` are -required. +The following subsections cover the attributes you should specify when creating +your logging object. Only ``filename`` and ``format`` are required. ====================== =========== ================================================= Name Type Description @@ -381,16 +351,8 @@ to be logged: filters: - name: refreshhitfilter - accept: pssc MATCH REFRESH_HIT - -The following is an example of a filter that will cause the value of the first -query parameter named ``passwd`` to be wiped. - -.. code:: yaml - - filters: - - name: passwdfilter - wipe: passwd + action: accept + condition: pssc MATCH REFRESH_HIT The following is an example of a log specification that creates a local log file for the minimal format defined earlier. The log filename will be From cb407ab5d4204f84660b0d1f438bb39e3eda8248 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 26 Nov 2018 16:26:23 +0000 Subject: [PATCH 049/526] Add h2disable test. --- tests/gold_tests/h2/h2disable.test.py | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/gold_tests/h2/h2disable.test.py diff --git a/tests/gold_tests/h2/h2disable.test.py b/tests/gold_tests/h2/h2disable.test.py new file mode 100644 index 00000000000..1e64b92c00d --- /dev/null +++ b/tests/gold_tests/h2/h2disable.test.py @@ -0,0 +1,102 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test disabling H2 on a per domain basis +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work"), + Condition.HasCurlFeature('http2') +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port with http2 + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + ' disable_h2: true', +# '- fqdn: bob.*.com', +# ' disable_h2: true', +]) + +tr = Test.AddTestRun("Negotiate-h2") +tr.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Using HTTP2", "Curl should negotiate HTTP2") +tr.TimeOut = 5 + +tr2 = Test.AddTestRun("Do not negotiate h2") +tr2.Processes.Default.Command = "curl -v -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Using HTTP2", "Curl should not negotiate HTTP2") +tr2.TimeOut = 5 + +#tr2 = Test.AddTestRun("Do not negotiate h2") +#tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format(ts.Variables.ssl_port) +#tr2.ReturnCode = 0 +#tr2.StillRunningAfter = server +#tr2.Processes.Default.TimeOut = 5 +#tr2.StillRunningAfter = ts +#tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +#tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Using HTTP2", "Curl should not negotiate HTTP2") +#tr2.TimeOut = 5 + + From 34c122a04211b4cff2d2d52df417128a1ce173bd Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Fri, 16 Nov 2018 22:52:27 +0000 Subject: [PATCH 050/526] Generalize the wild-card fqdn's for user_agent attributes in ssl_server_name --- iocore/net/Makefile.am | 3 +- iocore/net/P_SNIActionPerformer.h | 37 +++- iocore/net/P_SSLConfig.h | 1 - iocore/net/P_SSLNetVConnection.h | 43 ++++- iocore/net/P_SSLSNI.h | 63 +++++-- iocore/net/P_SSLUtils.h | 45 ----- iocore/net/SSLConfig.cc | 4 - iocore/net/SSLNetVConnection.cc | 21 +-- iocore/net/SSLSNIConfig.cc | 127 ++++--------- iocore/net/SSLUtils.cc | 28 ++- iocore/net/YamlSNIConfig.h | 2 +- mgmt/RecordsConfig.cc | 2 - proxy/http/HttpSM.cc | 30 ++-- tests/gold_tests/h2/h2disable.test.py | 22 +-- tests/gold_tests/tls/ssl/signed-bob-bar.pem | 18 ++ tests/gold_tests/tls/ssl/signed-bob-foo.pem | 18 ++ tests/gold_tests/tls/tls_client_cert2.test.py | 169 ++++++++++++++++++ .../gold_tests/tls/tls_client_verify.test.py | 80 ++++++++- .../gold_tests/tls/tls_client_verify2.test.py | 77 +++++++- tests/gold_tests/tls/tls_tunnel.test.py | 31 +++- tests/gold_tests/tls/tls_verify.test.py | 20 +-- tests/gold_tests/tls/tls_verify3.test.py | 147 +++++++++++++++ 22 files changed, 746 insertions(+), 242 deletions(-) create mode 100644 tests/gold_tests/tls/ssl/signed-bob-bar.pem create mode 100644 tests/gold_tests/tls/ssl/signed-bob-foo.pem create mode 100644 tests/gold_tests/tls/tls_client_cert2.test.py create mode 100644 tests/gold_tests/tls/tls_verify3.test.py diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index 4248b38e82a..e9efb3c3778 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -76,7 +76,7 @@ test_UDPNet_LDADD = \ $(top_builddir)/src/tscore/libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/ParentSelectionStrategy.o \ $(top_builddir)/lib/tsconfig/libtsconfig.la \ - @HWLOC_LIBS@ @OPENSSL_LIBS@ @YAMLCPP_LIBS@ + @HWLOC_LIBS@ @OPENSSL_LIBS@ @LIBPCRE@ @YAMLCPP_LIBS@ test_UDPNet_SOURCES = \ test_I_UDPNet.cc @@ -131,7 +131,6 @@ libinknet_a_SOURCES = \ P_UnixPollDescriptor.h \ P_UnixUDPConnection.h \ Socks.cc \ - SNIActionPerformer.cc \ SSLCertLookup.cc \ SSLSessionCache.cc \ SSLConfig.cc \ diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 739abbb10b7..8622ff2a033 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -51,7 +51,7 @@ enum PropertyActions { TS_VERIFY_SERVER = 200, TS_CLIENT_CERT }; class ActionItem { public: - virtual int SNIAction(Continuation *cont) = 0; + virtual int SNIAction(Continuation *cont) const = 0; virtual ~ActionItem(){}; }; @@ -62,9 +62,9 @@ class DisableH2 : public ActionItem ~DisableH2() override {} int - SNIAction(Continuation *cont) override + SNIAction(Continuation *cont) const override { - auto ssl_vc = reinterpret_cast(cont); + auto ssl_vc = dynamic_cast(cont); auto accept_obj = ssl_vc ? ssl_vc->accept_object : nullptr; if (accept_obj && accept_obj->snpa && ssl_vc) { if (auto it = snpsMap.find(accept_obj->id); it != snpsMap.end()) { @@ -75,6 +75,25 @@ class DisableH2 : public ActionItem } }; +class TunnelDestination : public ActionItem +{ +public: + TunnelDestination(const std::string_view &dest) : destination(dest) {} + ~TunnelDestination() {} + + int + SNIAction(Continuation *cont) const override + { + // Set the netvc option? + SSLNetVConnection *ssl_netvc = dynamic_cast(cont); + if (ssl_netvc) { + ssl_netvc->set_tunnel_destination(destination); + } + return SSL_TLSEXT_ERR_OK; + } + std::string destination; +}; + class VerifyClient : public ActionItem { uint8_t mode; @@ -84,9 +103,9 @@ class VerifyClient : public ActionItem VerifyClient(uint8_t param) : mode(param) {} ~VerifyClient() override {} int - SNIAction(Continuation *cont) override + SNIAction(Continuation *cont) const override { - auto ssl_vc = reinterpret_cast(cont); + auto ssl_vc = dynamic_cast(cont); Debug("ssl_sni", "action verify param %d", this->mode); setClientCertLevel(ssl_vc->ssl, this->mode); return SSL_TLSEXT_ERR_OK; @@ -98,7 +117,7 @@ class SNI_IpAllow : public ActionItem IpMap ip_map; public: - SNI_IpAllow(std::string const &ip_allow_list, const char *servername) + SNI_IpAllow(std::string &ip_allow_list, const std::string &servername) { // the server identified by item.fqdn requires ATS to do IP filtering if (ip_allow_list.length()) { @@ -113,7 +132,7 @@ class SNI_IpAllow : public ActionItem Debug("ssl_sni", "%.*s is not a valid format", static_cast(list.size()), list.data()); break; } else { - Debug("ssl_sni", "%.*s added to the ip_allow list %s", static_cast(list.size()), list.data(), servername); + Debug("ssl_sni", "%.*s added to the ip_allow list %s", static_cast(list.size()), list.data(), servername.c_str()); ip_map.fill(IpEndpoint().assign(addr1), IpEndpoint().assign(addr2), reinterpret_cast(1)); } } @@ -121,14 +140,14 @@ class SNI_IpAllow : public ActionItem } // end function SNI_IpAllow int - SNIAction(Continuation *cont) override + SNIAction(Continuation *cont) const override { // i.e, ip filtering is not required if (ip_map.count() == 0) { return SSL_TLSEXT_ERR_OK; } - auto ssl_vc = reinterpret_cast(cont); + auto ssl_vc = dynamic_cast(cont); auto ip = ssl_vc->get_remote_endpoint(); // check the allowed ips diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index b67944ba8b8..a691e5ba990 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -106,7 +106,6 @@ struct SSLConfigParams : public ConfigInfo { static size_t session_cache_number_buckets; static size_t session_cache_max_bucket_size; static bool session_cache_skip_on_lock_contention; - static bool sni_map_enable; // TS-3435 Wiretracing for SSL Connections static int ssl_wire_trace_enabled; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 72ef4d76d30..163709a059d 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -178,12 +178,6 @@ class SSLNetVConnection : public UnixNetVConnection return transparentPassThrough; } - bool - GetSNIMapping() - { - return SNIMapping; - } - void setTransparentPassThrough(bool val) { @@ -318,6 +312,40 @@ class SSLNetVConnection : public UnixNetVConnection return ssl ? SSL_get_cipher_name(ssl) : nullptr; } + bool + has_tunnel_destination() const + { + return tunnel_host != nullptr; + } + + const char * + get_tunnel_host() const + { + return tunnel_host; + } + + ushort + get_tunnel_port() const + { + return tunnel_port; + } + + void + set_tunnel_destination(const std::string_view &destination) + { + auto pos = destination.find(":"); + if (nullptr != tunnel_host) { + ats_free(tunnel_host); + } + if (pos != std::string::npos) { + tunnel_port = std::stoi(destination.substr(pos + 1).data()); + tunnel_host = ats_strndup(destination.substr(0, pos).data(), pos); + } else { + tunnel_port = 0; + tunnel_host = ats_strndup(destination.data(), destination.length()); + } + } + int populate_protocol(std::string_view *results, int n) const override; const char *protocol_contains(std::string_view tag) const override; @@ -378,8 +406,9 @@ class SSLNetVConnection : public UnixNetVConnection Continuation *npnEndpoint = nullptr; SessionAccept *sessionAcceptPtr = nullptr; bool sslTrace = false; - bool SNIMapping = false; int64_t redoWriteSize = 0; + char *tunnel_host = nullptr; + in_port_t tunnel_port = 0; }; typedef int (SSLNetVConnection::*SSLNetVConnHandler)(int, void *); diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index c6545aba3b7..781c4da8132 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -42,32 +42,73 @@ // Properties for the next hop server struct NextHopProperty { - const char *name = nullptr; // name of the server + std::string name; // name of the server YamlSNIConfig::Policy verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; // whether to verify the next hop YamlSNIConfig::Property verifyServerProperties = YamlSNIConfig::Property::NONE; // what to verify on the next hop SSL_CTX *ctx = nullptr; // ctx generated off the certificate to present to this server NextHopProperty(); }; -using actionVector = std::vector; -using SNIMap = std::unordered_map; -using NextHopPropertyTable = std::unordered_map; +using actionVector = std::vector>; + +struct namedElement { +public: + namedElement() {} + + void + setGlobName(std::string name) + { + std::string::size_type pos = 0; + while ((pos = name.find(".", pos)) != std::string::npos) { + name.replace(pos, 1, "\\."); + pos += 2; + } + pos = 0; + while ((pos = name.find("*", pos)) != std::string::npos) { + name.replace(pos, 1, ".{0,}"); + } + Debug("ssl_sni", "Regexed fqdn=%s", name.c_str()); + setRegexName(name); + } + + void + setRegexName(const std::string ®exName) + { + const char *err_ptr; + int err_offset = 0; + match = pcre_compile(regexName.c_str(), 0, &err_ptr, &err_offset, nullptr); + } + + pcre *match = nullptr; +}; + +struct actionElement : public namedElement { +public: + actionVector actions; +}; + +struct NextHopItem : public namedElement { +public: + NextHopProperty prop; +}; + +// typedef HashMap SNIMap; +typedef std::vector SNIList; +// typedef HashMap NextHopPropertyTable; +typedef std::vector NextHopPropertyList; struct SNIConfigParams : public ConfigInfo { char *sni_filename = nullptr; - SNIMap sni_action_map; - SNIMap wild_sni_action_map; - NextHopPropertyTable next_hop_table; - NextHopPropertyTable wild_next_hop_table; + SNIList sni_action_list; + NextHopPropertyList next_hop_list; YamlSNIConfig Y_sni; - NextHopProperty *getPropertyConfig(const char *servername) const; + const NextHopProperty *getPropertyConfig(const std::string &servername) const; SNIConfigParams(); ~SNIConfigParams() override; void cleanup(); int Initialize(); void loadSNIConfig(); - actionVector *get(const char *servername) const; - void printSNImap() const; + const actionVector *get(const std::string &servername) const; }; struct SNIConfig { diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index f70072b8047..3d2b01a1baf 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -242,50 +242,5 @@ struct ats_wildcard_matcher { DFA regex; }; -class TunnelHashMap -{ -public: - struct HostStruct { - std::string hostname; - int port; - HostStruct(const std::string_view &name, int port_) : hostname(name), port(port_) {} - }; - using Tunnel_hashMap = std::unordered_map; - Tunnel_hashMap TunnelhMap; - - void - emplace(const std::string &key, const std::string &hostname) - { - std::string_view addr, port; - if (ats_ip_parse(std::string_view(hostname), &addr, &port) == 0) { - TunnelhMap.emplace(key, HostStruct(addr, atoi(port.data()))); - } - } - - void - emplace(const std::string &key, const std::string &name, int port_) - { - TunnelhMap.emplace(key, HostStruct(name, port_)); - } - - Tunnel_hashMap::const_iterator - find(const std::string &key) const - { - return TunnelhMap.find(key); - } - - Tunnel_hashMap::const_iterator - begin() const - { - return TunnelhMap.begin(); - } - - Tunnel_hashMap::const_iterator - end() const - { - return TunnelhMap.end(); - } -}; - typedef ats_scoped_resource scoped_X509; typedef ats_scoped_resource scoped_BIO; diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index ba03ba4a32a..9c1a6d9c5ba 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -57,7 +57,6 @@ bool SSLConfigParams::session_cache_skip_on_lock_contention = false; size_t SSLConfigParams::session_cache_max_bucket_size = 100; init_ssl_ctx_func SSLConfigParams::init_ssl_ctx_cb = nullptr; load_ssl_file_func SSLConfigParams::load_ssl_file_cb = nullptr; -bool SSLConfigParams::sni_map_enable = false; // TS-3534 Wiretracing for SSL Connections int SSLConfigParams::ssl_wire_trace_enabled = 0; @@ -437,9 +436,6 @@ SSLConfigParams::initialize() REC_ReadConfigStringAlloc(client_groups_list, "proxy.config.ssl.client.groups_list"); - // Enable/disable sni mapping - REC_ReadConfigInteger(sni_map_enable, "proxy.config.ssl.sni.map.enable"); - REC_ReadConfigInt32(ssl_allow_client_renegotiation, "proxy.config.ssl.allow_client_renegotiation"); // SSL Wire Trace configurations diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index c16d52daac9..e222d8ef431 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -914,6 +914,8 @@ SSLNetVConnection::free(EThread *t) } con.close(); + ats_free(tunnel_host); + clear(); SET_CONTINUATION_HANDLER(this, (SSLNetVConnHandler)&SSLNetVConnection::startEvent); ink_assert(con.fd == NO_FD); @@ -972,9 +974,7 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) this->ssl = nullptr; return EVENT_DONE; } else { - SSLConfig::scoped_config params; - this->SNIMapping = params->sni_map_enable; - hookOpRequested = SSL_HOOK_OP_TUNNEL; + hookOpRequested = SSL_HOOK_OP_TUNNEL; } } @@ -1080,7 +1080,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // without data replay. // Note we can't arrive here if a hook is active. - if (SSL_HOOK_OP_TUNNEL == hookOpRequested && !SNIMapping) { + if (SSL_HOOK_OP_TUNNEL == hookOpRequested) { this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; SSL_free(this->ssl); this->ssl = nullptr; @@ -1607,8 +1607,6 @@ SSLNetVConnection::sslContextSet(void *ctx) return zret; } -extern TunnelHashMap TunnelMap; // stores the name of the servers to tunnel to - bool SSLNetVConnection::callHooks(TSEvent eventId) { @@ -1712,15 +1710,12 @@ SSLNetVConnection::callHooks(TSEvent eventId) bool reenabled = true; this->serverName = const_cast(SSL_get_servername(this->ssl, TLSEXT_NAMETYPE_host_name)); - if (this->serverName) { - if (auto it = TunnelMap.find(this->serverName); it != TunnelMap.end()) { - this->SNIMapping = true; - this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; - return reenabled; - } + if (this->has_tunnel_destination()) { + this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; + return reenabled; } - if (SSL_HOOK_OP_TUNNEL == hookOpRequested && SNIMapping) { + if (SSL_HOOK_OP_TUNNEL == hookOpRequested) { this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; // Don't mark the handshake as complete yet, // Will be checking for that flag not being set after diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index ad988c8fdf6..ca4f1934d82 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -35,56 +35,49 @@ #include "P_SSLConfig.h" #include "tscore/ink_memory.h" #include "tscpp/util/TextView.h" +#include +#include static ConfigUpdateHandler *sniConfigUpdate; struct NetAccept; std::unordered_map snpsMap; -extern TunnelHashMap TunnelMap; NextHopProperty::NextHopProperty() {} -NextHopProperty * -SNIConfigParams::getPropertyConfig(const char *servername) const +const NextHopProperty * +SNIConfigParams::getPropertyConfig(const std::string &servername) const { - if (auto it = next_hop_table.find(servername); it != next_hop_table.end()) { - return it->second; - } - if (auto it = wild_next_hop_table.find(servername); it != wild_next_hop_table.end()) { - return it->second; + const NextHopProperty *nps = nullptr; + for (auto &&item : next_hop_list) { + if (pcre_exec(item.match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { + // Found a match + nps = &item.prop; + break; + } } - return nullptr; + return nps; } void SNIConfigParams::loadSNIConfig() { - for (const auto &item : Y_sni.items) { - actionVector *aiVec = new actionVector(); + for (auto &item : Y_sni.items) { + auto ai = sni_action_list.emplace(sni_action_list.end()); + ai->setGlobName(item.fqdn); Debug("ssl", "name: %s", item.fqdn.data()); - const char *servername = item.fqdn.data(); - ats_wildcard_matcher w_Matcher; - auto wildcard = w_Matcher.match(servername); // set SNI based actions to be called in the ssl_servername_only callback - auto ai1 = new DisableH2(); - aiVec->push_back(ai1); - auto ai2 = new VerifyClient(item.verify_client_level); - aiVec->push_back(ai2); - if (wildcard) { - ts::TextView domain{servername, strlen(servername)}; - domain.take_prefix_at('.'); - if (!domain.empty()) { - wild_sni_action_map.emplace(domain, aiVec); - } - } else { - sni_action_map.emplace(servername, aiVec); + if (item.disable_h2) { + ai->actions.push_back(std::make_unique()); } - - if (item.tunnel_destination.length()) { - TunnelMap.emplace(item.fqdn, item.tunnel_destination); + if (item.verify_client_level != 255) { + ai->actions.push_back(std::make_unique(item.verify_client_level)); + } + if (item.tunnel_destination.length() > 0) { + ai->actions.push_back(std::make_unique(item.tunnel_destination)); } - auto ai3 = new SNI_IpAllow(item.ip_allow, servername); - aiVec->push_back(ai3); + ai->actions.push_back(std::make_unique(item.ip_allow, item.fqdn)); + // set the next hop properties SSLConfig::scoped_config params; auto clientCTX = params->getClientSSL_CTX(); @@ -93,19 +86,12 @@ SNIConfigParams::loadSNIConfig() if (certFile) { clientCTX = params->getNewCTX(certFile, keyFile); } - if (servername) { // a safety check - NextHopProperty *nps = new NextHopProperty(); - - nps->name = ats_strdup(servername); - nps->verifyServerPolicy = item.verify_server_policy; - nps->verifyServerProperties = item.verify_server_properties; - nps->ctx = clientCTX; - if (wildcard) { - wild_next_hop_table.emplace(nps->name, nps); - } else { - next_hop_table.emplace(nps->name, nps); - } - } + + auto nps = next_hop_list.emplace(next_hop_list.end()); + nps->setGlobName(item.fqdn); + nps->prop.verifyServerPolicy = item.verify_server_policy; + nps->prop.verifyServerProperties = item.verify_server_properties; + nps->prop.ctx = clientCTX; } // end for } @@ -113,29 +99,15 @@ int SNIConfig::configid = 0; /*definition of member functions of SNIConfigParams*/ SNIConfigParams::SNIConfigParams() {} -actionVector * -SNIConfigParams::get(const char *servername) const +const actionVector * +SNIConfigParams::get(const std::string &servername) const { - auto action_it = sni_action_map.find(servername); - if (action_it != sni_action_map.end()) { - for (const auto &it : wild_sni_action_map) { - std::string_view sv{servername}; - std::string_view key_sv{it.first}; - if (sv.size() >= key_sv.size() && sv.substr(sv.size() - key_sv.size()) == key_sv) { - auto wild_action_it = wild_sni_action_map.find(key_sv.data()); - return wild_action_it != wild_sni_action_map.end() ? wild_action_it->second : nullptr; - } + for (auto retval = sni_action_list.begin(); retval != sni_action_list.end(); ++retval) { + if (pcre_exec(retval->match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { + return &retval->actions; } } - return action_it != sni_action_map.end() ? action_it->second : nullptr; -} - -void -SNIConfigParams::printSNImap() const -{ - for (const auto &it : sni_action_map) { - Debug("ssl", "Domain name in the map %s: # of registered action items %lu", it.first.c_str(), it.second->size()); - } + return nullptr; } int @@ -166,34 +138,9 @@ SNIConfigParams::Initialize() return 0; } -void -SNIConfigParams::cleanup() -{ - for (const auto &it : sni_action_map) { - auto actionVec = it.second; - for (const auto &ai : *actionVec) { - delete ai; - } - delete actionVec; - } - for (const auto &it : wild_sni_action_map) { - auto actionVec = it.second; - for (const auto &ai : *actionVec) { - delete ai; - } - delete actionVec; - } - for (const auto &it : next_hop_table) { - delete it.second; - } - for (const auto &it : wild_next_hop_table) { - delete it.second; - } -} - SNIConfigParams::~SNIConfigParams() { - cleanup(); + // sni_action_list and next_hop_list should cleanup with the params object } /*definition of member functions of SNIConfig*/ diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 8830d7054ec..335ae53a601 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -30,6 +30,7 @@ #include "SSLSessionCache.h" #include "InkAPIInternal.h" #include "SSLDynlock.h" +#include "P_SSLSNI.h" #include #include @@ -45,7 +46,6 @@ #include #include #include -#include "P_SNIActionPerformer.h" #if HAVE_OPENSSL_EVP_H #include @@ -88,7 +88,6 @@ #endif #endif -TunnelHashMap TunnelMap; // stores the name of the servers to tunnel to /* * struct ssl_user_config: gather user provided settings from ssl_multicert.config in to this single struct * ssl_ticket_enabled - session ticket enabled @@ -446,10 +445,27 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) return retval; } +static int +PerformAction(Continuation *cont, const char *servername) +{ + SNIConfig::scoped_config params; + const actionVector *actionvec = params->get(servername); + if (!actionvec) { + Debug("ssl_sni", "%s not available in the map", servername); + } else { + for (auto &&item : *actionvec) { + auto ret = item->SNIAction(cont); + if (ret != SSL_TLSEXT_ERR_OK) { + return ret; + } + } + } + return SSL_TLSEXT_ERR_OK; +} + /* * Cannot stop this callback. Always reeneabled */ -extern SNIActionPerformer sni_action_performer; static int ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) { @@ -458,7 +474,7 @@ ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); Debug("ssl", "Requested servername is %s", servername); if (servername != nullptr) { - ret = sni_action_performer.PerformAction(netvc, servername); + ret = PerformAction(netvc, servername); } if (ret != SSL_TLSEXT_ERR_OK) return SSL_TLSEXT_ERR_ALERT_FATAL; @@ -1579,6 +1595,10 @@ setClientCertLevel(SSL *ssl, uint8_t certLevel) server_verify_client = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT | SSL_VERIFY_CLIENT_ONCE; } else if (certLevel == 1) { server_verify_client = SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE; + } else if (certLevel == 0) { + server_verify_client = SSL_VERIFY_NONE; + } else { + ink_release_assert(!"Invalid client verify level"); } Debug("ssl", "setting cert level to %d", server_verify_client); diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h index 444ae6b7968..e36fe9199e6 100644 --- a/iocore/net/YamlSNIConfig.h +++ b/iocore/net/YamlSNIConfig.h @@ -58,7 +58,7 @@ struct YamlSNIConfig { struct Item { std::string fqdn; bool disable_h2 = false; - uint8_t verify_client_level = 0; + uint8_t verify_client_level = 255; std::string tunnel_destination; Policy verify_server_policy = Policy::DISABLED; Property verify_server_properties = Property::NONE; diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index dff7e283a16..41fd5a1503b 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1180,8 +1180,6 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.handshake_timeout_in", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-65535]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.sni.map.enable", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} - , {RECT_CONFIG, "proxy.config.ssl.wire_trace_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.wire_trace_addr", RECD_STRING, nullptr , RECU_DYNAMIC, RR_NULL, RECC_IP, R"([0-255]\.[0-255]\.[0-255]\.[0-255])", RECA_NULL} diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 58cf9d5a50f..52d4dde2390 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -70,7 +70,6 @@ #define USE_NEW_EMPTY_MIOBUFFER extern int cache_config_read_while_writer; -extern TunnelHashMap TunnelMap; // stores the name of the servers to tunnel to // We have a debugging list that can use to find stuck // state machines @@ -565,7 +564,7 @@ HttpSM::setup_blind_tunnel_port() NetVConnection *netvc = ua_txn->get_netvc(); SSLNetVConnection *ssl_vc = dynamic_cast(netvc); int host_len; - if (ssl_vc && ssl_vc->GetSNIMapping()) { + if (ssl_vc) { if (!t_state.hdr_info.client_request.url_get()->host_get(&host_len)) { // the URL object has not been created in the start of the transaction. Hence, we need to create the URL here URL u; @@ -575,10 +574,11 @@ HttpSM::setup_blind_tunnel_port() t_state.hdr_info.client_request.url_create(&u); u.scheme_set(URL_SCHEME_TUNNEL, URL_LEN_TUNNEL); t_state.hdr_info.client_request.url_set(&u); - if (auto it = TunnelMap.find(ssl_vc->serverName); it != TunnelMap.end()) { - t_state.hdr_info.client_request.url_get()->host_set(it->second.hostname.c_str(), it->second.hostname.size()); - if (it->second.port > 0) { - t_state.hdr_info.client_request.url_get()->port_set(it->second.port); + if (ssl_vc->has_tunnel_destination()) { + const char *tunnel_host = ssl_vc->get_tunnel_host(); + t_state.hdr_info.client_request.url_get()->host_set(tunnel_host, strlen(tunnel_host)); + if (ssl_vc->get_tunnel_port() > 0) { + t_state.hdr_info.client_request.url_get()->port_set(ssl_vc->get_tunnel_port()); } else { t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } @@ -1384,18 +1384,18 @@ plugins required to work with sni_routing. NetVConnection *netvc = ua_txn->get_netvc(); SSLNetVConnection *ssl_vc = dynamic_cast(netvc); - if (ssl_vc && ssl_vc->GetSNIMapping()) { - if (auto it = TunnelMap.find(ssl_vc->serverName); it != TunnelMap.end()) { - t_state.hdr_info.client_request.url_get()->host_set(it->second.hostname.c_str(), it->second.hostname.size()); - if (it->second.port > 0) { - t_state.hdr_info.client_request.url_get()->port_set(it->second.port); - } else { - t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); - } + if (ssl_vc && ssl_vc->has_tunnel_destination()) { + const char *tunnel_host = ssl_vc->get_tunnel_host(); + t_state.hdr_info.client_request.url_get()->host_set(tunnel_host, strlen(tunnel_host)); + ushort tunnel_port = ssl_vc->get_tunnel_port(); + if (tunnel_port > 0) { + t_state.hdr_info.client_request.url_get()->port_set(tunnel_port); } else { - t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } + } else if (ssl_vc) { + t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); + t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } // FALLTHROUGH diff --git a/tests/gold_tests/h2/h2disable.test.py b/tests/gold_tests/h2/h2disable.test.py index 1e64b92c00d..ebeb68812ff 100644 --- a/tests/gold_tests/h2/h2disable.test.py +++ b/tests/gold_tests/h2/h2disable.test.py @@ -63,8 +63,8 @@ ts.Disk.ssl_server_name_yaml.AddLines([ '- fqdn: bar.com', ' disable_h2: true', -# '- fqdn: bob.*.com', -# ' disable_h2: true', + '- fqdn: bob.*.com', + ' disable_h2: true', ]) tr = Test.AddTestRun("Negotiate-h2") @@ -89,14 +89,14 @@ tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Using HTTP2", "Curl should not negotiate HTTP2") tr2.TimeOut = 5 -#tr2 = Test.AddTestRun("Do not negotiate h2") -#tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format(ts.Variables.ssl_port) -#tr2.ReturnCode = 0 -#tr2.StillRunningAfter = server -#tr2.Processes.Default.TimeOut = 5 -#tr2.StillRunningAfter = ts -#tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -#tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Using HTTP2", "Curl should not negotiate HTTP2") -#tr2.TimeOut = 5 +tr2 = Test.AddTestRun("Do not negotiate h2") +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Using HTTP2", "Curl should not negotiate HTTP2") +tr2.TimeOut = 5 diff --git a/tests/gold_tests/tls/ssl/signed-bob-bar.pem b/tests/gold_tests/tls/ssl/signed-bob-bar.pem new file mode 100644 index 00000000000..cca4c2fe93c --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-bob-bar.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC2jCCAkMCCQC81MtBCwmQuTANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTgxMTIwMTQ1NDEyWhcNMjgxMTE3MTQ1NDEyWjBBMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxDzANBgNVBAoMBkFwYWNoZTEUMBIGA1UEAwwLYm9iLmJh +ci5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDuzA6e4MZp98m/ +XuhRAZMCaq9fM5zRBDsY5fpx3zUkJwJtZa4ALk6Adusovfh3t6qYJJ47TItnNG9Z +nAvYQhOccZNkj8xbPytuj5pQzu3CAR/IsBBWtG8O0SLRHNVOmgu3bdXuJca+3LKR +X3qFJbReqd9OI9vGxrxKg9NPUeGpzAuUTplmfs8O70g8l0ZBJ7l/0VmsA5lUDZxV +NjcacVYHc+ApujuLz5drVRKeLxTrtc6DksP7uruT5ZVrWbL35hoviwUilugz9ZDI +SQl0T6cIus8CyUjs8/iJEln9AnxAZ8Muz5+pQ3G/4BsabKlEHf0+GYinXxiTMT1V +Cr9botZ/AgMBAAEwDQYJKoZIhvcNAQELBQADgYEAnPW5NsSCI95ETnx9b1ZJv8wo +EIWqxzD7Wm9CIL8gUmVGrEJpHrsfpuXEN/jKqVFxUr3d2ZI40P8BrO73BGHo2F0M +ISDxiRSs7tR5xl5WmmErzK8TpDHU9fhkCqlblk7lzPLalIXCQ0y8qDjACBbh3tfj +0grbVmTy66u6TxI5FTE= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/signed-bob-foo.pem b/tests/gold_tests/tls/ssl/signed-bob-foo.pem new file mode 100644 index 00000000000..05db399165d --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-bob-foo.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC2jCCAkMCCQC81MtBCwmQujANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTgxMTIwMTQ1NTA4WhcNMjgxMTE3MTQ1NTA4WjBBMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxDzANBgNVBAoMBkFwYWNoZTEUMBIGA1UEAwwLYm9iLmZv +by5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDJheOsr0o4C0QV +elo8viYl/IjWvkj4Y6YojnO/E9T5HSfWxkRikRT8X7yqs9ClrjNcYoPNNQP5vgl1 +qAIeixUrbZZklcZ3kHF/MXdzuYV//uT3YfWrsOTdfbcu6adHtBr/Qc0w9GSbBpp4 +2ZALxzjZ2hZZq51XmKvcz55/gm/9YvlGErzvsPzkejdH6U1GKeLqGaZOqkV7N2J4 +zKmJyiPzwrmR/8y7k+9jDHQX+A2wa5gaiAMGdIM8aTV+XsNNAvespvXyX7UhwxDs +w0er0GLv4ssYCru5hF+tUcjeOCHxSnCoEC45b2zDZHATLLjr7AIJ52TcSo1kTaTi +rAOAiHRLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEAvrrtNDsSbjVhwOlAm7MFwbxp +EUh1jYk07C6vxz7dFAcEXh33Sx4LsD0sI2TSbV5swGtPqMVTxbA+Ok6S/2lfxUeT +h7B6GrWvHtzQdPZbkoTEyehH5fOFLchbwGbecugYLjAK9wtatDZhFnuKRq8aiqml +GuDJPQLfKB9APLyxIbA= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py new file mode 100644 index 00000000000..0c085fd042a --- /dev/null +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -0,0 +1,169 @@ +''' +Test offering client cert to origin +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test client certs to origin selected via wildcard names in ssl_server_name +''' + +Test.SkipUnless(Condition.HasProgram("grep", "grep needs to be installed on system for this test to work")) + +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=False) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": "true"}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": "true"}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server.Setup.Copy("ssl/signer.pem") +server.Setup.Copy("ssl/signer2.pem") +server.Setup.Copy("ssl/signed-foo.pem") +server.Setup.Copy("ssl/signed-foo.key") +server.Setup.Copy("ssl/signed2-foo.pem") +server.Setup.Copy("ssl/signed2-bar.pem") +server.Setup.Copy("ssl/signed-bar.key") +server2.Setup.Copy("ssl/signer.pem") +server2.Setup.Copy("ssl/signer2.pem") +server2.Setup.Copy("ssl/signed-foo.pem") +server2.Setup.Copy("ssl/signed-foo.key") +server2.Setup.Copy("ssl/signed2-foo.pem") +server2.Setup.Copy("ssl/signed2-bar.pem") +server2.Setup.Copy("ssl/signed-bar.key") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed2-foo.pem") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed2-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_verify_test', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr' : 1, +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map /case1 https://127.0.0.1:{0}/'.format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + 'map /case2 https://127.0.0.1:{0}/'.format(server2.Variables.Port) +) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bob.bar.com', + ' client_cert: {0}/signed-bar.pem'.format(ts.Variables.SSLDir), + ' client_key: {0}/signed-bar.key'.format(ts.Variables.SSLDir), + '- fqdn: bob.*.com', + ' client_cert: {0}/signed-foo.pem'.format(ts.Variables.SSLDir), + ' client_key: {0}/signed-foo.key'.format(ts.Variables.SSLDir), + '- fqdn: "*bar.com"', + ' client_cert: {0}/signed2-bar.pem'.format(ts.Variables.SSLDir), + ' client_key: {0}/signed-bar.key'.format(ts.Variables.SSLDir), +]) + + +# Should succeed +tr = Test.AddTestRun("bob.bar.com to server 1") +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(server2) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:bob.bar.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") +tr.TimeOut = 5 + +#Should fail +trfail = Test.AddTestRun("bob.bar.com to server 2") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:bob.bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") +trfail.TimeOut = 5 + +# Should succeed +tr = Test.AddTestRun("bob.foo.com to server 1") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:bob.foo.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") +tr.TimeOut = 5 + +#Should fail +trfail = Test.AddTestRun("bob.foo.com to server 2") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:bob.foo.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") +trfail.TimeOut = 5 + +# Should succeed +tr = Test.AddTestRun("random.bar.com to server 2") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:random.bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") +tr.TimeOut = 5 + +#Should fail +trfail = Test.AddTestRun("random.bar.com to server 1") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:random.bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") +trfail.TimeOut = 5 + diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py index d6298531370..f0abd7ecc9f 100644 --- a/tests/gold_tests/tls/tls_client_verify.test.py +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -68,8 +68,12 @@ # Scenario 1: Default no client cert required. cert required for bar.com ts.Disk.ssl_server_name_yaml.AddLines([ - '- fqdn: bar.com', + '- fqdn: bob.bar.com', ' verify_client: NONE', + '- fqdn: bob.*.com', + ' verify_client: NONE', + '- fqdn: "*bar.com"', + ' verify_client: STRICT', ]) # to foo.com w/o client cert. Should fail @@ -107,15 +111,77 @@ tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr.TimeOut = 5 -tr = Test.AddTestRun("Connect to bar.com without cert") +tr = Test.AddTestRun("Connect to bob.bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bob.bar.com with cert") +tr.Setup.Copy("ssl/signed-bob-bar.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-bar.pem --key signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bob.bar.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bob.foo.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 5 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") tr.TimeOut = 5 +tr = Test.AddTestRun("Connect to bob.foo.com with cert") +tr.Setup.Copy("ssl/signed-bob-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-foo.pem --key signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bob.foo.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bar.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert handshake failure", "TLS handshake should fail") +tr.TimeOut = 5 + tr = Test.AddTestRun("Connect to bar.com with cert") tr.Setup.Copy("ssl/signed-bar.pem") tr.Setup.Copy("ssl/signed-bar.key") @@ -124,7 +190,7 @@ tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 5 -tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bar.com with bad cert") @@ -133,9 +199,9 @@ tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) -# Should fail with badly signed certs -tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.ReturnCode = 35 tr.Processes.Default.TimeOut = 5 -tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") +tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") tr.TimeOut = 5 + diff --git a/tests/gold_tests/tls/tls_client_verify2.test.py b/tests/gold_tests/tls/tls_client_verify2.test.py index 8848dc24acd..a435e944884 100644 --- a/tests/gold_tests/tls/tls_client_verify2.test.py +++ b/tests/gold_tests/tls/tls_client_verify2.test.py @@ -68,8 +68,12 @@ # Scenario 1: Default no client cert required. cert required for bar.com ts.Disk.ssl_server_name_yaml.AddLines([ - '- fqdn: bar.com', + '- fqdn: bob.bar.com', ' verify_client: STRICT', + '- fqdn: bob.*.com', + ' verify_client: STRICT', + '- fqdn: "*bar.com"', + ' verify_client: NONE', ]) # to foo.com w/o client cert. Should succeed @@ -95,15 +99,77 @@ tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr.TimeOut = 5 -tr = Test.AddTestRun("Connect to bar.com without cert") +tr = Test.AddTestRun("Connect to bob.bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bob.bar.com with cert") +tr.Setup.Copy("ssl/signed-bob-bar.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-bar.pem --key signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bob.bar.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("error", "TLS handshake should fail") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bob.foo.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 tr.Processes.Default.TimeOut = 5 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") tr.TimeOut = 5 +tr = Test.AddTestRun("Connect to bob.foo.com with cert") +tr.Setup.Copy("ssl/signed-bob-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-foo.pem --key signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bob.foo.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("error", "TLS handshake should fail") +tr.TimeOut = 5 + +tr = Test.AddTestRun("Connect to bar.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") +tr.TimeOut = 5 + tr = Test.AddTestRun("Connect to bar.com with cert") tr.Setup.Copy("ssl/signed-bar.pem") tr.Setup.Copy("ssl/signed-bar.key") @@ -121,9 +187,8 @@ tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) -# Should fail with badly signed certs -tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 5 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert unknown ca", "TLS handshake should succeed") tr.TimeOut = 5 diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index 390a11e6bb9..e92c4a168ed 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -76,9 +76,12 @@ # bar.com should terminate. Forward its tcp stream to server_bar ts.Disk.ssl_server_name_yaml.AddLines([ '- fqdn: foo.com', - " tunnel_route: localhost:{0}".format(server_foo.Variables.Port) ]) + " tunnel_route: localhost:{0}".format(server_foo.Variables.Port), + "- fqdn: bob.*.com", + " tunnel_route: localhost:{0}".format(server_foo.Variables.Port), +]) -tr = Test.AddTestRun("Tunnel-test") +tr = Test.AddTestRun("foo.com Tunnel-test") tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.Processes.Default.StartBefore(server_foo) @@ -91,5 +94,29 @@ tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") + +tr = Test.AddTestRun("bob.bar.com Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'bob.bar.com:{0}:127.0.0.1' -k https://bob.bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") + +tr = Test.AddTestRun("bar.com no Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") + diff --git a/tests/gold_tests/tls/tls_verify.test.py b/tests/gold_tests/tls/tls_verify.test.py index d95ec160143..81164e83e97 100644 --- a/tests/gold_tests/tls/tls_verify.test.py +++ b/tests/gold_tests/tls/tls_verify.test.py @@ -86,18 +86,14 @@ 'proxy.config.url_remap.pristine_host_hdr': 1 }) -ts.Disk.ssl_server_name_yaml.AddLine( - '- fqdn: bar.com') -ts.Disk.ssl_server_name_yaml.AddLine( - ' verify_server_policy: ENFORCED') -ts.Disk.ssl_server_name_yaml.AddLine( - ' verify_server_properties: ALL') -ts.Disk.ssl_server_name_yaml.AddLine( - '- fqdn: bad_bar.com') -ts.Disk.ssl_server_name_yaml.AddLine( - ' verify_server_policy: ENFORCED') -ts.Disk.ssl_server_name_yaml.AddLine( - ' verify_server_properties: ALL') +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bad_bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL' +]) tr = Test.AddTestRun("Permissive-Test") tr.Setup.Copy("ssl/signed-foo.key") diff --git a/tests/gold_tests/tls/tls_verify3.test.py b/tests/gold_tests/tls/tls_verify3.test.py new file mode 100644 index 00000000000..222222d9bf2 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify3.test.py @@ -0,0 +1,147 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: badfoo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: badbar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map https://foo.com:{1}/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port, ts.Variables.ssl_port)) +ts.Disk.remap_config.AddLine( + 'map https://bob.foo.com:{1}/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port, ts.Variables.ssl_port)) +ts.Disk.remap_config.AddLine( + 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port, ts.Variables.ssl_port)) +ts.Disk.remap_config.AddLine( + 'map https://bob.bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port,ts.Variables.ssl_port)) +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl|http|url', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.ssl.client.verify.server.properties': 'ALL', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bob.bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bob.*.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: SIGNATURE', + "- fqdn: '*bar.com'", + ' verify_server_policy: DISABLED', +]) + +tr = Test.AddTestRun("foo.com Permissive-Test") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.TimeOut = 5 + +tr2 = Test.AddTestRun("bob.bar.com Override-enforcing-Test") +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.TimeOut = 5 + +tr3 = Test.AddTestRun("bob.foo.com override-enforcing-name-test") +tr3.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/".format(ts.Variables.ssl_port) +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should not fail") +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +tr3.Processes.Default.TimeOut = 5 +tr3.TimeOut = 5 + +tr3 = Test.AddTestRun("random.bar.com override-no-test") +tr3.Processes.Default.Command = "curl -v -k --resolve 'random.bar.com:{0}:127.0.0.1' https://random.bar.com:{0}".format(ts.Variables.ssl_port) +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should not fail") +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +tr3.Processes.Default.TimeOut = 5 +tr3.TimeOut = 5 + + +# Over riding the built in ERROR check since we expect tr3 to fail +ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bob.bar.com\) not in certificate", "Make sure bob.bar name checked failed.") From c09657323bd872d027d8684ab0be1f68562a5462 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 27 Nov 2018 18:36:21 +0000 Subject: [PATCH 051/526] Remove ssl.enable flag --- mgmt/RecordsConfig.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 41fd5a1503b..5799a0152c7 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1085,8 +1085,6 @@ static const RecordElement RecordsConfig[] = //# SSL Termination //# //############################################################################## - {RECT_CONFIG, "proxy.config.ssl.enabled", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} - , {RECT_CONFIG, "proxy.config.ssl.server.session_ticket.enable", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.TLSv1", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} From 4743718007717b73e9c1f6fa1a52fe266975b590 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 27 Nov 2018 17:08:01 -0600 Subject: [PATCH 052/526] Doc: Document TSHttpTxnServerRespGet. --- .../functions/TSHttpTxnServerRespGet.en.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst index 9150f1d5092..3113297b871 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst @@ -30,3 +30,25 @@ Synopsis Description =========== + +Get the response header sent by the upstream server. This will only be useful in a callback on a +hook that is called after the upstream responds, and if there was an upstream response. For +instance, if the inbound request has no remap rule and :ts:cv:`remap is required +` then there will be no server response because no outbound +connection was made. In this case the function will return :c:macro:`TS_ERROR`. + +The response header is returned in :arg:`bufp` and :arg:`offset`. :arg:`bufp` is the heap in which +the header resides, and :arg:`offset` is the location in that heap. These will be used in subsequent +calls to retrieve information from the header. + +.. code-block:: cpp + :emphasize-lines: 4 + + int get_response_status(TSHttpTxn txn) { + TSMBuffer resp_heap = nullptr; + TSMLoc resp_hdr = nullptr; + if (TS_SUCCESS == TSHttpTxnServerRespGet(tnx, &resp_heap, &resp_hdr)) { + return TSHttpHdrStatusGet(resp_headp, resp_hdr); + } + return HTTP_STATUS_NONE; + } From f3acdc9c7b6d593569d8285916f68c5a6be3cd8d Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 11 Oct 2018 20:47:58 -0700 Subject: [PATCH 053/526] Removes the old expander feature of header_rewrite The new string concatenations are just better, and faster. --- doc/admin-guide/plugins/header_rewrite.en.rst | 82 +++------ plugins/header_rewrite/Makefile.inc | 2 - plugins/header_rewrite/conditions.cc | 25 --- plugins/header_rewrite/conditions.h | 17 -- plugins/header_rewrite/expander.cc | 155 ------------------ plugins/header_rewrite/expander.h | 37 ----- plugins/header_rewrite/header_rewrite_test.cc | 15 +- plugins/header_rewrite/operators.cc | 30 ---- plugins/header_rewrite/value.cc | 27 +-- plugins/header_rewrite/value.h | 7 - .../headers/domain-blacklist-30x.test.py | 2 +- 11 files changed, 41 insertions(+), 358 deletions(-) delete mode 100644 plugins/header_rewrite/expander.cc delete mode 100644 plugins/header_rewrite/expander.h diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index 9b38438b804..edb39359735 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -608,7 +608,8 @@ be specified multiple times, such as ``Set-Cookie``, but for headers which may only be specified once you may prefer to use `set-header`_ instead. The header's ```` may be specified as a literal string, or it may take -advantage of :ref:`header-rewrite-expansion` to calculate a dynamic value for the header. +advantage of :ref:`header-rewrite-concatenations` to calculate a dynamic value +for the header. counter ~~~~~~~ @@ -721,7 +722,8 @@ Replaces the value of header ```` with ````, creating the header if necessary. The header's ```` may be specified according to `Header Values`_ or take -advantage of :ref:`header-rewrite-expansion` to calculate a dynamic value for the header. +advantage of :ref:`header-rewrite-concatenations` to calculate a dynamic value +for the header. set-redirect ~~~~~~~~~~~~ @@ -732,8 +734,8 @@ set-redirect When invoked, sends a redirect response to the client, with HTTP status ````, and a new location of ````. If the ``QSA`` flag is enabled, the original query string will be preserved and added to the new -location automatically. This operator supports :ref:`header-rewrite-expansion` for -````. +location automatically. This operator supports +:ref:`header-rewrite-concatenations` for ````. set-status ~~~~~~~~~~ @@ -804,66 +806,30 @@ L Last rule, do not continue. QSA Append the results of the rule to the query string. ====== ======================================================================== -.. _header-rewrite-expansion: +.. _header-rewrite-concatenations: -Values and Variable Expansion ------------------------------ - -.. note:: - - This feature is replaced with a new string concatenations as of ATS v8.1.0. In v9.0.0 the special - %<> string expansions below are no longer available, instead use the following mapping: - -======================= ================================================================================== -Variable New condition variable to use -======================= ================================================================================== -% %{CLIENT-URL:SCHEME} -% %{CLIENT-URL:PORT} -% %{IP:CLIENT}, %{INBOUND:REMOTE-ADDR} or e.g. %{CIDR:24,48} -% %{CLIENT-HEADER:Content-Length} -% %{METHOD} -% %[CLIENT-URL} -% %{CLIENT-URL:PATH} -======================= ================================================================================== - -The % tags can now be replaced with the %{INBOUND:...} equivalent. +String concatenations +--------------------- You can concatenate values using strings, condition values and variable expansions on the same line. - add-header CustomHeader "Hello from %{IP:SERVER}:%" - -However, the above example is somewhat contrived to show the old tags, it should instead be written as - add-header CustomHeader "Hello from %{IP:SERVER}:%{INBOUND:LOCAL-PORT}" - -Concatenation is not supported in condition testing. - -Supported substitutions are currently the following table, however they are deprecated and you should -instead use the equivalent %{} conditions as shown above: - -======================= ================================================================================== -Variable Description -======================= ================================================================================== -% (Deprecated) Protocol -% (Deprecated) Port -% (Deprecated) Client IP -% (Deprecated) Client request length -% (Deprecated) Client HTTP method -% (Deprecated) Client effective URI -% (Deprecated) Client unmapped URI path -% (Deprecated) The local (ATS) address for the inbound connection. -% (Deprecated) The local (ATS) port for the inbound connection. -% (Deprecated) The client address for the inbound connectoin. -% (Deprecated) The client port for the inbound connectoin. -% (Deprecated) The TLS protocol for the inbound connection if it is over TLS, otherwise the - empty string. -% (Deprecated) The string "h2" if the inbound connection is HTTP/2, otherwise the empty string. -% (Deprecated) The string "ipv4" if the inbound connection is IPv4, otherwise the emtpy string. -% (Deprecated) The string "ipv6" if the inbound connection is IPv6, otherwise the empty string. -% (Deprecated) The IP family of the inbound connection (either "ipv4" or "ipv6"). -% (Deprecated) The full protocol stack of the inbound connection separated by ','. -======================= ================================================================================== +String concatenation is not yet supported in condition testing. + +Note: In versions prior to ATS v9.0.0, an alternative string expansion was available. those +expansions are no longer available, but the following table can help migrations: + +======================== ================================================================================== +Old expansion variable Condition variable to use with concatenatinos +======================== ================================================================================== +% %{CLIENT-URL:SCHEME} +% %{CLIENT-URL:PORT} +% %{IP:CLIENT}, %{INBOUND:REMOTE-ADDR} or e.g. %{CIDR:24,48} +% %{CLIENT-HEADER:Content-Length} +% %{METHOD} +% %[CLIENT-URL} +% %{CLIENT-URL:PATH} Header Values ------------- diff --git a/plugins/header_rewrite/Makefile.inc b/plugins/header_rewrite/Makefile.inc index c4b81d6af9b..0b5cfe8bb62 100644 --- a/plugins/header_rewrite/Makefile.inc +++ b/plugins/header_rewrite/Makefile.inc @@ -23,8 +23,6 @@ header_rewrite_header_rewrite_la_SOURCES = \ header_rewrite/condition.h \ header_rewrite/conditions.cc \ header_rewrite/conditions.h \ - header_rewrite/expander.cc \ - header_rewrite/expander.h \ header_rewrite/factory.cc \ header_rewrite/factory.h \ header_rewrite/header_rewrite.cc \ diff --git a/plugins/header_rewrite/conditions.cc b/plugins/header_rewrite/conditions.cc index 1ee6ea1bdba..2920172e76f 100644 --- a/plugins/header_rewrite/conditions.cc +++ b/plugins/header_rewrite/conditions.cc @@ -30,7 +30,6 @@ #include "ts/ts.h" #include "conditions.h" -#include "expander.h" #include "lulu.h" // ConditionStatus @@ -1394,27 +1393,3 @@ ConditionStringLiteral::eval(const Resources &res) return static_cast(_matcher)->test(_literal); } - -ConditionExpandableString::ConditionExpandableString(const std::string &v) -{ - TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionExpandableString"); - _value = v; -} - -bool -ConditionExpandableString::eval(const Resources &res) -{ - std::string s; - - append_value(s, res); - - return static_cast(_matcher)->test(s); -} - -void -ConditionExpandableString::append_value(std::string &s, const Resources &res) -{ - VariableExpander ve(_value); - s += ve.expand(res); - TSDebug(PLUGIN_NAME, "Appending to evaluation value -> %s", s.c_str()); -} diff --git a/plugins/header_rewrite/conditions.h b/plugins/header_rewrite/conditions.h index 791bb41e132..6c22c7f5a04 100644 --- a/plugins/header_rewrite/conditions.h +++ b/plugins/header_rewrite/conditions.h @@ -552,20 +552,3 @@ class ConditionStringLiteral : public Condition std::string _literal; DISALLOW_COPY_AND_ASSIGN(ConditionStringLiteral); }; - -class ConditionExpandableString : public Condition -{ - typedef Matchers MatcherType; - -public: - explicit ConditionExpandableString(const std::string &v); - - void append_value(std::string &s, const Resources &res) override; - -protected: - bool eval(const Resources &res) override; - -private: - std::string _value; - DISALLOW_COPY_AND_ASSIGN(ConditionExpandableString); -}; diff --git a/plugins/header_rewrite/expander.cc b/plugins/header_rewrite/expander.cc deleted file mode 100644 index dd3d1ab397c..00000000000 --- a/plugins/header_rewrite/expander.cc +++ /dev/null @@ -1,155 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -////////////////////////////////////////////////////////////////////////////////////////////// -// expander.cc: Implementation of the Variable Expander base class -// -#include "ts/ts.h" - -#include -#include - -#include "lulu.h" -#include "statement.h" -#include "parser.h" -#include "expander.h" -#include "conditions.h" - -// Main expander method -std::string -VariableExpander::expand(const Resources &res) -{ - std::string result; - - result.reserve(512); // TODO: Can be optimized - result.assign(_source); - - while (true) { - std::string::size_type start = result.find("%<"); - if (start == std::string::npos) { - break; - } - - std::string::size_type end = result.find('>', start); - if (end == std::string::npos) { - break; - } - - std::string first_part = result.substr(0, start); - std::string last_part = result.substr(end + 1); - - // Now evaluate the variable - std::string variable = result.substr(start, end - start + 1); - - // This will be the value to replace the "variable" section of the string with - std::string resolved_variable = ""; - - // Initialize some stuff - TSMBuffer bufp; - TSMLoc hdr_loc; - TSMLoc url_loc; - - if (variable == "%") { - // Protocol of the incoming request - if (TSHttpTxnPristineUrlGet(res.txnp, &bufp, &url_loc) == TS_SUCCESS) { - int len; - const char *tmp = TSUrlSchemeGet(bufp, url_loc, &len); - if ((tmp != nullptr) && (len > 0)) { - resolved_variable.assign(tmp, len); - } else { - resolved_variable.assign(""); - } - TSHandleMLocRelease(bufp, TS_NULL_MLOC, url_loc); - } - } else if (variable == "%") { - // Original port of the incoming request - if (TSHttpTxnClientReqGet(res.txnp, &bufp, &hdr_loc) == TS_SUCCESS) { - if (TSHttpHdrUrlGet(bufp, hdr_loc, &url_loc) == TS_SUCCESS) { - std::stringstream out; - out << TSUrlPortGet(bufp, url_loc); - resolved_variable = out.str(); - TSHandleMLocRelease(bufp, hdr_loc, url_loc); - } - TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); - } - } else if (variable == "%") { - // IP address of the client's host machine - resolved_variable = getIP(TSHttpTxnClientAddrGet(res.txnp)); - } else if (variable == "%") { - // The client request header length; the header length in the client request to Traffic Server. - std::stringstream out; - out << TSHttpHdrLengthGet(res.client_bufp, res.client_hdr_loc); - resolved_variable = out.str(); - } else if (variable == "%") { - // The HTTP method in the client request to Traffic Server: GET, POST, and so on (subset of cqtx). - int method_len; - const char *methodp = TSHttpHdrMethodGet(res.client_bufp, res.client_hdr_loc, &method_len); - if (methodp && method_len) { - resolved_variable.assign(methodp, method_len); - } - } else if (variable == "%") { - // The client request unmapped URL path. This field records a URL path - // before it is remapped (reverse proxy mode). - if (TSHttpTxnPristineUrlGet(res.txnp, &bufp, &url_loc) == TS_SUCCESS) { - int path_len; - const char *path = TSUrlPathGet(bufp, url_loc, &path_len); - - if (path && path_len) { - resolved_variable.assign(path, path_len); - } - TSHandleMLocRelease(bufp, TS_NULL_MLOC, url_loc); - } - } else if (variable == "%") { - // The client request effective URL. - int url_len = 0; - char *url = TSHttpTxnEffectiveUrlStringGet(res.txnp, &url_len); - if (url && url_len) { - resolved_variable.assign(url, url_len); - } - free(url); - url = nullptr; - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_REMOTE_ADDR); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_REMOTE_PORT); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_LOCAL_ADDR); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_LOCAL_PORT); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_TLS); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_H2); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_IPV4); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_IPV6); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_IP_FAMILY); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_STACK); - } - - // TODO(SaveTheRbtz): Can be optimized - result.assign(first_part); - result.append(resolved_variable); - result.append(last_part); - } - - return result; -} diff --git a/plugins/header_rewrite/expander.h b/plugins/header_rewrite/expander.h deleted file mode 100644 index 61553c00b58..00000000000 --- a/plugins/header_rewrite/expander.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -////////////////////////////////////////////////////////////////////////////////////////////// -// Public interface for the Variable Expander -// -#pragma once - -#include - -#include "ts/ts.h" -#include "resources.h" - -class VariableExpander -{ -public: - explicit VariableExpander(const std::string &source) : _source(source) {} - std::string expand(const Resources &res); - -private: - std::string _source; -}; diff --git a/plugins/header_rewrite/header_rewrite_test.cc b/plugins/header_rewrite/header_rewrite_test.cc index 6a4e18172f2..437b76c849d 100644 --- a/plugins/header_rewrite/header_rewrite_test.cc +++ b/plugins/header_rewrite/header_rewrite_test.cc @@ -330,17 +330,6 @@ test_parsing() END_TEST(); } - { - ParserTest p(R"(add-header Client-IP "%")"); - - CHECK_EQ(p.getTokens().size(), 3UL); - CHECK_EQ(p.getTokens()[0], "add-header"); - CHECK_EQ(p.getTokens()[1], "Client-IP"); - CHECK_EQ(p.getTokens()[2], R"(%)"); - - END_TEST(); - } - { ParserTest p(R"(add-header X-Url "http://trafficserver.apache.org/")"); @@ -487,10 +476,10 @@ test_tokenizer() CHECK_EQ(p.get_tokens()[1], "%{NOW:YEAR}"); } { - SimpleTokenizerTest p("A racoon's favorite tag is % in %{NOW:YEAR}!"); + SimpleTokenizerTest p("A racoon's favorite tag is %{METHOD} in %{NOW:YEAR}!"); CHECK_EQ(p.get_tokens().size(), 5UL); CHECK_EQ(p.get_tokens()[0], "A racoon's favorite tag is "); - CHECK_EQ(p.get_tokens()[1], "%"); + CHECK_EQ(p.get_tokens()[1], "%{METHOD}c"); CHECK_EQ(p.get_tokens()[2], " in "); CHECK_EQ(p.get_tokens()[3], "%{NOW:YEAR}"); CHECK_EQ(p.get_tokens()[4], "!"); diff --git a/plugins/header_rewrite/operators.cc b/plugins/header_rewrite/operators.cc index 3b57f7f2158..c2050116b40 100644 --- a/plugins/header_rewrite/operators.cc +++ b/plugins/header_rewrite/operators.cc @@ -25,7 +25,6 @@ #include "ts/ts.h" #include "operators.h" -#include "expander.h" #include "ts/apidefs.h" // OperatorConfig @@ -355,11 +354,6 @@ OperatorSetRedirect::exec(const Resources &res) const _location.append_value(value, res); - if (_location.need_expansion()) { - VariableExpander ve(value); - value = ve.expand(res); - } - bool remap = false; if (nullptr != res._rri) { remap = true; @@ -543,12 +537,6 @@ OperatorAddHeader::exec(const Resources &res) const _value.append_value(value, res); - if (_value.need_expansion()) { - VariableExpander ve(value); - - value = ve.expand(res); - } - // Never set an empty header (I don't think that ever makes sense?) if (value.empty()) { TSDebug(PLUGIN_NAME, "Would set header %s to an empty value, skipping", _header.c_str()); @@ -585,12 +573,6 @@ OperatorSetHeader::exec(const Resources &res) const _value.append_value(value, res); - if (_value.need_expansion()) { - VariableExpander ve(value); - - value = ve.expand(res); - } - // Never set an empty header (I don't think that ever makes sense?) if (value.empty()) { TSDebug(PLUGIN_NAME, "Would set header %s to an empty value, skipping", _header.c_str()); @@ -718,12 +700,6 @@ OperatorAddCookie::exec(const Resources &res) const _value.append_value(value, res); - if (_value.need_expansion()) { - VariableExpander ve(value); - - value = ve.expand(res); - } - if (res.bufp && res.hdr_loc) { TSDebug(PLUGIN_NAME, "OperatorAddCookie::exec() invoked on cookie %s", _cookie.c_str()); TSMLoc field_loc; @@ -769,12 +745,6 @@ OperatorSetCookie::exec(const Resources &res) const _value.append_value(value, res); - if (_value.need_expansion()) { - VariableExpander ve(value); - - value = ve.expand(res); - } - if (res.bufp && res.hdr_loc) { TSDebug(PLUGIN_NAME, "OperatorSetCookie::exec() invoked on cookie %s", _cookie.c_str()); TSMLoc field_loc; diff --git a/plugins/header_rewrite/value.cc b/plugins/header_rewrite/value.cc index bd664c63cee..4ddfcce3adc 100644 --- a/plugins/header_rewrite/value.cc +++ b/plugins/header_rewrite/value.cc @@ -43,28 +43,29 @@ Value::set_value(const std::string &val) { _value = val; - if (_value.find("%{") != std::string::npos || _value.find("%<") != std::string::npos) { + if (_value.find("%{") != std::string::npos) { SimpleTokenizer tokenizer(_value); - auto tokens = tokenizer.get_tokens(); - for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string token = *it; + for (auto it = tokens.begin(); it != tokens.end(); it++) { + std::string token = *it; Condition *tcond_val = nullptr; - if (token.substr(0, 2) == "%<") { - tcond_val = new ConditionExpandableString(*it); - } else if (token.substr(0, 2) == "%{") { + + if (token.substr(0, 2) == "%{") { std::string cond_token = token.substr(2, token.size() - 3); - tcond_val = condition_factory(cond_token); - } - if (tcond_val) { - Parser parser(_value); - tcond_val->initialize(parser); + if ((tcond_val = condition_factory(cond_token))) { + Parser parser(_value); + + tcond_val->initialize(parser); + } } else { tcond_val = new ConditionStringLiteral(token); } - _cond_vals.push_back(tcond_val); + + if (tcond_val) { + _cond_vals.push_back(tcond_val); + } } } else { _int_value = strtol(_value.c_str(), nullptr, 10); diff --git a/plugins/header_rewrite/value.h b/plugins/header_rewrite/value.h index 99859aaff5a..e15e67a8ec2 100644 --- a/plugins/header_rewrite/value.h +++ b/plugins/header_rewrite/value.h @@ -87,16 +87,9 @@ class Value : Statement return _value.empty(); } - bool - need_expansion() const - { - return _need_expander; - } - private: DISALLOW_COPY_AND_ASSIGN(Value); - bool _need_expander = false; int _int_value = 0; double _float_value = 0.0; std::string _value; diff --git a/tests/gold_tests/headers/domain-blacklist-30x.test.py b/tests/gold_tests/headers/domain-blacklist-30x.test.py index b8d90413f6d..19c972771c9 100644 --- a/tests/gold_tests/headers/domain-blacklist-30x.test.py +++ b/tests/gold_tests/headers/domain-blacklist-30x.test.py @@ -52,7 +52,7 @@ for x in (0, 301, 302, 307, 308): ts.Disk.MakeConfigFile("header_rewrite_rules_{0}.conf".format(x)).AddLine("""\ -set-redirect {0} "%" +set-redirect {0} "%{{CLIENT-URL}}" """.format(x)) Test.Setup.Copy(os.path.join(os.pardir, os.pardir, 'tools', 'tcp_client.py')) From 102ef72f0a5cb5a08d8052ec81de54c43e04b15b Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Tue, 6 Nov 2018 15:32:19 -0600 Subject: [PATCH 054/526] Include additional names of events for debug output. (cherry picked from commit 02f41846809acab503f5172ca1e5899bff1b2de8) --- lib/records/RecHttp.cc | 6 +- proxy/http/HttpDebugNames.cc | 261 ++++++++++++++---- src/traffic_server/InkAPITest.cc | 8 +- .../gold_tests/pluginTest/test_hooks/log.gold | 22 +- 4 files changed, 218 insertions(+), 79 deletions(-) diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index 85c7a4ae89b..f3da6f70d73 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -676,10 +676,8 @@ ts_session_protocol_well_known_name_indices_init() const char * RecNormalizeProtoTag(const char *tag) { - if (TSProtoTags.find(tag) != TSProtoTags.end()) { - return tag; - } - return nullptr; + auto findResult = TSProtoTags.find(tag); + return findResult == TSProtoTags.end() ? nullptr : findResult->data(); } SessionProtocolNameRegistry::SessionProtocolNameRegistry() : m_n(0) diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index a67916710ad..59beb0157c9 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -28,6 +28,8 @@ #include "Transform.h" #include "HttpSM.h" #include "HttpUpdateSM.h" +#include +#include //---------------------------------------------------------------------------- const char * @@ -103,119 +105,131 @@ const char * HttpDebugNames::get_event_name(int event) { switch (event) { - ///////////////////////// - // VCONNECTION EVENTS // - ///////////////////////// - case VC_EVENT_NONE: - return ("VC_EVENT_NONE"); - case VC_EVENT_IMMEDIATE: - return ("VC_EVENT_IMMEDIATE"); + case EVENT_NONE: + static_assert(static_cast(EVENT_NONE) == static_cast(VC_EVENT_NONE)); + return "EVENT_NONE/VC_EVENT_NONE"; + case EVENT_IMMEDIATE: + static_assert(static_cast(EVENT_IMMEDIATE) == static_cast(TS_EVENT_IMMEDIATE)); + static_assert(static_cast(EVENT_IMMEDIATE) == static_cast(VC_EVENT_IMMEDIATE)); + return "EVENT_IMMEDIATE/TS_EVENT_IMMEDIATE/VC_EVENT_IMMEDIATE"; + case EVENT_ERROR: + static_assert(static_cast(EVENT_ERROR) == static_cast(TS_EVENT_ERROR)); + static_assert(static_cast(EVENT_ERROR) == static_cast(VC_EVENT_ERROR)); + return "EVENT_ERROR/TS_EVENT_ERROR/VC_EVENT_ERROR"; + case EVENT_INTERVAL: + return "EVENT_INTERVAL"; case VC_EVENT_READ_READY: - return ("VC_EVENT_READ_READY"); + static_assert(static_cast(VC_EVENT_READ_READY) == static_cast(TS_EVENT_VCONN_READ_READY)); + return "VC_EVENT_READ_READY/TS_EVENT_VCONN_READ_READY"; case VC_EVENT_WRITE_READY: - return ("VC_EVENT_WRITE_READY"); + static_assert(static_cast(VC_EVENT_WRITE_READY) == static_cast(TS_EVENT_VCONN_WRITE_READY)); + return "VC_EVENT_WRITE_READY/TS_EVENT_VCONN_WRITE_READY"; case VC_EVENT_READ_COMPLETE: - return ("VC_EVENT_READ_COMPLETE"); + static_assert(static_cast(VC_EVENT_READ_COMPLETE) == static_cast(TS_EVENT_VCONN_READ_COMPLETE)); + return "VC_EVENT_READ_COMPLETE/TS_EVENT_VCONN_READ_COMPLETE"; case VC_EVENT_WRITE_COMPLETE: - return ("VC_EVENT_WRITE_COMPLETE"); + static_assert(static_cast(VC_EVENT_WRITE_COMPLETE) == static_cast(TS_EVENT_VCONN_WRITE_COMPLETE)); + return "VC_EVENT_WRITE_COMPLETE/TS_EVENT_VCONN_WRITE_COMPLETE"; case VC_EVENT_EOS: - return ("VC_EVENT_EOS"); - case VC_EVENT_ERROR: - return ("VC_EVENT_ERROR"); + static_assert(static_cast(VC_EVENT_EOS) == static_cast(TS_EVENT_VCONN_EOS)); + return "VC_EVENT_EOS/TS_EVENT_VCONN_EOS"; case VC_EVENT_INACTIVITY_TIMEOUT: - return ("VC_EVENT_INACTIVITY_TIMEOUT"); + static_assert(static_cast(VC_EVENT_INACTIVITY_TIMEOUT) == static_cast(TS_EVENT_VCONN_INACTIVITY_TIMEOUT)); + return "VC_EVENT_INACTIVITY_TIMEOUT/TS_EVENT_VCONN_INACTIVITY_TIMEOUT"; case VC_EVENT_ACTIVE_TIMEOUT: - return ("VC_EVENT_ACTIVE_TIMEOUT"); - case EVENT_INTERVAL: - return ("VC_EVENT_INTERVAL"); + static_assert(static_cast(VC_EVENT_ACTIVE_TIMEOUT) == static_cast(TS_EVENT_VCONN_ACTIVE_TIMEOUT)); + return "VC_EVENT_ACTIVE_TIMEOUT/TS_EVENT_VCONN_ACTIVE_TIMEOUT"; ///////////////// // NET EVENTS // ///////////////// case NET_EVENT_OPEN: - return ("NET_EVENT_OPEN"); + static_assert(static_cast(NET_EVENT_OPEN) == static_cast(TS_EVENT_NET_CONNECT)); + return "NET_EVENT_OPEN/TS_EVENT_NET_CONNECT"; case NET_EVENT_ACCEPT: - return ("NET_EVENT_ACCEPT"); + static_assert(static_cast(NET_EVENT_ACCEPT) == static_cast(TS_EVENT_NET_ACCEPT)); + return "NET_EVENT_ACCEPT/TS_EVENT_NET_ACCEPT"; case NET_EVENT_OPEN_FAILED: - return ("NET_EVENT_OPEN_FAILED"); + static_assert(static_cast(NET_EVENT_OPEN_FAILED) == static_cast(TS_EVENT_NET_CONNECT_FAILED)); + return "NET_EVENT_OPEN_FAILED/TS_EVENT_NET_CONNECT_FAILED"; //////////////////// // HOSTDB EVENTS // //////////////////// case EVENT_HOST_DB_LOOKUP: - return ("EVENT_HOST_DB_LOOKUP"); - + static_assert(static_cast(EVENT_HOST_DB_LOOKUP) == static_cast(TS_EVENT_HOST_LOOKUP)); + return "EVENT_HOST_DB_LOOKUP/TS_EVENT_HOST_LOOKUP"; case EVENT_HOST_DB_GET_RESPONSE: - return ("EVENT_HOST_DB_GET_RESPONSE"); - - //////////////////// - // HOSTDB EVENTS // - //////////////////// + return "EVENT_HOST_DB_GET_RESPONSE"; case EVENT_SRV_LOOKUP: - return ("EVENT_SRV_LOOKUP"); - + return "EVENT_SRV_LOOKUP"; case EVENT_SRV_IP_REMOVED: - return ("EVENT_SRV_IP_REMOVED"); - + return "EVENT_SRV_IP_REMOVED"; case EVENT_SRV_GET_RESPONSE: - return ("EVENT_SRV_GET_RESPONSE"); + return "EVENT_SRV_GET_RESPONSE"; //////////////////// // DNS EVENTS // //////////////////// case DNS_EVENT_LOOKUP: - return ("DNS_EVENT_LOOKUP"); - - //////////////////// - // CACHE EVENTS // - //////////////////// + return "DNS_EVENT_LOOKUP"; - case CACHE_EVENT_LOOKUP: - return ("CACHE_EVENT_LOOKUP"); + //////////////////// + // CACHE EVENTS // + //////////////////// case CACHE_EVENT_LOOKUP_FAILED: - return ("CACHE_EVENT_LOOKUP_FAILED"); + return "CACHE_EVENT_LOOKUP_FAILED"; case CACHE_EVENT_OPEN_READ: - return ("CACHE_EVENT_OPEN_READ"); + static_assert(static_cast(CACHE_EVENT_OPEN_READ) == static_cast(TS_EVENT_CACHE_OPEN_READ)); + return "CACHE_EVENT_OPEN_READ/TS_EVENT_CACHE_OPEN_READ"; case CACHE_EVENT_OPEN_READ_FAILED: - return ("CACHE_EVENT_OPEN_READ_FAILED"); + static_assert(static_cast(CACHE_EVENT_OPEN_READ_FAILED) == static_cast(TS_EVENT_CACHE_OPEN_READ_FAILED)); + return "CACHE_EVENT_OPEN_READ_FAILED/TS_EVENT_CACHE_OPEN_READ_FAILED"; case CACHE_EVENT_OPEN_WRITE: - return ("CACHE_EVENT_OPEN_WRITE"); + static_assert(static_cast(CACHE_EVENT_OPEN_WRITE) == static_cast(TS_EVENT_CACHE_OPEN_WRITE)); + return "CACHE_EVENT_OPEN_WRITE/TS_EVENT_CACHE_OPEN_WRITE"; case CACHE_EVENT_OPEN_WRITE_FAILED: - return ("CACHE_EVENT_OPEN_WRITE_FAILED"); + static_assert(static_cast(CACHE_EVENT_OPEN_WRITE_FAILED) == static_cast(TS_EVENT_CACHE_OPEN_WRITE_FAILED)); + return "CACHE_EVENT_OPEN_WRITE_FAILED/TS_EVENT_CACHE_OPEN_WRITE_FAILED"; case CACHE_EVENT_REMOVE: - return ("CACHE_EVENT_REMOVE"); + static_assert(static_cast(CACHE_EVENT_REMOVE) == static_cast(TS_EVENT_CACHE_REMOVE)); + return "CACHE_EVENT_REMOVE/TS_EVENT_CACHE_REMOVE"; case CACHE_EVENT_REMOVE_FAILED: - return ("CACHE_EVENT_REMOVE_FAILED"); + static_assert(static_cast(CACHE_EVENT_REMOVE_FAILED) == static_cast(TS_EVENT_CACHE_REMOVE_FAILED)); + return "CACHE_EVENT_REMOVE_FAILED/TS_EVENT_CACHE_REMOVE_FAILED"; case CACHE_EVENT_UPDATE: - return ("CACHE_EVENT_UPDATE"); + return "CACHE_EVENT_UPDATE"; case CACHE_EVENT_UPDATE_FAILED: - return ("CACHE_EVENT_UPDATE_FAILED"); + return "CACHE_EVENT_UPDATE_FAILED"; case STAT_PAGE_SUCCESS: - return ("STAT_PAGE_SUCCESS"); + return "STAT_PAGE_SUCCESS"; case STAT_PAGE_FAILURE: - return ("STAT_PAGE_FAILURE"); + return "STAT_PAGE_FAILURE"; case TRANSFORM_READ_READY: - return ("TRANSFORM_READ_READY"); + static_assert(static_cast(TRANSFORM_READ_READY) == static_cast(TS_EVENT_SSL_SESSION_GET)); + return "TRANSFORM_READ_READY/TS_EVENT_SSL_SESSION_GET"; ///////////////////////// - // HttpTunnel Events // + // HttpTunnel Events // ///////////////////////// case HTTP_TUNNEL_EVENT_DONE: - return ("HTTP_TUNNEL_EVENT_DONE"); + return "HTTP_TUNNEL_EVENT_DONE"; case HTTP_TUNNEL_EVENT_PRECOMPLETE: - return ("HTTP_TUNNEL_EVENT_PRECOMPLETE"); + return "HTTP_TUNNEL_EVENT_PRECOMPLETE"; case HTTP_TUNNEL_EVENT_CONSUMER_DETACH: - return ("HTTP_TUNNEL_EVENT_CONSUMER_DETACH"); + return "HTTP_TUNNEL_EVENT_CONSUMER_DETACH"; - ////////////////////////////// + ///////////////////////////// // Plugin Events - ////////////////////////////// + ///////////////////////////// case HTTP_API_CONTINUE: - return ("HTTP_API_CONTINUE"); + static_assert(static_cast(HTTP_API_CONTINUE) == static_cast(TS_EVENT_HTTP_CONTINUE)); + return "HTTP_API_CONTINUE/TS_EVENT_HTTP_CONTINUE"; case HTTP_API_ERROR: - return ("HTTP_API_ERROR"); + static_assert(static_cast(HTTP_API_ERROR) == static_cast(TS_EVENT_HTTP_ERROR)); + return "HTTP_API_ERROR/TS_EVENT_HTTP_ERROR"; /////////////////////////////// // Scheduled Update Events @@ -232,6 +246,133 @@ HttpDebugNames::get_event_name(int event) return "HTTP_SCH_UPDATE_EVENT_ERROR"; case HTTP_SCH_UPDATE_EVENT_NO_ACTION: return "HTTP_SCH_UPDATE_EVENT_NO_ACTION"; + + case TS_EVENT_NET_ACCEPT_FAILED: + return "TS_EVENT_NET_ACCEPT_FAILED"; + case TS_EVENT_INTERNAL_206: + return "TS_EVENT_INTERNAL_206"; + case TS_EVENT_INTERNAL_207: + return "TS_EVENT_INTERNAL_207"; + case TS_EVENT_INTERNAL_208: + return "TS_EVENT_INTERNAL_208"; + case TS_EVENT_INTERNAL_209: + return "TS_EVENT_INTERNAL_209"; + case TS_EVENT_INTERNAL_210: + return "TS_EVENT_INTERNAL_210"; + case TS_EVENT_INTERNAL_211: + return "TS_EVENT_INTERNAL_211"; + case TS_EVENT_INTERNAL_212: + return "TS_EVENT_INTERNAL_212"; + case TS_EVENT_CACHE_SCAN: + return "TS_EVENT_CACHE_SCAN"; + case TS_EVENT_CACHE_SCAN_FAILED: + return "TS_EVENT_CACHE_SCAN_FAILED"; + case TS_EVENT_CACHE_SCAN_OBJECT: + return "TS_EVENT_CACHE_SCAN_OBJECT"; + case TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED: + return "TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED"; + case TS_EVENT_CACHE_SCAN_OPERATION_FAILED: + return "TS_EVENT_CACHE_SCAN_OPERATION_FAILED"; + case TS_EVENT_CACHE_SCAN_DONE: + return "TS_EVENT_CACHE_SCAN_DONE"; + case TS_EVENT_CACHE_LOOKUP: + return "TS_EVENT_CACHE_LOOKUP"; + case TS_EVENT_CACHE_READ: + return "TS_EVENT_CACHE_READ"; + case TS_EVENT_CACHE_DELETE: + return "TS_EVENT_CACHE_DELETE"; + case TS_EVENT_CACHE_WRITE: + return "TS_EVENT_CACHE_WRITE"; + case TS_EVENT_CACHE_WRITE_HEADER: + return "TS_EVENT_CACHE_WRITE_HEADER"; + case TS_EVENT_CACHE_CLOSE: + return "TS_EVENT_CACHE_CLOSE"; + case TS_EVENT_CACHE_LOOKUP_READY: + return "TS_EVENT_CACHE_LOOKUP_READY"; + case TS_EVENT_CACHE_LOOKUP_COMPLETE: + return "TS_EVENT_CACHE_LOOKUP_COMPLETE"; + case TS_EVENT_CACHE_READ_READY: + return "TS_EVENT_CACHE_READ_READY"; + case TS_EVENT_CACHE_READ_COMPLETE: + return "TS_EVENT_CACHE_READ_COMPLETE"; + case TS_EVENT_INTERNAL_1200: + return "TS_EVENT_INTERNAL_1200"; + case TS_EVENT_SSL_SESSION_NEW: + return "TS_EVENT_SSL_SESSION_NEW"; + case TS_EVENT_SSL_SESSION_REMOVE: + return "TS_EVENT_SSL_SESSION_REMOVE"; + case TS_EVENT_AIO_DONE: + return "TS_EVENT_AIO_DONE"; + case TS_EVENT_HTTP_READ_REQUEST_HDR: + return "TS_EVENT_HTTP_READ_REQUEST_HDR"; + case TS_EVENT_HTTP_OS_DNS: + return "TS_EVENT_HTTP_OS_DNS"; + case TS_EVENT_HTTP_SEND_REQUEST_HDR: + return "TS_EVENT_HTTP_SEND_REQUEST_HDR"; + case TS_EVENT_HTTP_READ_CACHE_HDR: + return "TS_EVENT_HTTP_READ_CACHE_HDR"; + case TS_EVENT_HTTP_READ_RESPONSE_HDR: + return "TS_EVENT_HTTP_READ_RESPONSE_HDR"; + case TS_EVENT_HTTP_SEND_RESPONSE_HDR: + return "TS_EVENT_HTTP_SEND_RESPONSE_HDR"; + case TS_EVENT_HTTP_REQUEST_TRANSFORM: + return "TS_EVENT_HTTP_REQUEST_TRANSFORM"; + case TS_EVENT_HTTP_RESPONSE_TRANSFORM: + return "TS_EVENT_HTTP_RESPONSE_TRANSFORM"; + case TS_EVENT_HTTP_SELECT_ALT: + return "TS_EVENT_HTTP_SELECT_ALT"; + case TS_EVENT_HTTP_TXN_START: + return "TS_EVENT_HTTP_TXN_START"; + case TS_EVENT_HTTP_TXN_CLOSE: + return "TS_EVENT_HTTP_TXN_CLOSE"; + case TS_EVENT_HTTP_SSN_START: + return "TS_EVENT_HTTP_SSN_START"; + case TS_EVENT_HTTP_SSN_CLOSE: + return "TS_EVENT_HTTP_SSN_CLOSE"; + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: + return "TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE"; + case TS_EVENT_HTTP_PRE_REMAP: + return "TS_EVENT_HTTP_PRE_REMAP"; + case TS_EVENT_HTTP_POST_REMAP: + return "TS_EVENT_HTTP_POST_REMAP"; + case TS_EVENT_LIFECYCLE_PORTS_INITIALIZED: + return "TS_EVENT_LIFECYCLE_PORTS_INITIALIZED"; + case TS_EVENT_LIFECYCLE_PORTS_READY: + return "TS_EVENT_LIFECYCLE_PORTS_READY"; + case TS_EVENT_LIFECYCLE_CACHE_READY: + return "TS_EVENT_LIFECYCLE_CACHE_READY"; + case TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED: + return "TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED"; + case TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED: + return "TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED"; + case TS_EVENT_VCONN_START: + return "TS_EVENT_VCONN_START"; + case TS_EVENT_VCONN_CLOSE: + return "TS_EVENT_VCONN_CLOSE"; + case TS_EVENT_LIFECYCLE_MSG: + return "TS_EVENT_LIFECYCLE_MSG"; + case TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE: + return "TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE"; + case TS_EVENT_MGMT_UPDATE: + return "TS_EVENT_MGMT_UPDATE"; + case TS_EVENT_INTERNAL_60200: + return "TS_EVENT_INTERNAL_60200"; + case TS_EVENT_INTERNAL_60201: + return "TS_EVENT_INTERNAL_60201"; + case TS_EVENT_INTERNAL_60202: + return "TS_EVENT_INTERNAL_60202"; + case TS_EVENT_SSL_CERT: + return "TS_EVENT_SSL_CERT"; + case TS_EVENT_SSL_SERVERNAME: + return "TS_EVENT_SSL_SERVERNAME"; + case TS_EVENT_SSL_VERIFY_SERVER: + return "TS_EVENT_SSL_VERIFY_SERVER"; + case TS_EVENT_SSL_VERIFY_CLIENT: + return "TS_EVENT_SSL_VERIFY_CLIENT"; + case TS_EVENT_VCONN_OUTBOUND_START: + return "TS_EVENT_VCONN_OUTBOUND_START"; + case TS_EVENT_VCONN_OUTBOUND_CLOSE: + return "TS_EVENT_VCONN_OUTBOUND_CLOSE"; } return ("unknown event"); diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 886a176813f..e044e5cd929 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -8978,7 +8978,7 @@ REGRESSION_TEST(SDK_API_DEBUG_NAME_LOOKUPS)(RegressionTest *test, int /* atype A bool success = true; const char state_name[] = "INACTIVE_TIMEOUT"; const char hook_name[] = "TS_HTTP_READ_RESPONSE_HDR_HOOK"; - const char event_name[] = "VC_EVENT_IMMEDIATE"; + const char event_name[] = "TS_EVENT_IMMEDIATE"; const char *str; *pstatus = REGRESSION_TEST_INPROGRESS; @@ -9002,9 +9002,9 @@ REGRESSION_TEST(SDK_API_DEBUG_NAME_LOOKUPS)(RegressionTest *test, int /* atype A } str = TSHttpEventNameLookup(TS_EVENT_IMMEDIATE); - if ((strlen(str) != strlen(event_name) || strcmp(str, event_name))) { - SDK_RPRINT(test, "TSHttpEventNameLookup", "TestCase1", TC_FAIL, "Failed on %d, expected %s, got %s", TS_EVENT_IMMEDIATE, - hook_name, str); + if (strstr(str, event_name) == nullptr) { + SDK_RPRINT(test, "TSHttpEventNameLookup", "TestCase1", TC_FAIL, "Failed on %d, expected %s to be within %s", TS_EVENT_IMMEDIATE, + event_name, str); success = false; } else { SDK_RPRINT(test, "TSHttpEventNameLookup", "TestCase1", TC_PASS, "ok"); diff --git a/tests/gold_tests/pluginTest/test_hooks/log.gold b/tests/gold_tests/pluginTest/test_hooks/log.gold index 0c6258d6c44..5d5be50a427 100644 --- a/tests/gold_tests/pluginTest/test_hooks/log.gold +++ b/tests/gold_tests/pluginTest/test_hooks/log.gold @@ -1,11 +1,11 @@ -Global: event=unknown event -Global: event=unknown event -Session: event=unknown event -Global: event=unknown event -Session: event=unknown event -Transaction: event=unknown event -Global: event=unknown event -Session: event=unknown event -Transaction: event=unknown event -Global: event=unknown event -Session: event=unknown event +Global: event=TS_EVENT_HTTP_SSN_START +Global: event=TS_EVENT_HTTP_TXN_START +Session: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Session: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Global: event=TS_EVENT_HTTP_TXN_CLOSE +Session: event=TS_EVENT_HTTP_TXN_CLOSE +Transaction: event=TS_EVENT_HTTP_TXN_CLOSE +Global: event=TS_EVENT_HTTP_SSN_CLOSE +Session: event=TS_EVENT_HTTP_SSN_CLOSE From fc785d39450d3846d3928fb099b021b230a2d2af Mon Sep 17 00:00:00 2001 From: Takuya Kitano Date: Thu, 1 Nov 2018 14:10:14 +0900 Subject: [PATCH 055/526] rewrite url after running all remap plugins --- proxy/http/remap/RemapPlugins.cc | 20 ++++++++++---------- proxy/http/remap/RemapPlugins.h | 11 ++++++++--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/proxy/http/remap/RemapPlugins.cc b/proxy/http/remap/RemapPlugins.cc index ebddec2bacd..198f85d0470 100644 --- a/proxy/http/remap/RemapPlugins.cc +++ b/proxy/http/remap/RemapPlugins.cc @@ -101,24 +101,24 @@ RemapPlugins::run_single_remap() return 1; } - if (TSREMAP_NO_REMAP == plugin_retcode || TSREMAP_NO_REMAP_STOP == plugin_retcode) { - // After running the first plugin, rewrite the request URL. This is doing the default rewrite rule - // to handle the case where no plugin ever rewrites. - // - // XXX we could probably optimize this a bit more by keeping a flag and only rewriting the request URL - // if no plugin has rewritten it already. - if (_cur == 1) { - Debug("url_rewrite", "plugin did not change host, port or path, copying from mapping rule"); - url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); - } + if (TSREMAP_DID_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP == plugin_retcode) { + _rewriten++; } if (TSREMAP_NO_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP_STOP == plugin_retcode) { + if (_rewriten == 0) { + Debug("url_rewrite", "plugin did not change host, port or path, copying from mapping rule"); + url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); + } Debug("url_rewrite", "breaking remap plugin chain since last plugin said we should stop"); return 1; } if (_cur >= map->plugin_count()) { + if (_rewriten == 0) { + Debug("url_rewrite", "plugin did not change host, port or path, copying from mapping rule"); + url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); + } // Normally, we would callback into this function but we dont have anything more to do! Debug("url_rewrite", "completed all remap plugins for rule id %d", map->map_id); return 1; diff --git a/proxy/http/remap/RemapPlugins.h b/proxy/http/remap/RemapPlugins.h index bd7647710c4..ed7d55d8574 100644 --- a/proxy/http/remap/RemapPlugins.h +++ b/proxy/http/remap/RemapPlugins.h @@ -38,13 +38,17 @@ * A class that represents a queue of plugins to run **/ struct RemapPlugins : public Continuation { - RemapPlugins() : _cur(0) {} + RemapPlugins() : _cur(0), _rewriten(0) {} RemapPlugins(HttpTransact::State *s, URL *u, HTTPHdr *h, host_hdr_info *hi) - : _cur(0), _s(s), _request_url(u), _request_header(h), _hh_ptr(hi) + : _cur(0), _rewriten(0), _s(s), _request_url(u), _request_header(h), _hh_ptr(hi) { } - ~RemapPlugins() override { _cur = 0; } + ~RemapPlugins() override + { + _cur = 0; + _rewriten = 0; + } // Some basic setters void setState(HttpTransact::State *state) @@ -75,6 +79,7 @@ struct RemapPlugins : public Continuation { private: unsigned int _cur = 0; + unsigned int _rewriten = 0; HttpTransact::State *_s = nullptr; URL *_request_url = nullptr; HTTPHdr *_request_header = nullptr; From 539a579b380362b398b41c13f335c89af2946e84 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 27 Nov 2018 21:28:45 +0000 Subject: [PATCH 056/526] Add verify_server test without pristine header. --- .../tls/tls_verify_not_pristine.test.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 tests/gold_tests/tls/tls_verify_not_pristine.test.py diff --git a/tests/gold_tests/tls/tls_verify_not_pristine.test.py b/tests/gold_tests/tls/tls_verify_not_pristine.test.py new file mode 100644 index 00000000000..2418063827a --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_not_pristine.test.py @@ -0,0 +1,114 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification without a pristine host header +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) +dns = Test.MakeDNServer("dns") + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: badfoo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map https://bar.com:{0}/ https://foo.com:{1}'.format(ts.Variables.ssl_port, server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map https://foo.com:{0}/ https://bar.com:{1}'.format(ts.Variables.ssl_port, server_foo.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|ssl|dns|hostdb', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'ENFORCED', + 'proxy.config.ssl.client.verify.server.properties': 'ALL', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 0, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL' +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + +# bar.com in. foo.com out. Should verify +tr = Test.AddTestRun("Enforced-Good-Test") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.TimeOut = 5 + +# foo.com in. bar.com out. Should not verify +tr2 = Test.AddTestRun("Enforced-bad-test") +tr2.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should not have succeeded") +tr2.TimeOut = 5 + +# Over riding the built in ERROR check since we expect tr3 to fail +ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate", "Make sure bad_bar name checked failed.") From c46cb2280e503c850db60c2ddabe8b2f5eea9439 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 29 Nov 2018 19:47:24 +0000 Subject: [PATCH 057/526] Allow empty fqdn in ssl_server_name for unsent SNI case. --- iocore/net/P_SSLNetVConnection.h | 2 +- iocore/net/P_SSLSNI.h | 6 +- iocore/net/SSLNetVConnection.cc | 6 - iocore/net/SSLSNIConfig.cc | 4 +- iocore/net/SSLUtils.cc | 64 ++++++----- proxy/http/HttpSM.cc | 4 +- tests/gold_tests/tls/tls_tunnel.test.py | 31 +++++- .../tls/tls_tunnel_plugin_rename.test.py | 103 ++++++++++++++++++ tests/tools/plugins/ssl_sni_rename_test.cc | 73 +++++++++++++ 9 files changed, 250 insertions(+), 43 deletions(-) create mode 100644 tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py create mode 100644 tests/tools/plugins/ssl_sni_rename_test.cc diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 163709a059d..e2b7fdc72a4 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -361,7 +361,7 @@ class SSLNetVConnection : public UnixNetVConnection ink_hrtime sslHandshakeEndTime = 0; ink_hrtime sslLastWriteTime = 0; int64_t sslTotalBytesSent = 0; - char *serverName = nullptr; + std::string serverName; /// Set by asynchronous hooks to request a specific operation. SslVConnOp hookOpRequested = SSL_HOOK_OP_DEFAULT; diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index 781c4da8132..15580ff05ff 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -76,7 +76,11 @@ struct namedElement { { const char *err_ptr; int err_offset = 0; - match = pcre_compile(regexName.c_str(), 0, &err_ptr, &err_offset, nullptr); + if (!regexName.empty()) { + match = pcre_compile(regexName.c_str(), 0, &err_ptr, &err_offset, nullptr); + } else { + match = nullptr; + } } pcre *match = nullptr; diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index e222d8ef431..86e150ce608 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -1709,12 +1709,6 @@ SSLNetVConnection::callHooks(TSEvent eventId) bool reenabled = true; - this->serverName = const_cast(SSL_get_servername(this->ssl, TLSEXT_NAMETYPE_host_name)); - if (this->has_tunnel_destination()) { - this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; - return reenabled; - } - if (SSL_HOOK_OP_TUNNEL == hookOpRequested) { this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; // Don't mark the handshake as complete yet, diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index ca4f1934d82..21986b7d47a 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -103,7 +103,9 @@ const actionVector * SNIConfigParams::get(const std::string &servername) const { for (auto retval = sni_action_list.begin(); retval != sni_action_list.end(); ++retval) { - if (pcre_exec(retval->match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { + if (retval->match == nullptr && servername.length() == 0) { + return &retval->actions; + } else if (pcre_exec(retval->match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { return &retval->actions; } } diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 335ae53a601..f68c522b936 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -405,6 +405,24 @@ ssl_verify_client_callback(int preverify_ok, X509_STORE_CTX *ctx) return preverify_ok; } +static int +PerformAction(Continuation *cont, const char *servername) +{ + SNIConfig::scoped_config params; + const actionVector *actionvec = params->get(servername); + if (!actionvec) { + Debug("ssl_sni", "%s not available in the map", servername); + } else { + for (auto &&item : *actionvec) { + auto ret = item->SNIAction(cont); + if (ret != SSL_TLSEXT_ERR_OK) { + return ret; + } + } + } + return SSL_TLSEXT_ERR_OK; +} + // Use the certificate callback for openssl 1.0.2 and greater // otherwise use the SNI callback #if TS_USE_CERT_CB @@ -445,41 +463,25 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) return retval; } -static int -PerformAction(Continuation *cont, const char *servername) -{ - SNIConfig::scoped_config params; - const actionVector *actionvec = params->get(servername); - if (!actionvec) { - Debug("ssl_sni", "%s not available in the map", servername); - } else { - for (auto &&item : *actionvec) { - auto ret = item->SNIAction(cont); - if (ret != SSL_TLSEXT_ERR_OK) { - return ret; - } - } - } - return SSL_TLSEXT_ERR_OK; -} - /* * Cannot stop this callback. Always reeneabled */ static int ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) { - int ret = SSL_TLSEXT_ERR_OK; SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); - Debug("ssl", "Requested servername is %s", servername); - if (servername != nullptr) { - ret = PerformAction(netvc, servername); - } - if (ret != SSL_TLSEXT_ERR_OK) + netvc->callHooks(TS_EVENT_SSL_SERVERNAME); + + const char *name = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + netvc->serverName = std::string{name ? name : ""}; + int ret = PerformAction(netvc, netvc->serverName.c_str()); + if (ret != SSL_TLSEXT_ERR_OK) { return SSL_TLSEXT_ERR_ALERT_FATAL; + } + if (netvc->has_tunnel_destination()) { + netvc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; + } - netvc->callHooks(TS_EVENT_SSL_SERVERNAME); return SSL_TLSEXT_ERR_OK; } @@ -491,6 +493,16 @@ ssl_servername_and_cert_callback(SSL *ssl, int * /* ad */, void * /*arg*/) bool reenabled; int retval = 1; + const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + if (servername == nullptr) { + servername = ""; + } + Debug("ssl", "Requested servername is %s", servername); + int ret = PerformAction(netvc, servername); + if (ret != SSL_TLSEXT_ERR_OK) { + return SSL_TLSEXT_ERR_ALERT_FATAL; + } + // If we are in tunnel mode, don't select a cert. Pause! if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == netvc->attributes) { return -1; // Pause diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 52d4dde2390..b5289323f4f 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -583,7 +583,7 @@ HttpSM::setup_blind_tunnel_port() t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } else { - t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); + t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName.c_str(), ssl_vc->serverName.length()); t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } @@ -1394,7 +1394,7 @@ plugins required to work with sni_routing. t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } else if (ssl_vc) { - t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); + t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName.data(), ssl_vc->serverName.length()); t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index e92c4a168ed..2c61a788bc1 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -29,13 +29,14 @@ # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=False) server_foo = Test.MakeOriginServer("server_foo", ssl=True) -server_bar = Test.MakeOriginServer("server_bar", ssl=False) +server_bar = Test.MakeOriginServer("server_bar", ssl=True) -request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -server_foo.addResponse("sessionlog.json", request_foo_header, response_header) -server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "foo ok"} +response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "bar ok"} +server_foo.addResponse("sessionlog.json", request_foo_header, response_foo_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_bar_header) # add ssl materials like key, certificates for the server ts.addSSLfile("ssl/signed-foo.pem") @@ -73,12 +74,15 @@ }) # foo.com should not terminate. Just tunnel to server_foo -# bar.com should terminate. Forward its tcp stream to server_bar +# bar.com should terminate. +# empty SNI should tunnel to server_bar ts.Disk.ssl_server_name_yaml.AddLines([ '- fqdn: foo.com', " tunnel_route: localhost:{0}".format(server_foo.Variables.Port), "- fqdn: bob.*.com", " tunnel_route: localhost:{0}".format(server_foo.Variables.Port), + "- fqdn: ''", # No SNI sent + " tunnel_route: localhost:{0}".format(server_bar.Variables.Port) ]) tr = Test.AddTestRun("foo.com Tunnel-test") @@ -95,6 +99,7 @@ tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from bar") tr = Test.AddTestRun("bob.bar.com Tunnel-test") tr.Processes.Default.Command = "curl -v --resolve 'bob.bar.com:{0}:127.0.0.1' -k https://bob.bar.com:{0}".format(ts.Variables.ssl_port) @@ -107,6 +112,7 @@ tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from bar") tr = Test.AddTestRun("bar.com no Tunnel-test") tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) @@ -118,5 +124,18 @@ tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") +tr = Test.AddTestRun("no SNI Tunnel-test") +tr.Processes.Default.Command = "curl -v -k https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("bar ok", "Should get a response from bar") + + diff --git a/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py new file mode 100644 index 00000000000..920ed9f9337 --- /dev/null +++ b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py @@ -0,0 +1,103 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tunneling based on SNI renaming +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_bar = Test.MakeOriginServer("server_bar", ssl=True) +server_random = Test.MakeOriginServer("server_random", ssl=True) + +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_random_header = {"headers": "GET / HTTP/1.1\r\nHost: random.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok bar"} +response_random_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok random"} +server_bar.addResponse("sessionlog_bar.json", request_bar_header, response_bar_header) +server_random.addResponse("sessionlog_random.json", request_random_header, response_random_header) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_sni_rename_test.cc'), ts) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_bar.Variables.Port,server_random.Variables.Port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# bar.com should terminate. +# empty should tunnel to server_random (should not happen) +# newname should tunnel to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + "- fqdn: newname", + " tunnel_route: localhost:{0}".format(server_bar.Variables.Port), + "- fqdn: ''", #default case + " tunnel_route: localhost:{0}".format(server_random.Variables.Port), + ]) + +# Plugin should add "newname" to the empty sni and go to _bar instead of random.com +tr = Test.AddTestRun("no-sni-tunnel-test") +tr.Processes.Default.Command = "curl --http1.1 -v -k https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server_random) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server_random +tr.Processes.Default.TimeOut = 5 +tr.StillRunningAfter = ts +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ok bar", "Body is expected") + diff --git a/tests/tools/plugins/ssl_sni_rename_test.cc b/tests/tools/plugins/ssl_sni_rename_test.cc new file mode 100644 index 00000000000..99a55ef4d74 --- /dev/null +++ b/tests/tools/plugins/ssl_sni_rename_test.cc @@ -0,0 +1,73 @@ +/** @file + + SSL Preaccept test plugin + Implements blind tunneling based on the client IP address + The client ip addresses are specified in the plugin's + config file as an array of IP addresses or IP address ranges under the + key "client-blind-tunnel" + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define PN "ssl_rename_test" +#define PCP "[" PN " Plugin] " + +std::map bad_names; + +int +CB_server_rename(TSCont cont, TSEvent event, void *edata) +{ + TSVConn ssl_vc = reinterpret_cast(edata); + + TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + SSL *ssl = (SSL *)sslobj; + const char *sni_name = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + if (!sni_name) { + SSL_set_tlsext_host_name(ssl, "newname"); + } + + // All done, reactivate things + TSVConnReenable(ssl_vc); + return TS_SUCCESS; +} + +// Called by ATS as our initialization point +void +TSPluginInit(int argc, const char *argv[]) +{ + TSPluginRegistrationInfo info; + info.plugin_name = const_cast("SSL rename test"); + info.vendor_name = const_cast("apache"); + info.support_email = const_cast("shinrich@apache.org"); + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[%s] Plugin registration failed", PN); + } + TSCont cb = TSContCreate(&CB_server_rename, TSMutexCreate()); + TSHttpHookAdd(TS_SSL_SERVERNAME_HOOK, cb); + + return; +} From 944e51dda358b04b5ad2beb9daa159518cae5e11 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Fri, 30 Nov 2018 11:30:59 -0600 Subject: [PATCH 058/526] Runroot: Update verify command and refactoring --- .../command-line/traffic_layout.en.rst | 9 +- src/traffic_layout/engine.cc | 589 +++++++++--------- src/traffic_layout/engine.h | 18 +- src/traffic_layout/traffic_layout.cc | 2 +- src/tscore/ArgParser.cc | 7 +- tests/gold_tests/runroot/runroot_init.test.py | 1 - .../gold_tests/runroot/runroot_verify.test.py | 15 +- 7 files changed, 337 insertions(+), 304 deletions(-) diff --git a/doc/appendices/command-line/traffic_layout.en.rst b/doc/appendices/command-line/traffic_layout.en.rst index 3d4fedd33b2..0898443f7ae 100644 --- a/doc/appendices/command-line/traffic_layout.en.rst +++ b/doc/appendices/command-line/traffic_layout.en.rst @@ -54,7 +54,7 @@ First we need to create a runroot. It can be created simply by calling command ` A runroot will be created in ``/path/to/runroot``, available for other programs to use. If the path is not specified, the current working directory will be used. -To run traffic_manager, for example, using the runroot, there are several ways: +For example, to run traffic_manager, using the runroot, there are several ways: #. ``/path/to/runroot/bin/traffic_manager`` #. ``traffic_manager --run-root=/path/to/runroot`` #. ``traffic_manager --run-root=/path/to/runroot/runroot.yaml`` @@ -127,6 +127,13 @@ Example: :: traffic_layout verify (--path /path/to/sandbox/) (--fix) (--with-user root) +.. Warning:: + + If a custom layout is used and system files are included in some directories, ``--fix`` option might potentially have unexpected behaviors. + For example, if sysconfdir is defined as ``/etc`` instead of ``/etc/trafficserver`` in ``runroot.yaml``, + ``--fix`` may perform permission changes on the system configuration files. With normally created runroot with default layout, + there is no such issue since traffic server related files are filtered. + ======= Options ======= diff --git a/src/traffic_layout/engine.cc b/src/traffic_layout/engine.cc index ea2c96b7b4a..e6a2703afcd 100644 --- a/src/traffic_layout/engine.cc +++ b/src/traffic_layout/engine.cc @@ -39,16 +39,20 @@ #include #include #include +#include static const long MAX_LOGIN = ink_login_name_max(); static constexpr int MAX_GROUP_NUM = 32; -// for nftw check_directory -std::string directory_check; +// Personally I don't really like the way that nftw needs global variables. Right now there is no other options. +// This iteration through directory can be avoided after std::filesystem is in. +// for nftw check_directory +static std::string empty_check_directory; // for 'verify --fix' nftw iteration -int permission; -std::string cur_fix_dir; +static PermissionEntry permission_entry; +// if fix_flag is set, permission handler will perform fixing operation +static bool fix_flag = false; // the function for the checking of the yaml file in the passed in path // if found return the path, if not return empty string @@ -85,51 +89,27 @@ check_parent_path(const std::string &path) return {}; } -// check if we can create the runroot using path -// return true if the path is good to use -static bool -check_run_path(const std::string &arg, const bool forceflag) -{ - if (arg.empty()) { - return false; - } - // the condition of force create - if (is_directory(arg) && forceflag) { - std::cout << "Forcing creating runroot ..." << std::endl; - return true; - } - - // if directory already exist - if (is_directory(arg)) { - return true; - } else { - // try to create & remove - if (!create_directory(arg)) { - return false; - } - remove_directory(arg); - return true; - } -} - // handle the path of the engine during parsing static std::string path_handler(const std::string &path, bool run_flag, const std::string &command) { + std::string cur_working_dir = ""; char cwd[PATH_MAX]; - if (getcwd(cwd, sizeof(cwd)) == nullptr) { - ink_fatal("unexcepted failure from getcwd() code=%d", errno); + if (!getcwd(cwd, sizeof(cwd))) { + ink_warning("unexcepted failure from getcwd() - %s", strerror(errno)); + } else { + cur_working_dir = cwd; } if (run_flag) { // no path passed in, use cwd if (path.empty()) { - return cwd; + return cur_working_dir; } if (path[0] == '/') { return path; } else { - return Layout::relative_to(cwd, path); + return Layout::relative_to(cur_working_dir, path); } } @@ -140,7 +120,7 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) if (path[0] == '/') { cmdpath = check_path(path); } else { - cmdpath = check_path(Layout::relative_to(cwd, path)); + cmdpath = check_path(Layout::relative_to(cur_working_dir, path)); } if (!cmdpath.empty()) { return cmdpath; @@ -148,7 +128,7 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) } // 2. find cwd or parent path of cwd to check - std::string cwdpath = check_parent_path(cwd); + std::string cwdpath = check_parent_path(cur_working_dir); if (!cwdpath.empty()) { return cwdpath; } @@ -157,8 +137,9 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) char RealBinPath[PATH_MAX] = {0}; if (!command.empty() && realpath(command.c_str(), RealBinPath) != nullptr) { std::string bindir = RealBinPath; - bindir = bindir.substr(0, bindir.find_last_of("/")); // getting the bin dir not executable path - bindir = check_parent_path(bindir); + + bindir = bindir.substr(0, bindir.find_last_of("/")); // getting the bin dir not executable path + bindir = check_parent_path(bindir); if (!bindir.empty()) { return bindir; } @@ -167,29 +148,11 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) return path; } -// if directory is not empty, ask input from user Y/N +// check if there is any directory inside empty_check_directory to see if it is empty or not static int -check_directory(const char *path, const struct stat *s, int flag, struct FTW *f) +check_directory_empty(const char *path, const struct stat *s, int flag) { - std::string checkpath = path; - if (checkpath != directory_check) { - // check for valid Y/N - for (int i = 0; i < 3; i++) { - std::cout << "Are you sure to create runroot inside a non-empty directory Y/N: "; - std::string input; - std::cin >> input; - if (input == "Y" || input == "y") { - // terminate nftw - return 1; - } - if (input == "N" || input == "n") { - exit(0); - } - } - ink_error("Invalid input Y/N"); - exit(70); - } - return 0; + return std::string(path) != empty_check_directory ? -1 : 0; } void @@ -208,14 +171,20 @@ void LayoutEngine::create_runroot() { // set up options - std::string path = path_handler(arguments.get("path").value(), true, _argv[0]); + std::string path = path_handler(arguments.get("path").value(), true, _argv[0]); + if (path.empty()) { + ink_error("Path not valid for creating"); + status_code = EX_SOFTWARE; + return; + } bool force_flag = arguments.get("force"); bool abs_flag = arguments.get("absolute"); std::string layout_file = arguments.get("layout").value(); if (layout_file.find("runroot.yaml") != std::string::npos) { ink_error( "'runroot.yaml' is a potentially dangerous name for '--layout' option.\nPlease set other name to the file for '--layout'"); - exit(70); + status_code = EX_SOFTWARE; + return; } // deal with the copy style CopyStyle copy_style = HARD; @@ -230,13 +199,10 @@ LayoutEngine::create_runroot() copy_style = HARD; } else { ink_error("Unknown copy style: '%s'", style.c_str()); - exit(70); + status_code = EX_USAGE; + return; } } - // check validity of the path - if (!check_run_path(path, force_flag)) { - ink_fatal("Failed to create runroot with path '%s'", path.c_str()); - } std::string original_root = Layout::get()->prefix; std::string ts_runroot = path; // check for existing runroot to use rather than create new one @@ -247,15 +213,35 @@ LayoutEngine::create_runroot() return; } if (!force_flag && !check_parent_path(ts_runroot).empty()) { - ink_fatal("Cannot create runroot inside another runroot"); + ink_error("Cannot create runroot inside another runroot"); + status_code = EX_SOFTWARE; + return; } std::cout << "creating runroot - " + ts_runroot << std::endl; // check if directory is empty when --force is not used if (is_directory(ts_runroot) && !force_flag) { - directory_check = ts_runroot; - nftw(ts_runroot.c_str(), check_directory, OPEN_MAX_FILE, FTW_DEPTH); + empty_check_directory = ts_runroot; + if (ftw(ts_runroot.c_str(), check_directory_empty, OPEN_MAX_FILE) != 0) { + // if the directory is not empty, check for valid Y/N + for (int i = 0; i < 3; i++) { + std::cout << "Are you sure to create runroot inside a non-empty directory Y/N: "; + std::string input; + std::cin >> input; + if (input == "Y" || input == "y") { + break; + } + if (input == "N" || input == "n") { + return; + } + if (i == 2) { + ink_error("Invalid input Y/N"); + status_code = EX_SOFTWARE; + return; + } + } + } } // create new root & copy from original to new runroot. then fill in the map @@ -281,10 +267,10 @@ LayoutEngine::create_runroot() if (layout_file.size() != 0) { try { YAML::Node yamlfile = YAML::LoadFile(layout_file); - for (auto &&it : yamlfile) { + for (const auto &it : yamlfile) { std::string key = it.first.as(); std::string value = it.second.as(); - auto &&iter = new_map.find(key); + auto iter = new_map.find(key); if (iter != new_map.end()) { iter->second = value; } else { @@ -303,7 +289,7 @@ LayoutEngine::create_runroot() // symlink the executables // set up path_map for yaml to emit key-value pairs RunrootMapType path_map; - for (auto &&it : original_map) { + for (const auto &it : original_map) { // path handle std::string join_path; if (it.second[0] && it.second[0] == '/') { @@ -345,7 +331,7 @@ LayoutEngine::create_runroot() YAML::Emitter yamlfile; // emit key value pairs to the yaml file yamlfile << YAML::BeginMap; - for (auto &&it : dir_vector) { + for (const auto &it : dir_vector) { yamlfile << YAML::Key << it; yamlfile << YAML::Value << path_map[it]; } @@ -367,18 +353,14 @@ LayoutEngine::remove_runroot() std::string path = path_handler(arguments.get("path").value(), false, _argv[0]); // check validity of the path if (path.empty()) { - ink_fatal("Path not valid (runroot.yaml not found)"); + ink_error("Path not valid (runroot.yaml not found)"); + status_code = EX_IOERR; + return; } std::string clean_root = path; append_slash(clean_root); - char cwd[PATH_MAX]; - if (getcwd(cwd, sizeof(cwd)) == nullptr) { - ink_fatal("unexcepted failure from getcwd() code=%d", errno); - } - std::string cur_working_dir = cwd; - if (arguments.get("force")) { // the force remove std::cout << "Forcing removing runroot ..." << std::endl; @@ -390,11 +372,20 @@ LayoutEngine::remove_runroot() RunrootMapType map = runroot_map(Layout::relative_to(clean_root, "runroot.yaml")); map.erase(LAYOUT_PREFIX); map.erase(LAYOUT_EXEC_PREFIX); - for (auto &&it : map) { + // get current working directory + std::string cur_working_dir = ""; + char cwd[PATH_MAX]; + if (getcwd(cwd, sizeof(cwd)) == nullptr) { + ink_warning("unexcepted failure from getcwd() - %s", strerror(errno)); + } else { + cur_working_dir = cwd; + } + for (const auto &it : map) { std::string dir = it.second; append_slash(dir); // get the directory to remove: prefix/etc/trafficserver -> prefix/etc dir = dir.substr(0, dir.substr(clean_root.size()).find_first_of("/") + clean_root.size()); + // don't remove cwd if (cur_working_dir != dir) { remove_directory(dir); } else { @@ -418,230 +409,250 @@ LayoutEngine::remove_runroot() } } -// return an array containing gid and uid of provided user -static std::pair -get_giduid(std::string &user) -{ - passwd *pwd; - if (user[0] == '#') { - // Numeric user notation. - uid_t uid = (uid_t)atoi(&user[1]); - pwd = getpwuid(uid); - } else { - pwd = getpwnam(user.c_str()); - } - if (!pwd) { - // null ptr - ink_fatal("missing password database entry for '%s'", user.c_str()); - } - user = pwd->pw_name; - std::pair giduid = {pwd->pw_gid, pwd->pw_uid}; - return giduid; -} - -// output read permission -static void -output_read_permission(const std::string &permission) +// check permission for verify +static int +permission_handler(const char *path, const struct stat *s, int flag) { - if (permission[0] == '1') { - std::cout << "\tRead Permission: \033[1;32mPASSED\033[0m" << std::endl; - } else { - std::cout << "\tRead Permission: \033[1;31mFAILED\033[0m" << std::endl; + std::string cur_directory = permission_entry.name; + // filter traffic server related files + if (!filter_ts_files(cur_directory, path)) { + return 0; } -} - -// output write permission -static void -output_write_permission(const std::string &permission) -{ - if (permission[1] == '1') { - std::cout << "\tWrite Permission: \033[1;32mPASSED\033[0m" << std::endl; - } else { - std::cout << "\tWrite Permission: \033[1;31mFAILED\033[0m" << std::endl; + if (flag == FTW_NS || !s) { + ink_warning("unable to stat() destination path %s - %s", path, strerror(errno)); + return -1; } -} -// output execute permission -static void -output_execute_permission(const std::string &permission) -{ - if (permission[2] == '1') { - std::cout << "\tExecute Permission: \033[1;32mPASSED\033[0m" << std::endl; + int ret = 0; + // --------- for directories --------- + if (flag == FTW_D || flag == FTW_DNR) { + // always need read permission for directories + if (!(s->st_mode & permission_entry.r_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.r_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed read permission: " << path << std::endl; + } + } else { + std::cout << "Read permission failed for directory: " << path << std::endl; + ret = -1; + } + } + // need write permission for logdir, runtimedir and cachedir + if (cur_directory == LAYOUT_LOGDIR || cur_directory == LAYOUT_RUNTIMEDIR || cur_directory == LAYOUT_CACHEDIR) { + if (!(s->st_mode & permission_entry.w_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.w_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed write permission: " << path << std::endl; + } + } else { + std::cout << "Write permission failed for directory: " << path << std::endl; + ret = -1; + } + } + } + // always need execute permission for directories + if (!(s->st_mode & permission_entry.e_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.e_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed execute permission: " << path << std::endl; + } + } else { + std::cout << "Execute permission failed for directory: " << path << std::endl; + ret = -1; + } + } } else { - std::cout << "\tExecute Permission: \033[1;31mFAILED\033[0m" << std::endl; + // --------- for files --------- + // always need read permission for all files + if (!(s->st_mode & permission_entry.r_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.r_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed read permission: " << path << std::endl; + } + } else { + std::cout << "Read permission failed for file " << path << std::endl; + ret = -1; + } + } + // need write permission for files in logdir, runtimedir and cachedir + if (cur_directory == LAYOUT_LOGDIR || cur_directory == LAYOUT_RUNTIMEDIR || cur_directory == LAYOUT_CACHEDIR) { + if (!(s->st_mode & permission_entry.w_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.w_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed write permission: " << path << std::endl; + } + } else { + std::cout << "Write permission failed for file " << path << std::endl; + ret = -1; + } + } + } + // execute permission on files in bindir, sbindir, libdir and libexecdir + if (cur_directory == LAYOUT_BINDIR || cur_directory == LAYOUT_SBINDIR || cur_directory == LAYOUT_LIBDIR || + cur_directory == LAYOUT_LIBEXECDIR) { + std::string tmp_path = path; + // skip the files in perl5 and pkgconfig from libdir + if (tmp_path.find("/perl5/") != std::string::npos || tmp_path.find("/pkgconfig/") != std::string::npos) { + return 0; + } + if (!(s->st_mode & permission_entry.e_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.e_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed execute permission: " << path << std::endl; + } + } else { + std::cout << "Execute permission failed for file: " << path << std::endl; + ret = -1; + } + } + } } + return ret; } -// chmod the file permission -static int -chmod_files_permission(const char *path, const struct stat *s, int flag, struct FTW *f) +// used for prefix, exec_prefix and localstatedir +// only check the read/execute permission on those directories +static bool +check_directory_permission(const char *path) { - // ----- filter traffic server related files ----- - if (!filter_ts_files(cur_fix_dir, path)) { - return 0; - } struct stat stat_buffer; if (stat(path, &stat_buffer) < 0) { - ink_warning("unable to stat() destination path %s: %s", path, strerror(errno)); - return 0; + ink_warning("unable to stat() destination path %s - %s", path, strerror(errno)); + return false; } - if (chmod(path, stat_buffer.st_mode | permission) < 0) { - ink_warning("Unable to change file mode on %s: %s", path, strerror(errno)); + if (!(stat_buffer.st_mode & permission_entry.r_mode)) { + std::cout << "Read permission failed for: " << path << std::endl; + return false; } - return 0; -} - -// the fixing permission of runroot used by verify command -static void -fix_runroot(RunrootMapType &path_map, RunrootMapType &permission_map, RunrootMapType &usergroup_map) -{ - if (getuid() != 0) { - ink_error("To fix permission issues, root privileges are required.\nPlease run with sudo."); - exit(70); - } - for (auto &&it : permission_map) { - std::string name = it.first; - std::string usergroup = usergroup_map[name]; - std::string path = path_map[name]; - std::cout << "Fixing " + name + "..." << std::endl; - - int read_permission; - int write_permission; - int execute_permission; - if (usergroup == "owner") { - read_permission = S_IRUSR; - write_permission = S_IWUSR; - execute_permission = S_IXUSR; - } else if (usergroup == "group") { - read_permission = S_IRGRP; - write_permission = S_IWGRP; - execute_permission = S_IXGRP; - } else { - read_permission = S_IROTH; - write_permission = S_IWOTH; - execute_permission = S_IXOTH; - } - // fix read permission - permission = read_permission; - cur_fix_dir = name; - nftw(path.c_str(), chmod_files_permission, OPEN_MAX_FILE, FTW_DEPTH); - // fix write permission - if (name == LAYOUT_LOGDIR || name == LAYOUT_RUNTIMEDIR || name == LAYOUT_CACHEDIR) { - permission = write_permission; - nftw(path.c_str(), chmod_files_permission, OPEN_MAX_FILE, FTW_DEPTH); - } - // fix execute permission - if (name == LAYOUT_BINDIR || name == LAYOUT_SBINDIR || name == LAYOUT_LIBDIR || name == LAYOUT_LIBEXECDIR) { - permission = execute_permission; - nftw(path.c_str(), chmod_files_permission, OPEN_MAX_FILE, FTW_DEPTH); - } + if (!(stat_buffer.st_mode & permission_entry.e_mode)) { + std::cout << "Execute permission failed for: " << path << std::endl; + return false; } + return true; } +#if defined(darwin) +// on Darwin, getgrouplist() takes int. +using gid_type = int; +#else +using gid_type = gid_t; +#endif + // helper function to check if user is from the same group of path_gid static bool -from_group(std::string const &user, gid_t group_id, gid_t path_gid) +from_group(const char *user, gid_type group_id, gid_type path_gid) { int ngroups = MAX_GROUP_NUM; - // Retrieve group list -#if defined(darwin) - // on Darwin, getgrouplist() takes int. - int groups[ngroups]; - if (getgrouplist(user.c_str(), static_cast(group_id), groups, &ngroups) == -1) { - ink_fatal("Failed to get group list as user '%s'\n", user.c_str()); - } - for (int i = 0; i < ngroups; i++) { - if (static_cast(path_gid) == groups[i]) { - return true; - } - } -#else - gid_t groups[ngroups]; - if (getgrouplist(user.c_str(), group_id, groups, &ngroups) == -1) { - ink_fatal("Failed to get group list as user '%s'\n", user.c_str()); + gid_type groups[ngroups]; + if (getgrouplist(user, group_id, groups, &ngroups) == -1) { + ink_warning("Unable to get group list as user '%s'\n", user); + return false; } for (int i = 0; i < ngroups; i++) { if (path_gid == groups[i]) { return true; } } -#endif return false; } // set permission to the map in verify runroot static void -set_permission(std::vector const &dir_vector, RunrootMapType &path_map, RunrootMapType &permission_map, - RunrootMapType &usergroup_map, std::string &user) +set_permission(PermissionMapType &permission_map, const struct passwd *pwd) { - // active group and user of the path - std::pair giduid = get_giduid(user); - - gid_t ts_gid = giduid.first; - uid_t ts_uid = giduid.second; - - struct stat stat_buffer; + gid_t ts_gid = pwd->pw_gid; + uid_t ts_uid = pwd->pw_uid; + bool new_line = false; // set up permission map for all permissions - for (auto &&it : dir_vector) { - std::string name = it; - std::string value = path_map[name]; - - if (name == LAYOUT_PREFIX || name == LAYOUT_EXEC_PREFIX) { - continue; - } + for (auto &it : permission_map) { + std::string name = it.first; + std::string value = it.second.path; + struct stat stat_buffer; if (stat(value.c_str(), &stat_buffer) < 0) { - ink_warning("unable to stat() destination path %s: %s", value.c_str(), strerror(errno)); + ink_warning("unable to stat() destination path %s - %s", value.c_str(), strerror(errno)); + it.second.result = false; + new_line = true; continue; } gid_t path_gid = stat_buffer.st_gid; uid_t path_uid = stat_buffer.st_uid; - permission_map[name] = "000"; // default rwx all 0 if (ts_uid == path_uid) { - // check for owner permission of st_mode - usergroup_map[name] = "owner"; - if (stat_buffer.st_mode & S_IRUSR) { - permission_map[name][0] = '1'; - } - if (stat_buffer.st_mode & S_IWUSR) { - permission_map[name][1] = '1'; - } - if (stat_buffer.st_mode & S_IXUSR) { - permission_map[name][2] = '1'; - } - } else if (from_group(user, ts_gid, path_gid)) { // current user is in the path_gid group - // check for group permission of st_mode - usergroup_map[name] = "group"; - if (stat_buffer.st_mode & S_IRGRP) { - permission_map[name][0] = '1'; - } - if (stat_buffer.st_mode & S_IWGRP) { - permission_map[name][1] = '1'; - } - if (stat_buffer.st_mode & S_IXGRP) { - permission_map[name][2] = '1'; - } + it.second.r_mode = S_IRUSR; + it.second.w_mode = S_IWUSR; + it.second.e_mode = S_IXUSR; + } else if (from_group(pwd->pw_name, ts_gid, path_gid)) { + it.second.r_mode = S_IRGRP; + it.second.w_mode = S_IWGRP; + it.second.e_mode = S_IXGRP; } else { - // check for others permission of st_mode - usergroup_map[name] = "others"; - if (stat_buffer.st_mode & S_IROTH) { - permission_map[name][0] = '1'; - } - if (stat_buffer.st_mode & S_IWOTH) { - permission_map[name][1] = '1'; + it.second.r_mode = S_IROTH; + it.second.w_mode = S_IWOTH; + it.second.e_mode = S_IXOTH; + } + // set the result in permission entry + permission_entry = it.second; + it.second.result = true; + // Special case with prefix, exec_prefix and localstatedir. We only need to check the directory itself + // but not the files within because they are just container dir for others. + if (name == LAYOUT_PREFIX || name == LAYOUT_EXEC_PREFIX || name == LAYOUT_LOCALSTATEDIR) { + if (!check_directory_permission(value.c_str())) { + it.second.result = false; + new_line = true; } - if (stat_buffer.st_mode & S_IXOTH) { - permission_map[name][2] = '1'; + } else { + // go through all files to check permission + if (ftw(value.c_str(), permission_handler, OPEN_MAX_FILE) != 0) { + it.second.result = false; + new_line = true; } } } + if (new_line) { + std::cout << std::endl; // print a new line after the failed permission messages + } +} + +// the fixing permission of runroot used by verify command +static void +fix_runroot(PermissionMapType &permission_map, const struct passwd *pwd) +{ + fix_flag = true; + for (const auto &it : permission_map) { + std::string name = it.first; + std::string value = it.second.path; + permission_entry = permission_map[name]; + ftw(value.c_str(), permission_handler, OPEN_MAX_FILE); + } + fix_flag = false; + set_permission(permission_map, pwd); } void LayoutEngine::verify_runroot() { + // require sudo permission for --fix + if (arguments.get("fix") && getuid() != 0) { + ink_error("To fix permission issues, root privilege is required.\nPlease run with sudo."); + status_code = EX_SOFTWARE; + return; + } + + // retrieve information std::string path = path_handler(arguments.get("path").value(), false, _argv[0]); std::string user; char user_buf[MAX_LOGIN + 1]; @@ -656,49 +667,53 @@ LayoutEngine::verify_runroot() user = TS_PKGSYSUSER; } } + // Numeric user notation for user + if (user[0] == '#') { + struct passwd *pwd = getpwuid(atoi(&user[1])); + if (!pwd) { + ink_error("No user found under id '%s'", user.c_str()); + status_code = EX_OSERR; + return; + } + user = pwd->pw_name; + } + std::cout << "Verifying permission as user: \x1b[1m" << user << "\x1b[0m" << std::endl << std::endl; + // try to find the user from password file + struct passwd *pwd = getpwnam(user.c_str()); + if (!pwd) { + ink_error("No user found under name '%s'", user.c_str()); + status_code = EX_OSERR; + return; + } // put paths from yaml file or default paths to path_map RunrootMapType path_map; if (path.empty()) { path_map = runroot_map_default(); - std::cout << "Verifying default build ..." << std::endl; } else { path_map = runroot_map(Layout::relative_to(path, "runroot.yaml")); } - - RunrootMapType permission_map; // contains rwx bits - RunrootMapType usergroup_map; // map: owner, group, others - - set_permission(dir_vector, path_map, permission_map, usergroup_map, user); - - std::cout << "Verifying as user: " << user << std::endl << std::endl; - - // if --fix is used - if (arguments.get("fix")) { - fix_runroot(path_map, permission_map, usergroup_map); - set_permission(dir_vector, path_map, permission_map, usergroup_map, user); - } else { - std::cout << "Note: This only shows directory permissions, please use '--fix' to correct permissions on all files" << std::endl; + // setup the permission map + PermissionMapType permission_map; + for (const auto &it : dir_vector) { + permission_map[it].name = it; + permission_map[it].path = path_map[it]; + } + // root always has full access and no need to check for root + if (user != "root") { + set_permission(permission_map, pwd); + // if --fix is used + if (arguments.get("fix")) { + fix_runroot(permission_map, pwd); + } } // display pass or fail for permission required - for (uint i = 2; i < dir_vector.size(); i++) { - std::string name = dir_vector[i]; - std::string permission = permission_map[dir_vector[i]]; - std::cout << name << ": \x1b[1m" + path_map[name] + "\x1b[0m" << std::endl; - - // output read permission - if (name == LAYOUT_LOCALSTATEDIR || name == LAYOUT_INCLUDEDIR || name == LAYOUT_SYSCONFDIR || name == LAYOUT_DATADIR) { - output_read_permission(permission); - } - // output write permission - if (name == LAYOUT_LOGDIR || name == LAYOUT_RUNTIMEDIR || name == LAYOUT_CACHEDIR) { - output_read_permission(permission); - output_write_permission(permission); - } - // output execute permission - if (name == LAYOUT_BINDIR || name == LAYOUT_SBINDIR || name == LAYOUT_LIBDIR || name == LAYOUT_LIBEXECDIR) { - output_read_permission(permission); - output_execute_permission(permission); + for (const auto &it : dir_vector) { + if (permission_map[it].result) { + std::cout << it << ": \x1b[1m" << path_map[it] << "\x1b[0m \033[1;32mPASSED\033[0m" << std::endl; + } else { + std::cout << it << ": \x1b[1m" << path_map[it] << "\x1b[0m \033[1;31mFAILED\033[0m" << std::endl; + status_code = EX_SOFTWARE; } } } diff --git a/src/traffic_layout/engine.h b/src/traffic_layout/engine.h index 9154f1b8358..769b6e71ca9 100644 --- a/src/traffic_layout/engine.h +++ b/src/traffic_layout/engine.h @@ -26,7 +26,21 @@ #include "tscore/ArgParser.h" #include -typedef std::unordered_map RunrootMapType; +// used by runroot verify +struct PermissionEntry { + std::string name; // sysconfdir, libdir ... + std::string path; // real path of the directory + // required permission + mode_t r_mode; + mode_t w_mode; + mode_t e_mode; + // result set by set_permission() + bool result = true; +}; + +// this map contain the corresponding permission information of directories +// PermissionEntry contains the read/write/execute mode and the result of output +using PermissionMapType = std::unordered_map; // structure for informaiton of the runroot passing around struct LayoutEngine { @@ -49,4 +63,6 @@ struct LayoutEngine { ts::Arguments arguments; // mordern argv std::vector _argv; + + int status_code = 0; }; diff --git a/src/traffic_layout/traffic_layout.cc b/src/traffic_layout/traffic_layout.cc index 72edd203327..fb26e343c3c 100644 --- a/src/traffic_layout/traffic_layout.cc +++ b/src/traffic_layout/traffic_layout.cc @@ -74,5 +74,5 @@ main(int argc, const char **argv) engine.arguments.invoke(); - return 0; + return engine.status_code; } diff --git a/src/tscore/ArgParser.cc b/src/tscore/ArgParser.cc index 6f3f477355d..47e8c2ebfc0 100644 --- a/src/tscore/ArgParser.cc +++ b/src/tscore/ArgParser.cc @@ -34,6 +34,10 @@ std::string global_usage; std::string parser_program_name; std::string default_command; +// by default return EX_USAGE(64) when usage is called. +// if -h or --help is called specifically, return 0 +int usage_return_code = EX_USAGE; + namespace ts { ArgParser::ArgParser() {} @@ -107,7 +111,7 @@ ArgParser::Command::help_message(std::string_view err) const std::cout << "\nExample Usage: " << _example_usage << std::endl; } // standard return code - exit(EX_USAGE); + exit(usage_return_code); } void @@ -436,6 +440,7 @@ ArgParser::Command::append_option_data(Arguments &ret, AP_StrVec &args, int inde } command = &it->second; } + usage_return_code = 0; command->help_message(); } // deal with normal --arg val1 val2 ... diff --git a/tests/gold_tests/runroot/runroot_init.test.py b/tests/gold_tests/runroot/runroot_init.test.py index ec8a2f673bb..5e8e371a644 100644 --- a/tests/gold_tests/runroot/runroot_init.test.py +++ b/tests/gold_tests/runroot/runroot_init.test.py @@ -59,7 +59,6 @@ tr.Processes.Default.ReturnCode = 0 f = tr.Disk.File(os.path.join(path4, "runroot.yaml")) f.Exists = True -tr.Processes.Default.Streams.All = Testers.ContainsExpression("Forcing creating runroot", "force message") # create runroot with junk to guarantee only traffic server related files are copied bin_path = Test.Variables.BINDIR[Test.Variables.BINDIR.find(Test.Variables.PREFIX) + len(Test.Variables.PREFIX) + 1:] diff --git a/tests/gold_tests/runroot/runroot_verify.test.py b/tests/gold_tests/runroot/runroot_verify.test.py index a9253b4229b..d98058c8041 100644 --- a/tests/gold_tests/runroot/runroot_verify.test.py +++ b/tests/gold_tests/runroot/runroot_verify.test.py @@ -42,12 +42,7 @@ os.path.join(path, "bin"), "example bindir output") tr.Processes.Default.Streams.All = Testers.ContainsExpression( os.path.join(path, "var/log/trafficserver"), "example logdir output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Read Permission: ", "read permission output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Execute Permission: ", "execute permission output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Write Permission: ", "write permission output") +tr.Processes.Default.Streams.All = Testers.ContainsExpression("PASSED", "contain passed message") # verify test #2 bin_path = Test.Variables.BINDIR[Test.Variables.BINDIR.find( @@ -60,9 +55,5 @@ os.path.join(path, "bin"), "example bindir output") tr.Processes.Default.Streams.All = Testers.ContainsExpression( os.path.join(path, "var/log/trafficserver"), "example logdir output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Read Permission: ", "read permission output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Execute Permission: ", "execute permission output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Write Permission: ", "write permission output") +tr.Processes.Default.Streams.All = Testers.ContainsExpression("PASSED", "contain passed message") + From 98be1f45a9206c8f388a01cd325d1594437caedd Mon Sep 17 00:00:00 2001 From: Emanuele Rocca Date: Mon, 19 Nov 2018 10:39:11 +0100 Subject: [PATCH 059/526] Do not reload remap.config when calling verify_config Calling `verify_config` should not swap the currently loaded configuration, but simply check whether the configuration files are valid. This fixes issue #4466 --- proxy/ReverseProxy.cc | 17 +++++++++++++++-- proxy/ReverseProxy.h | 1 + src/traffic_server/traffic_server.cc | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/proxy/ReverseProxy.cc b/proxy/ReverseProxy.cc index 81f18a5959b..99c7677c4fd 100644 --- a/proxy/ReverseProxy.cc +++ b/proxy/ReverseProxy.cc @@ -117,6 +117,12 @@ struct UR_UpdateContinuation : public Continuation { } }; +bool +urlRewriteVerify() +{ + return UrlRewrite().is_valid(); +} + /** Called when the remap.config file changes. Since it called infrequently, we do the load of new file as blocking I/O and lock aquire is also @@ -126,7 +132,7 @@ struct UR_UpdateContinuation : public Continuation { bool reloadUrlRewrite() { - UrlRewrite *newTable; + UrlRewrite *newTable, *oldTable; Debug("url_rewrite", "remap.config updated, reloading..."); newTable = new UrlRewrite(); @@ -136,7 +142,14 @@ reloadUrlRewrite() // Hold at least one lease, until we reload the configuration newTable->acquire(); - ink_atomic_swap(&rewrite_table, newTable)->release(); // Swap configurations, and release the old one + // Swap configurations + oldTable = ink_atomic_swap(&rewrite_table, newTable); + + ink_assert(oldTable != nullptr); + + // Release the old one + oldTable->release(); + Debug("url_rewrite", "%s", msg); Note("%s", msg); return true; diff --git a/proxy/ReverseProxy.h b/proxy/ReverseProxy.h index 0ba5bc3de0c..dd205cfdd04 100644 --- a/proxy/ReverseProxy.h +++ b/proxy/ReverseProxy.h @@ -56,5 +56,6 @@ bool response_url_remap(HTTPHdr *response_header, UrlRewrite *table); // Reload Functions bool reloadUrlRewrite(); +bool urlRewriteVerify(); int url_rewrite_CB(const char *name, RecDataT data_type, RecData data, void *cookie); diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 3080045a7a0..a69aeec49b9 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -848,7 +848,7 @@ cmd_verify(char * /* cmd ATS_UNUSED */) Layout::get()->update_sysconfdir(conf_dir); } - if (!reloadUrlRewrite()) { + if (!urlRewriteVerify()) { exitStatus |= (1 << 0); fprintf(stderr, "ERROR: Failed to load remap.config, exitStatus %d\n\n", exitStatus); } else { From 816f0d43744542a9f4e37d521d06f37ed082aed7 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 29 Nov 2018 18:02:43 -0600 Subject: [PATCH 060/526] Cleanup: This is just a stylistic cleanup of PR 4531. Mainly this is making the code more C++, removing a bit of duplication. --- proxy/http/remap/RemapPlugins.cc | 44 +++++++++++++------------------- proxy/http/remap/RemapPlugins.h | 13 +++------- 2 files changed, 22 insertions(+), 35 deletions(-) diff --git a/proxy/http/remap/RemapPlugins.cc b/proxy/http/remap/RemapPlugins.cc index 198f85d0470..cd71cb77917 100644 --- a/proxy/http/remap/RemapPlugins.cc +++ b/proxy/http/remap/RemapPlugins.cc @@ -84,7 +84,7 @@ RemapPlugins::run_single_remap() url_mapping *map = _s->url_map.getMapping(); remap_plugin_info *plugin = map->get_plugin(_cur); // get the nth plugin in our list of plugins TSRemapStatus plugin_retcode = TSREMAP_NO_REMAP; - + int zret = 1; Debug("url_rewrite", "running single remap rule id %d for the %d%s time", map->map_id, _cur, _cur == 1 ? "st" : _cur == 2 ? "nd" : _cur == 3 ? "rd" : "th"); @@ -94,38 +94,30 @@ RemapPlugins::run_single_remap() plugin_retcode = run_plugin(plugin); } - _cur++; - - // If the plugin redirected, we need to end the remap chain now. - if (_s->remap_redirect) { - return 1; - } + ++_cur; - if (TSREMAP_DID_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP == plugin_retcode) { - _rewriten++; - } + // If the plugin redirected, we need to end the remap chain now. Otherwise see what's next. + if (!_s->remap_redirect) { + if (TSREMAP_DID_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP == plugin_retcode) { + ++_rewritten; + } - if (TSREMAP_NO_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP_STOP == plugin_retcode) { - if (_rewriten == 0) { - Debug("url_rewrite", "plugin did not change host, port or path, copying from mapping rule"); - url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); + if (TSREMAP_NO_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP_STOP == plugin_retcode) { + Debug("url_rewrite", "breaking remap plugin chain since last plugin said we should stop after %d rewrites", _rewritten); + } else if (_cur >= map->plugin_count()) { + Debug("url_rewrite", "completed all remap plugins for rule id %d, changed by %d plugins", map->map_id, _rewritten); + } else { + Debug("url_rewrite", "completed single remap, attempting another via immediate callback"); + zret = 0; // not done yet. } - Debug("url_rewrite", "breaking remap plugin chain since last plugin said we should stop"); - return 1; - } - if (_cur >= map->plugin_count()) { - if (_rewriten == 0) { - Debug("url_rewrite", "plugin did not change host, port or path, copying from mapping rule"); + // If the chain is finished, and the URL hasn't been rewritten, do the rule remap. + if (zret && 0 == _rewritten) { + Debug("url_rewrite", "plugins did not change host, port or path, copying from mapping rule"); url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); } - // Normally, we would callback into this function but we dont have anything more to do! - Debug("url_rewrite", "completed all remap plugins for rule id %d", map->map_id); - return 1; } - - Debug("url_rewrite", "completed single remap, attempting another via immediate callback"); - return 0; + return zret; } int diff --git a/proxy/http/remap/RemapPlugins.h b/proxy/http/remap/RemapPlugins.h index ed7d55d8574..fad40976e12 100644 --- a/proxy/http/remap/RemapPlugins.h +++ b/proxy/http/remap/RemapPlugins.h @@ -38,17 +38,12 @@ * A class that represents a queue of plugins to run **/ struct RemapPlugins : public Continuation { - RemapPlugins() : _cur(0), _rewriten(0) {} + RemapPlugins() = default; RemapPlugins(HttpTransact::State *s, URL *u, HTTPHdr *h, host_hdr_info *hi) - : _cur(0), _rewriten(0), _s(s), _request_url(u), _request_header(h), _hh_ptr(hi) + : _s(s), _request_url(u), _request_header(h), _hh_ptr(hi) { } - ~RemapPlugins() override - { - _cur = 0; - _rewriten = 0; - } // Some basic setters void setState(HttpTransact::State *state) @@ -78,8 +73,8 @@ struct RemapPlugins : public Continuation { Action action; private: - unsigned int _cur = 0; - unsigned int _rewriten = 0; + unsigned _cur = 0; + unsigned _rewritten = 0; HttpTransact::State *_s = nullptr; URL *_request_url = nullptr; HTTPHdr *_request_header = nullptr; From b156314dd8d7c6f918a3154b33340a80bc0dcf95 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 28 Nov 2018 13:19:38 -0600 Subject: [PATCH 061/526] TLS Bridge: Fix error where connect failure lead to a silent timeout. The error reporting back to the original user agent was also improved. The documentation was updated with more details on configuring without disabling remap required. --- .../plugins/example-plugins/tls_bridge.en.rst | 84 ++++++++++++------- plugins/experimental/tls_bridge/tls_bridge.cc | 39 +++------ 2 files changed, 67 insertions(+), 56 deletions(-) diff --git a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst index 0530afa194e..85836c220e0 100644 --- a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst +++ b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst @@ -20,10 +20,10 @@ |Name| ********** -This plugin is used to provide secured TLS tunnels for connections between a Client and a Service -via two gateway |TS| instances. By configuring the |TS| instances the level of security in the -tunnel can be easily controlled for all communications across the tunnels without having to update -the client or service. +This plugin is used to provide TLS tunnels for connections between a Client and a Service via two +gateway |TS| instances using explicit proxying. By configuring the |TS| instances the level of +security in the tunnel can be easily controlled for all communications across the tunnels without +having to update the client or service. Description =========== @@ -47,24 +47,26 @@ The tunnel is sustained by two instances of |TS|. [Ingress ATS] ..> [tls_bridge\nPlugin] : Uses -The ingress |TS| accepts a connection from the Client. This connection gets intercepted by the -|Name| plugin inside |TS|. The plugin then makes a TLS connection to the peer |TS| using the -configured level of security. The original request from the Client to the ingress |TS| is then sent -to the peer |TS| to create a connection from the peer |TS| to the Service. After this the -Client has a virtual circut to the Service and can use any TCP based communication (including TLS). -Effectively the plugin causes the connectivity to work as if the Client had done the ``CONNECT`` -directly to the peer |TS|. Note this means the DNS lookup for the Service is done by the peer |TS|, -not the ingress |TS|. +The ingress |TS| accepts an HTTP ``CONNECT`` request from the Client. This connection gets +intercepted by the |Name| plugin inside |TS| if the destination matches one of the configured +destinations. The plugin then makes a TLS connection to the peer |TS| using the configured level of +security. The original ``CONNECT`` request from the Client to the ingress |TS| is then sent to the +peer |TS| to create a connection from the peer |TS| to the Service. After this the Client has a +virtual circut to the Service and can use any TCP based communication (including TLS). Effectively +the plugin causes the explicit proxy to work as if the Client had done the ``CONNECT`` directly to +the peer |TS|. Note this means the DNS lookup for the Service is done by the peer |TS|, not the +ingress |TS|. -The plugin is configured with a mapping of Service names to peer |TS| instances. The Service -names are URLs which will in the original HTTP request made by the Client after connecting to the -ingress |TS|. This means the FQDN for the Service is not resolved in the environment of the peer +The plugin is configured with a mapping of Service names to peer |TS| instances. The Service names +are URLs which will be in the original HTTP request made by the Client after connecting to the +ingress |TS|. This means the FQDN for the Service is resolved in the environment of the peer |TS| and not the ingress |TS|. Configuration ============= -|Name| requires at least two instances of |TS| (Ingress and Peer). +|Name| requires at least two instances of |TS| (Ingress and Peer). The client connects to the +ingress |TS|, and the peer |TS| connects to the service. #. Disable caching on |TS| in ``records.config``:: @@ -72,32 +74,46 @@ Configuration #. Configure the ports. - * The Peer |TS| must be listening on an SSL enabled proxy port. For instance, if the proxy port for the Peer is 4443, then configuration in ``records.config`` would have:: + * The Peer |TS| must be listening on an SSL enabled proxy port. For instance, if the proxy port + for the Peer is 4443, then configuration in ``records.config`` would have:: CONFIG proxy.config.http.server_ports STRING 4443:ssl - * The Ingress |TS| must allow ``CONNECT`` to the Peer proxy port. This would be set in ``records.config`` by:: + * The Ingress |TS| must allow ``CONNECT`` to the Peer proxy port. This would be set in + ``records.config`` by:: CONFIG proxy.config.http.connect_ports STRING 4443 The Ingress |TS| also needs ``proxy.config.http.server_ports`` configured to have proxy ports to which the Client can connect. -#. Remap is not required, however, |TS| requires remap in order to accept the request. This can be done by disabling the remap requirement:: +#. Remap on the ingress is not required, however, |TS| requires remap in order to accept the + request. This can be done by disabling the remap requirement:: CONFIG proxy.config.url_remap.remap_required INT 0 - In this case |TS| will act as an open proxy which is unlikely to be a good idea. |TS| will need - to run in a restricted environment or use access control (via ``ip_allow.config`` or - ``iptables``). + In this case |TS| will act as an open proxy which is unlikely to be a good idea, therefore if + this approach is used |TS| will need to run in a restricted environment or use access control + (via ``ip_allow.config`` or ``iptables``). - If this is unsuitable then an identity remap rule can be added for the peer |TS|. If the peer |TS| - was named "peer.ats" then the remap rule would be :: + If this is unsuitable then an identity remap rule can be added for the peer |TS|. If the peer + |TS| was named "peer.ats" and it listens on port 4443, then the remap rule would be :: - remap https://peer.ats https://peer.ats + remap https://peer.ats:4443 https://peer.ats:4443 Remapping will be disabled for the user agent connection and so it will not need a rule. +#. If remap is required on the peer to enable the outbound connection from the peer to the service + (it is not explicitly disabled) the destination port must be + explicitly stated [#]_. E.g. :: + + map https://service:4443 https://service:4443 + + Note this remap rule cannot alter the actual HTTP transactions between the client and service + because those happen inside what is effectively a tunnel between the client and service, supported + by the two |TS| instances. This rule just allows the ``CONNECT`` sent from the ingress to cause a + tunnel connection from the peer to the service. + #. Configure the Ingress |TS| to verify the Peer server certificate:: CONFIG proxy.config.ssl.client.verify.server.policy STRING ENFORCED @@ -119,16 +135,21 @@ Configuration #. Enable the |Name| plugin in ``plugin.config``. The plugin is configured by arguments in ``plugin.config``. These are arguments are in pairs of a *destination* and a *peer*. The destination is an anchored regular expression which is matched against the host name in the Client - ``CONNECT``. The destinations are checked in order and the first match is used to select the Peer + ``CONNECT``. The destinations are checked in order and the first match is used to select the peer |TS|. The peer should be an FQDN or IP address with an optional port. For the example above, if - the Peer |TS| was named "peer.example.com" on port 4443 and the Service at ``*.service.com``, the - peer argument would be "peer.example.com:4443". In ``plugin.config`` this would be:: + the Peer |TS| was named "peer.ats" on port 4443 and the Service at ``*.service.com``, the + peer argument would be "peer.ats:4443". In ``plugin.config`` this would be:: - tls_bridge.so .*[.]service[.]com peer.example.com:4443 + tls_bridge.so .*[.]service[.]com peer.ats:4443 Note the '.' characters are escaped with brackets so that, for instance, "someservice.com" does not match the rule. + If there was another service, "\*.altsvc.ats", via a different peer "altpeer.ats" on port 4443, + the configuration would be :: + + tls_bridge.so .*[.]service[.]com peer.ats:4443 .*[.]altsvc.ats altpeer.ats:4443 + Notes ===== @@ -255,3 +276,8 @@ Debugging --------- Debugging messages for the plugin can be enabled with the "tls_bridge" debug tag. + + +.. rubric:: Footnotes + +.. [#] This is likely due to a bug in |TS|, currently under investigation. diff --git a/plugins/experimental/tls_bridge/tls_bridge.cc b/plugins/experimental/tls_bridge/tls_bridge.cc index 94657ac1136..7cfe1e52e8f 100644 --- a/plugins/experimental/tls_bridge/tls_bridge.cc +++ b/plugins/experimental/tls_bridge/tls_bridge.cc @@ -186,8 +186,6 @@ struct Bridge { TSHttpStatus _out_response_code = TS_HTTP_STATUS_NONE; /// Response reason, if not TS_HTTP_STATUS_OK std::string _out_response_reason; - /// Is the response to the user agent suspended? - bool _ua_response_suspended = false; /// Bridge requires a continuation for scheduling and the transaction. Bridge(TSCont cont, TSHttpTxn txn, TextView peer); @@ -231,7 +229,7 @@ Bridge::net_accept(TSVConn vc) char buff[1024]; int64_t n = snprintf(buff, sizeof(buff), CONNECT_FORMAT, static_cast(_peer.size()), _peer.data()); - TSDebug(PLUGIN_TAG, "Received UA VConn"); + TSDebug(PLUGIN_TAG, "Received UA VConn, connecting to peer %.*s", int(_peer.size()), _peer.data()); // UA side intercepted. _ua.init(vc); _ua.do_read(_self_cont, INT64_MAX); @@ -315,12 +313,6 @@ Bridge::check_outbound_OK() } // 519 is POOMA, useful for debugging, but may want to change this later. _out_response_code = c ? c : static_cast(519); - if (_ua_response_suspended) { - this->update_ua_response(); - TSHttpTxnReenable(_ua_txn, TS_EVENT_HTTP_CONTINUE); - _ua_response_suspended = false; - TSDebug(PLUGIN_TAG, "TXN resumed"); - } _out.consume(block.data() - raw.data()); zret = true; TSDebug(PLUGIN_TAG, "Outbound status %d", c); @@ -415,24 +407,19 @@ Bridge::eos(TSVIO vio) } _out.do_close(); _ua.do_close(); - _out_resp_state = EOS; - if (_ua_response_suspended) { - TSHttpTxnReenable(_ua_txn, TS_EVENT_HTTP_CONTINUE); + if (_out_resp_state != ERROR) { + _out_resp_state = EOS; } } void Bridge::send_response_cb() { - // If the upstream response hasn't been parsed yet, make the UA response wait for that. - // Set a flag so the upstream response parser knows to update response and reenable. - if (_out_resp_state < OK) { - _ua_response_suspended = true; - TSDebug(PLUGIN_TAG, "TXN suspended"); - } else { // Already have all the data needed to do the update, so do it and move on. - this->update_ua_response(); - TSHttpTxnReenable(_ua_txn, TS_EVENT_HTTP_CONTINUE); - } + // This happens either after the upstream connection and the writing the response there, + // or because the upstream connection was blocked. In either case the upstream work is + // done and the original transaction can proceed. + this->update_ua_response(); + TSHttpTxnReenable(_ua_txn, TS_EVENT_HTTP_CONTINUE); } void @@ -441,12 +428,10 @@ Bridge::update_ua_response() TSMBuffer mbuf; TSMLoc hdr_loc; if (TS_SUCCESS == TSHttpTxnClientRespGet(_ua_txn, &mbuf, &hdr_loc)) { - // A 200 for @a out_response_code only means there wasn't an internal failure on the upstream - // CONNECT. Network and other failures get reported in this response. This response code will - // be more accurate, so use it unless it's 200, in which case use the stored response code if - // that's not 200. - TSHttpStatus status = TSHttpHdrStatusGet(mbuf, hdr_loc); - if (TS_HTTP_STATUS_OK == status && TS_HTTP_STATUS_OK != _out_response_code) { + // If there is a non-200 upstream code then that's the most accurate because it was from + // an actual upstream connection. Otherwise, let the original connection response code + // ride. + if (_out_response_code != TS_HTTP_STATUS_OK && _out_response_code != TS_HTTP_STATUS_NONE) { TSHttpHdrStatusSet(mbuf, hdr_loc, _out_response_code); if (!_out_response_reason.empty()) { TSHttpHdrReasonSet(mbuf, hdr_loc, _out_response_reason.data(), _out_response_reason.size()); From 6bd38f602f9cf7f164466daeb24f256180ab4cbf Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 28 Nov 2018 16:27:37 -0600 Subject: [PATCH 062/526] Doc: TSHttpHdrStatusGet. --- .../api/functions/TSHttpHdrStatusGet.en.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst index 50b2f1da2c6..7d2e31dbfbe 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst @@ -30,3 +30,13 @@ Synopsis Description =========== + +Retrieve the status code value from the HTTP response header identified by :arg:`bufp` and +:arg:`offset`. The value should be an enumeration value of :c:type:`TSHttpStatus`, although because +plugins can call :c:func:`TSHttpHdrStatusSet` this may not be true. If the header is not a valid +response then :c:macro:`TS_HTTP_STATUS_NONE` is returned. + +See Also +======== + +:c:func:`TSHttpTxnClientRespGet`, :c:func:`TSHttpTxnServerRespGet`, :c:func:`TSHttpHdrTypeGet`. From fa79d91fe9f0dac9e062954a538698b5a8633512 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Tue, 27 Nov 2018 14:25:29 -0600 Subject: [PATCH 063/526] Changes external links to internal --- .../plugins/example-plugins/query_remap/index.en.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/developer-guide/plugins/example-plugins/query_remap/index.en.rst b/doc/developer-guide/plugins/example-plugins/query_remap/index.en.rst index c2a31d6dae5..5e046f88ca5 100644 --- a/doc/developer-guide/plugins/example-plugins/query_remap/index.en.rst +++ b/doc/developer-guide/plugins/example-plugins/query_remap/index.en.rst @@ -53,16 +53,16 @@ Required Functions A remap plugin is required to implement the following functions: -- `TSRemapInit `_: +- :c:func:`TSRemapInit`: the remap initialization function, called once when the plugin is loaded -- `TSRemapNewInstance `_: +- :c:func:`TSRemapNewInstance`: a new instance is created for each rule associated with the plugin. Called each time the plugin used in a remap rule (this function is what processes the pparam values) -- `TSRemapDoRemap `_: +- :c:func:`TSRemapDoRemap`: the entry point used by Traffic Server to find the new URL to which it remaps; called every time a request comes in From 1ef8b2ad64f248db1a31d1eebaed4b011a4669b5 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Fri, 30 Nov 2018 15:13:30 -0600 Subject: [PATCH 064/526] Runroot: Add a new doc on runroot for programs --- .../command-line/traffic_layout.en.rst | 12 +-- doc/developer-guide/index.en.rst | 3 +- doc/developer-guide/layout/index.en.rst | 28 ++++++ doc/developer-guide/layout/runroot.en.rst | 99 +++++++++++++++++++ src/traffic_layout/README | 2 +- 5 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 doc/developer-guide/layout/index.en.rst create mode 100644 doc/developer-guide/layout/runroot.en.rst diff --git a/doc/appendices/command-line/traffic_layout.en.rst b/doc/appendices/command-line/traffic_layout.en.rst index 0898443f7ae..443c8235823 100644 --- a/doc/appendices/command-line/traffic_layout.en.rst +++ b/doc/appendices/command-line/traffic_layout.en.rst @@ -21,13 +21,11 @@ traffic_layout ************** -======== Synopsis ======== :program:`traffic_layout` SUBCOMMAND [OPTIONS] -=========== Environment =========== @@ -35,15 +33,13 @@ Environment The path to the run root file. It has the same effect as the command line option :option:`--run-root`. -=========== Description =========== -Document for the special functionality of ``runroot`` inside :program:`traffic_layout`. This feature -is for the setup of traffic server runroot. It will create a runtime sandbox for any program of -traffic server to run under. +Document for the :program:`traffic_layout` about the ``runroot``. This feature is for the setup of traffic server runroot. +It will create a runtime sandbox for any program of traffic server to run under. +For details about runroot for programs, please refer to ``developer-guide/layout/runroot.en``. -===== Usage ===== @@ -65,7 +61,6 @@ For example, to run traffic_manager, using the runroot, there are several ways: if none of the above is found as runroot, runroot will not be used and the program will fall back to the default. -=========== Subcommands =========== @@ -134,7 +129,6 @@ Example: :: ``--fix`` may perform permission changes on the system configuration files. With normally created runroot with default layout, there is no such issue since traffic server related files are filtered. -======= Options ======= diff --git a/doc/developer-guide/index.en.rst b/doc/developer-guide/index.en.rst index 688203c0a17..5deda01f4b6 100644 --- a/doc/developer-guide/index.en.rst +++ b/doc/developer-guide/index.en.rst @@ -54,4 +54,5 @@ duplicate bugs is encouraged, but not required. documentation/index.en host-resolution-proposal.en client-session-architecture.en - core-architecture/index.en \ No newline at end of file + core-architecture/index.en + layout/index.en \ No newline at end of file diff --git a/doc/developer-guide/layout/index.en.rst b/doc/developer-guide/layout/index.en.rst new file mode 100644 index 00000000000..4e17aebf324 --- /dev/null +++ b/doc/developer-guide/layout/index.en.rst @@ -0,0 +1,28 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _layout: + +Layout +****** + +.. toctree:: + :maxdepth: 2 + + runroot.en diff --git a/doc/developer-guide/layout/runroot.en.rst b/doc/developer-guide/layout/runroot.en.rst new file mode 100644 index 00000000000..7b9c049d169 --- /dev/null +++ b/doc/developer-guide/layout/runroot.en.rst @@ -0,0 +1,99 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _runroot: + +Runroot +******* + +Preface +======= + +Runroot is a powerful feature to detect and define layout at runtime. This document helps to guide through how runroot works. +For management and setup of runroots, please refer to ``appendices/command-line/traffic_layout.en`` + +Why do we need runroot +====================== + +Runroot is a replacing approach for the previous ``TS_ROOT`` logic. ``TS_ROOT`` is based on replacing a compile-time package install root location. +Layouts for many systems have data location that are absolute paths that are defined without ``$PREFIX`` variable. So, the current logic is difficult to follow +and not consistent between trafficserver programs. Furthermore, it is not easy to modify subdirectory locations. + +So, we have the runroot which makes ATS easier to use, develop and deploy. + +Main logic +========== + +Everything in runroot will go through the class:`Layout` class and all the layout is defined by a single YAML file: ``runroot.yaml``. + +Work flow: + +#. Command line option ``--run-root`` +#. ``$TS_RUNROOT`` Environment variable +#. Look in current directory and look up N (default 4) directories for ``runroot.yaml`` +#. Look in executable directory and look up N directories for ``runroot.yaml``. +#. ``$TS_ROOT`` Environment Variable +#. Compiler defaults in layout class + +Right now, the following programs are integrated with the runroot logic: +**traffic_server**, **traffic_manager**, **traffic_ctl**, **traffic_layout**, **traffic_crashlog**, **traffic_logcat**, **traffic_logstat**. + +The YAML file +============= + +The runroot file (runroot.yaml) can be placed at any location in the filesystem and still be used to define the locations for required |TS| files. +So we can run traffic_server, for example, by ``traffic_server --run-root=/some/directory/runroot.yaml``. + +Below is an example of ``runroot.yaml``. + +.. code-block:: yaml + + prefix: /home/myname/runroot + exec_prefix: /home/myname/runroot + bindir: /home/myname/runroot/bin + sbindir: /home/myname/runroot/bin + sysconfdir: /etc/trafficserver + datadir: /home/myname/runroot/share/trafficserver + includedir: /home/myname/runroot/include + libdir: /home/myname/runroot/lib + libexecdir: /libexec/trafficserver + localstatedir: /var + runtimedir: /var/trafficserver + logdir: /home/myname/runroot/logdir + cachedir: /var/trafficserver + +The path can be both relative or absolute. We can define wherever we want the directory to be. All the items we need to +put into ``runroot.yaml`` are shown above and the entries can be optional. For example, if sysconfdir is not in the file, runroot will +set the sysconfidir, at runtime, to be the default built time sysconfdir concatenated with the prefix. + +Runroot management +================== + +We can create, remove and verify runroot using ``traffic_layout`` program. It is fully documented in the appendices. + +Guide for development +===================== + +Basic runroot functionality is handled in ``runroot.cc`` and ``runroot.h`` of ``lib/tscore``. ``runroot_handler()`` and ``argparser_runroot_handler()`` +are the main methods for the runroot. The ``Layout`` class will then get the global variable from ``runroot.cc`` to set up the directories properly. + +Issue or bug +============ + +This functionality should be stable and if there is any issue or bug, please report it on the GitHub and @chitianhao. diff --git a/src/traffic_layout/README b/src/traffic_layout/README index 0c65d6164b3..c7b88b8bfb0 100644 --- a/src/traffic_layout/README +++ b/src/traffic_layout/README @@ -2,4 +2,4 @@ traffic_layout is a easy-to-use tool to: - show the location of various installation paths and resources. - deal with traffic server runroot. -For detailed information, please refer to doc/appendices/command-line/traffic_layout.en.rst. +For detailed information, please refer to doc/appendices/command-line/traffic_layout.en and doc/developer-guide/layout/runroot.en. From cbe34550940d0d68061bcdbae2124ab44ae6c43d Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 3 Dec 2018 12:32:43 -0600 Subject: [PATCH 065/526] TLS Bridge: Add "--file" option to configuration. --- .../plugins/example-plugins/tls_bridge.en.rst | 38 +++++++- plugins/experimental/tls_bridge/tls_bridge.cc | 90 +++++++++++++++++-- 2 files changed, 118 insertions(+), 10 deletions(-) diff --git a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst index 85836c220e0..57a4661d7cd 100644 --- a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst +++ b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst @@ -13,7 +13,7 @@ .. include:: ../../../common.defs -.. highlight:: cpp +.. highlight:: text .. default-domain:: cpp .. |Name| replace:: TLS Bridge @@ -150,6 +150,42 @@ ingress |TS|, and the peer |TS| connects to the service. tls_bridge.so .*[.]service[.]com peer.ats:4443 .*[.]altsvc.ats altpeer.ats:4443 + Mappings can also be specified in an external file. For instance, if the file named "bridge.config" in the default |TS| + configuration directory contained mappings, the ``plugin.config`` configuration line could look + like :: + + tls_bridge.so .*[.]service[.]com peer.ats:4443 --file bridge.config + + or + + tls_bridge.so --file bridge.config .*[.]service[.]com peer.ats:4443 + + These are not identical - direct mappings and file mappings are processed in order. This means in + the first example, the direct mapping is checked before any mappping in "bridge.config", and in + the latter example the mappings in "bridge.config" are checked before the direct mapping. There + can be multiple "--file" arguments, which are processed in the order they appear in + "plugin.config". The file name can be absolute, or relative. If the file name is relative, it is + relative to the |TS| configuration directory. Therefore, in these examples, "bridge.config" must + be in the same directory as ``plugin.config``. + + The contents of "bridge.config" must be one mapping per line, with a regular expression separated + by white space from the destination service. This is identical to the format in ``plugin.config`` + except there is only one pair per line. E.g., valid content for "bridge.config" could be :: + + # Primary service location. + .*[.]service[.]com peer.ats:4443 + + # Secondary. + .*[.]altsvc.ats altpeer.ats:4443 + + Leading whitespace on a line is ignored, and if the first non-whitespace character is '#' then + the entire line is ignored. Therefore if that is the content of "bridge.config", these two + lines in "plugin.config" would behave identically :: + + tls_bridge.so --file bridge.config + + tls_bridge.so .*[.]service[.]com peer.ats:4443 .*[.]altsvc.ats altpeer.ats:4443 + Notes ===== diff --git a/plugins/experimental/tls_bridge/tls_bridge.cc b/plugins/experimental/tls_bridge/tls_bridge.cc index 7cfe1e52e8f..368a808cfac 100644 --- a/plugins/experimental/tls_bridge/tls_bridge.cc +++ b/plugins/experimental/tls_bridge/tls_bridge.cc @@ -21,6 +21,7 @@ #include #include #include "tscpp/util/TextView.h" +#include "tscore/ts_file.h" #include "regex.h" #define PLUGIN_NAME "TLS Bridge" @@ -28,10 +29,15 @@ using ts::TextView; +namespace +{ // Base format string for making the internal CONNECT. char const CONNECT_FORMAT[] = "CONNECT https:%.*s HTTP/1.1\r\n\r\n"; const TextView METHOD_CONNECT{TS_HTTP_METHOD_CONNECT, TS_HTTP_LEN_CONNECT}; +constexpr TextView CONFIG_FILE_ARG{"--file"}; +const std::string TS_CONFIG_DIR{TSConfigDirGet()}; +}; // namespace /* ------------------------------------------------------------------------------------ */ // Utility functions @@ -64,11 +70,11 @@ class BridgeConfig using self_type = BridgeConfig; /// Construct an item. - Item(const char *pattern, Regex &&r, const char *dest) : _pattern(pattern), _r(std::move(r)), _dest(dest) {} + Item(std::string_view pattern, Regex &&r, std::string_view service) : _pattern(pattern), _r(std::move(r)), _service(service) {} std::string _pattern; ///< Original configuration regular expression. Regex _r; ///< Compiled regex. - std::string _dest; ///< Destination if matched. + std::string _service; ///< Destination service if matched. }; public: @@ -83,6 +89,14 @@ class BridgeConfig private: /// Configuration item storage. std::vector _items; + + /** Load a configuration item pair. + * + * @param rxp The regular expression to match. + * @param service The destination service. + * @param ln Line number, or 0 if from plugin.config. + */ + void load_pair(std::string_view rxp, std::string_view service, ts::file::path const &src, int ln = 0); }; inline int @@ -91,18 +105,76 @@ BridgeConfig::count() const return _items.size(); } +void +BridgeConfig::load_pair(std::string_view rxp, std::string_view service, ts::file::path const &src, int ln) +{ + Regex r; + if (r.compile(rxp.data(), Regex::ANCHORED)) { + _items.emplace_back(rxp, std::move(r), service); + } else { + char buff[std::numeric_limits::digits10 + 2]; + char const *place = ""; + if (ln) { + snprintf(buff, sizeof(buff), " on line %d", ln); + place = buff; + } + TSError("%s: Failed to compile regular expression '%.*s' in %s%s", PLUGIN_TAG, int(rxp.size()), rxp.data(), src.c_str(), place); + } +} + void BridgeConfig::load_config(int argc, const char *argv[]) { + static const ts::file::path plugin_config_fp{"plugin.config"}; + for (int i = 0; i < argc; i += 2) { - Regex r; - if (i + 1 >= argc) { - TSError("%s: Destination regular expression without peer", PLUGIN_TAG); + if (argv[i] == CONFIG_FILE_ARG) { + if (i + 1 >= argc) { + TSError("%s: Invalid '%.*s' argument - no file name found.", PLUGIN_TAG, int(CONFIG_FILE_ARG.size()), + CONFIG_FILE_ARG.data()); + } else { + ts::file::path fp(argv[i + 1]); + std::error_code ec; + if (!fp.is_absolute()) { + fp = ts::file::path{TS_CONFIG_DIR} / fp; // slap the config dir on it to make it absolute. + } + // bulk load the file. + std::string content{ts::file::load(fp, ec)}; + if (ec) { + TSError("%s: Invalid '%.*s' argument - unable to read file '%s' : %s.", PLUGIN_TAG, int(CONFIG_FILE_ARG.size()), + CONFIG_FILE_ARG.data(), fp.c_str(), ec.message().c_str()); + + } else { + // walk the lines. + int line_no = 0; + TextView src{content}; + while (!src.empty()) { + TextView line{src.take_prefix_at('\n').trim_if(&isspace)}; + ++line_no; + if (line.empty() || '#' == *line) + continue; // empty or comment, ignore. + + // Pick apart the line into the regular expression and destination service. + TextView service{line}; + TextView rxp{service.take_prefix_if(&isspace)}; + service.ltrim_if(&isspace); // dump extra separating space. + // Only need to check service, as if the line isn't empty rxp will also be non-empty. + if (service.empty()) { + TSError("%s: Invalid line %d in '%s' - no destination service found.", PLUGIN_TAG, line_no, fp.c_str()); + } else { + this->load_pair(rxp, service, fp, line_no); + } + } + } + } + } else if (argv[i][0] == '-') { + TSError("%s: Unrecognized option '%s'", PLUGIN_TAG, argv[i]); + i -= 1; // Don't skip next arg. } else { - if (r.compile(argv[i]), Regex::ANCHORED) { - _items.emplace_back(argv[i], std::move(r), argv[i + 1]); + if (i + 1 >= argc) { + TSError("%s: Regular expression '%s' without destination service", PLUGIN_TAG, argv[i]); } else { - TSError("%s: Failed to compile regular expression '%s'", PLUGIN_TAG, argv[i]); + this->load_pair(argv[i], argv[i + 1], plugin_config_fp); } } } @@ -113,7 +185,7 @@ BridgeConfig::match(TextView name) { for (auto &item : _items) { if (item._r.exec(name)) { - return {item._dest}; + return {item._service}; } } return {}; From 1309ee2629a5e2b983bd69ae777750a013ec605e Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 3 Dec 2018 16:58:24 -0800 Subject: [PATCH 066/526] Prevent linking everything against brotli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defines an empty function for "action-if-found" to prevent default autoconf behavior (prepending -llibrary to LIBS and defining ‘HAVE_LIBlibrary’) --- build/brotli.m4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/brotli.m4 b/build/brotli.m4 index a3189b6bc52..b9c02610584 100644 --- a/build/brotli.m4 +++ b/build/brotli.m4 @@ -77,7 +77,7 @@ fi ], [ AC_CHECK_HEADER([brotli/encode.h], [], [has_brotli=0]) -AC_CHECK_LIB([brotlienc], BrotliEncoderCreateInstance, [], [has_brotli=0]) +AC_CHECK_LIB([brotlienc], BrotliEncoderCreateInstance, [:], [has_brotli=0]) if test "x$has_brotli" == "x0"; then PKG_CHECK_EXISTS([LIBBROTLIENC], From e215d4dc5d0179c34f17f3bf105ee8519aa7dd9a Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 4 Dec 2018 15:31:09 +0000 Subject: [PATCH 067/526] Change serverName member back to const char * to avoid crash. --- iocore/net/P_SSLNetVConnection.h | 6 +++++- iocore/net/SSLUtils.cc | 8 +++++--- proxy/http/HttpSM.cc | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index e2b7fdc72a4..144cf3749c5 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -361,7 +361,11 @@ class SSLNetVConnection : public UnixNetVConnection ink_hrtime sslHandshakeEndTime = 0; ink_hrtime sslLastWriteTime = 0; int64_t sslTotalBytesSent = 0; - std::string serverName; + // The serverName is either a pointer to the name fetched from the + // SSL object or the empty string. Therefore, we do not allocate + // extra memory for this value. If plugins in the future can set the + // serverName value, this strategy will have to change. + const char *serverName = nullptr; /// Set by asynchronous hooks to request a specific operation. SslVConnOp hookOpRequested = SSL_HOOK_OP_DEFAULT; diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index f68c522b936..2df2374cedf 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -472,9 +472,11 @@ ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) SSLNetVConnection *netvc = SSLNetVCAccess(ssl); netvc->callHooks(TS_EVENT_SSL_SERVERNAME); - const char *name = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); - netvc->serverName = std::string{name ? name : ""}; - int ret = PerformAction(netvc, netvc->serverName.c_str()); + netvc->serverName = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + if (nullptr == netvc->serverName) { + netvc->serverName = ""; + } + int ret = PerformAction(netvc, netvc->serverName); if (ret != SSL_TLSEXT_ERR_OK) { return SSL_TLSEXT_ERR_ALERT_FATAL; } diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index b5289323f4f..52d4dde2390 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -583,7 +583,7 @@ HttpSM::setup_blind_tunnel_port() t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } else { - t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName.c_str(), ssl_vc->serverName.length()); + t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } @@ -1394,7 +1394,7 @@ plugins required to work with sni_routing. t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } else if (ssl_vc) { - t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName.data(), ssl_vc->serverName.length()); + t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } From 6ee5c616d165b969b42a86c4eb2ca0b1be86b881 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 3 Dec 2018 23:24:38 +0000 Subject: [PATCH 068/526] Remove the ssl wire trace feature. --- doc/admin-guide/files/records.config.en.rst | 19 ---- iocore/net/P_SSLConfig.h | 7 -- iocore/net/P_SSLNetVConnection.h | 13 --- iocore/net/SSLConfig.cc | 31 +---- iocore/net/SSLNetVConnection.cc | 119 -------------------- iocore/net/SSLUtils.cc | 5 - iocore/net/UnixNetVConnection.cc | 39 +------ mgmt/RecordsConfig.cc | 8 -- proxy/http/HttpSM.cc | 21 ---- 9 files changed, 3 insertions(+), 259 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index c14df8842a7..2936600caf9 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3450,25 +3450,6 @@ SSL Termination See :ref:`admin-performance-timeouts` for more discussion on |TS| timeouts. -.. ts:cv:: CONFIG proxy.config.ssl.wire_trace_enabled INT 0 - - When enabled this turns on wire tracing of SSL connections that meet - the conditions specified by wire_trace_percentage, wire_trace_addr - and wire_trace_server_name. - -.. ts:cv:: CONFIG proxy.config.ssl.wire_trace_percentage INT 0 - - This specifies the percentage of traffic meeting the other wire_trace - conditions to be traced. - -.. ts:cv:: CONFIG proxy.config.ssl.wire_trace_addr STRING NULL - - This specifies the client IP for which wire_traces should be printed. - -.. ts:cv:: CONFIG proxy.config.ssl.wire_trace_server_name STRING NULL - - This specifies the server name for which wire_traces should be printed. - Client-Related Configuration ---------------------------- diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index a691e5ba990..3bccf9f1516 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -107,13 +107,6 @@ struct SSLConfigParams : public ConfigInfo { static size_t session_cache_max_bucket_size; static bool session_cache_skip_on_lock_contention; - // TS-3435 Wiretracing for SSL Connections - static int ssl_wire_trace_enabled; - static char *ssl_wire_trace_addr; - static IpAddr *ssl_wire_trace_ip; - static int ssl_wire_trace_percentage; - static char *ssl_wire_trace_server_name; - static init_ssl_ctx_func init_ssl_ctx_cb; static load_ssl_file_func load_ssl_file_cb; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 144cf3749c5..f2eadb3d9ac 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -286,19 +286,6 @@ class SSLNetVConnection : public UnixNetVConnection } return retval; } - bool - getSSLTrace() const - { - return sslTrace || super::origin_trace; - } - - void - setSSLTrace(bool state) - { - sslTrace = state; - } - - bool computeSSLTrace(); const char * getSSLProtocol(void) const diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 9c1a6d9c5ba..a6dbed7eebb 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -58,14 +58,8 @@ size_t SSLConfigParams::session_cache_max_bucket_size = 100; init_ssl_ctx_func SSLConfigParams::init_ssl_ctx_cb = nullptr; load_ssl_file_func SSLConfigParams::load_ssl_file_cb = nullptr; -// TS-3534 Wiretracing for SSL Connections -int SSLConfigParams::ssl_wire_trace_enabled = 0; -char *SSLConfigParams::ssl_wire_trace_addr = nullptr; -IpAddr *SSLConfigParams::ssl_wire_trace_ip = nullptr; -int SSLConfigParams::ssl_wire_trace_percentage = 0; -char *SSLConfigParams::ssl_wire_trace_server_name = nullptr; -int SSLConfigParams::async_handshake_enabled = 0; -char *SSLConfigParams::engine_conf_file = nullptr; +int SSLConfigParams::async_handshake_enabled = 0; +char *SSLConfigParams::engine_conf_file = nullptr; static std::unique_ptr> sslCertUpdate; static std::unique_ptr> sslConfigUpdate; @@ -131,7 +125,6 @@ SSLConfigParams::cleanup() cipherSuite = (char *)ats_free_null(cipherSuite); client_cipherSuite = (char *)ats_free_null(client_cipherSuite); dhparamsFile = (char *)ats_free_null(dhparamsFile); - ssl_wire_trace_ip = (IpAddr *)ats_free_null(ssl_wire_trace_ip); server_tls13_cipher_suites = (char *)ats_free_null(server_tls13_cipher_suites); client_tls13_cipher_suites = (char *)ats_free_null(client_tls13_cipher_suites); @@ -438,26 +431,6 @@ SSLConfigParams::initialize() REC_ReadConfigInt32(ssl_allow_client_renegotiation, "proxy.config.ssl.allow_client_renegotiation"); - // SSL Wire Trace configurations - REC_EstablishStaticConfigInt32(ssl_wire_trace_enabled, "proxy.config.ssl.wire_trace_enabled"); - if (ssl_wire_trace_enabled) { - // wire trace specific source ip - REC_EstablishStaticConfigStringAlloc(ssl_wire_trace_addr, "proxy.config.ssl.wire_trace_addr"); - if (ssl_wire_trace_addr) { - ssl_wire_trace_ip = new IpAddr(); - ssl_wire_trace_ip->load(ssl_wire_trace_addr); - } else { - ssl_wire_trace_ip = nullptr; - } - // wire trace percentage of requests - REC_EstablishStaticConfigInt32(ssl_wire_trace_percentage, "proxy.config.ssl.wire_trace_percentage"); - REC_EstablishStaticConfigStringAlloc(ssl_wire_trace_server_name, "proxy.config.ssl.wire_trace_server_name"); - } else { - ssl_wire_trace_addr = nullptr; - ssl_wire_trace_ip = nullptr; - ssl_wire_trace_percentage = 0; - ssl_wire_trace_server_name = nullptr; - } // Enable client regardless of config file settings as remap file // can cause HTTP layer to connect using SSL. But only if SSL // initialization hasn't failed already. diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 86e150ce608..71e1b085162 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -214,7 +214,6 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) int64_t bytes_read = 0; ssl_error_t sslErr = SSL_ERROR_NONE; - bool trace = sslvc->getSSLTrace(); int64_t toread = buf.writer()->write_avail(); ink_release_assert(toread > 0); if (toread > s->vio.ntodo()) { @@ -237,15 +236,6 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) sslErr = SSLReadBuffer(sslvc->ssl, current_block, amount_to_read, nread); Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] nread=%d", (int)nread); - if (!sslvc->origin_trace) { - TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", (int)nread, - (int)nread, current_block); - } else { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(sslvc->origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", - origin_trace_ip, sslvc->origin_trace_port, (int)nread, (int)nread, current_block); - } switch (sslErr) { case SSL_ERROR_NONE: @@ -271,29 +261,24 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read)"); break; case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Want X509 lookup"); event = SSL_READ_WOULD_BLOCK; SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read/x509 lookup)"); break; case SSL_ERROR_SYSCALL: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Syscall Error: %s", strerror(errno)); SSL_INCREMENT_DYN_STAT(ssl_error_syscall); if (nread != 0) { // not EOF event = SSL_READ_ERROR; ret = errno; Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, underlying IO error: %s", strerror(errno)); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Underlying IO error: %d", errno); } else { // then EOF observed, treat it as EOS // Error("[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, EOF observed violating SSL protocol"); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "EOF observed violating SSL protocol"); event = SSL_READ_EOS; } break; case SSL_ERROR_ZERO_RETURN: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Connection closed by peer"); event = SSL_READ_EOS; SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_ZERO_RETURN"); @@ -303,8 +288,6 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) char buf[512]; unsigned long e = ERR_peek_last_error(); ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", - sslErr, e, buf, errno); event = SSL_READ_ERROR; ret = errno; SSL_CLR_ERR_INCR_DYN_STAT(sslvc, ssl_error_ssl, "[SSL_NetVConnection::ssl_read_from_net]: errno=%d", errno); @@ -697,8 +680,6 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf return this->super::load_buffer_and_write(towrite, buf, total_written, needs); } - bool trace = getSSLTrace(); - do { // What is remaining left in the next block? l = buf.reader()->block_read_avail(); @@ -747,16 +728,6 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf towrite, current_block); err = SSLWriteBuffer(ssl, current_block, l, num_really_written); - if (!origin_trace) { - TraceOut((0 < num_really_written && trace), get_remote_addr(), get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", - (int)num_really_written, (int)num_really_written, current_block); - } else { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - TraceOut((0 < num_really_written && trace), get_remote_addr(), get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", - origin_trace_ip, origin_trace_port, (int)num_really_written, (int)num_really_written, current_block); - } - // We wrote all that we thought we should if (num_really_written > 0) { total_written += num_really_written; @@ -793,7 +764,6 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf redoWriteSize = l; } else if (SSL_ERROR_WANT_X509_LOOKUP == err) { SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); - TraceOut(trace, get_remote_addr(), get_remote_port(), "Want X509 lookup"); } needs |= EVENTIO_WRITE; @@ -802,25 +772,18 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf break; } case SSL_ERROR_SYSCALL: - TraceOut(trace, get_remote_addr(), get_remote_port(), "Syscall Error: %s", strerror(errno)); num_really_written = -errno; SSL_INCREMENT_DYN_STAT(ssl_error_syscall); Debug("ssl.error", "SSL_write-SSL_ERROR_SYSCALL"); break; // end of stream case SSL_ERROR_ZERO_RETURN: - TraceOut(trace, get_remote_addr(), get_remote_port(), "SSL Error: zero return"); num_really_written = -errno; SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); Debug("ssl.error", "SSL_write-SSL_ERROR_ZERO_RETURN"); break; case SSL_ERROR_SSL: default: { - char buf[512]; - unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", err, e, buf, - errno); // Treat SSL_ERROR_SSL as EPIPE error. num_really_written = -EPIPE; SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSL_write-SSL_ERROR_SSL errno=%d", errno); @@ -1164,8 +1127,6 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) } } #endif - bool trace = getSSLTrace(); - if (ssl_error != SSL_ERROR_NONE) { err = errno; SSLVCDebug(this, "SSL handshake error: %s (%d), errno=%d", SSLErrorName(ssl_error), ssl_error, err); @@ -1195,9 +1156,6 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) sslHandshakeStatus = SSL_HANDSHAKE_DONE; - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake completed successfully"); - // do we want to include cert info in trace? - if (sslHandshakeBeginTime) { sslHandshakeEndTime = Thread::get_hrtime(); const ink_hrtime ssl_handshake_time = sslHandshakeEndTime - sslHandshakeBeginTime; @@ -1239,38 +1197,30 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) } Debug("ssl", "client selected next protocol '%.*s'", len, proto); - TraceIn(trace, get_remote_addr(), get_remote_port(), "client selected next protocol'%.*s'", len, proto); } else { Debug("ssl", "client did not select a next protocol"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "client did not select a next protocol"); } } return EVENT_DONE; case SSL_ERROR_WANT_CONNECT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_CONNECT"); return SSL_HANDSHAKE_WANT_CONNECT; case SSL_ERROR_WANT_WRITE: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_WRITE"); return SSL_HANDSHAKE_WANT_WRITE; case SSL_ERROR_WANT_READ: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_READ"); return SSL_HANDSHAKE_WANT_READ; // This value is only defined in openssl has been patched to // enable the sni callback to break out of the SSL_accept processing #ifdef SSL_ERROR_WANT_SNI_RESOLVE case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_X509_LOOKUP"); return EVENT_CONT; case SSL_ERROR_WANT_SNI_RESOLVE: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_SNI_RESOLVE"); #elif SSL_ERROR_WANT_X509_LOOKUP case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_X509_LOOKUP"); #endif #if defined(SSL_ERROR_WANT_SNI_RESOLVE) || defined(SSL_ERROR_WANT_X509_LOOKUP) if (this->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL || SSL_HOOK_OP_TUNNEL == hookOpRequested) { @@ -1285,32 +1235,22 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) #if TS_USE_TLS_ASYNC case SSL_ERROR_WANT_ASYNC: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_ASYNC"); return SSL_WAIT_FOR_ASYNC; #endif case SSL_ERROR_WANT_ACCEPT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_ACCEPT"); return EVENT_CONT; case SSL_ERROR_SSL: { SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSLNetVConnection::sslServerHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); - char buf[512]; - unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, get_remote_addr(), get_remote_port(), - "SSL server handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, e, buf, errno); return EVENT_ERROR; } case SSL_ERROR_ZERO_RETURN: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_ZERO_RETURN"); return EVENT_ERROR; case SSL_ERROR_SYSCALL: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_SYSCALL"); return EVENT_ERROR; default: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_OTHER"); return EVENT_ERROR; } } @@ -1318,7 +1258,6 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) int SSLNetVConnection::sslClientHandShakeEvent(int &err) { - bool trace = getSSLTrace(); ssl_error_t ssl_error; ink_assert(SSLNetVCAccess(ssl) == this); @@ -1372,49 +1311,39 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) SSL_INCREMENT_DYN_STAT(ssl_total_success_handshake_count_out_stat); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake completed successfully"); - // do we want to include cert info in trace? - sslHandshakeStatus = SSL_HANDSHAKE_DONE; return EVENT_DONE; case SSL_ERROR_WANT_WRITE: Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_WRITE"); SSL_INCREMENT_DYN_STAT(ssl_error_want_write); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_WRITE"); return SSL_HANDSHAKE_WANT_WRITE; case SSL_ERROR_WANT_READ: SSL_INCREMENT_DYN_STAT(ssl_error_want_read); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_READ"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_READ"); return SSL_HANDSHAKE_WANT_READ; case SSL_ERROR_WANT_X509_LOOKUP: SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_X509_LOOKUP"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_X509_LOOKUP"); break; case SSL_ERROR_WANT_ACCEPT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_ACCEPT"); return SSL_HANDSHAKE_WANT_ACCEPT; case SSL_ERROR_WANT_CONNECT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_CONNECT"); break; case SSL_ERROR_ZERO_RETURN: SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, EOS"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake EOS"); return EVENT_ERROR; case SSL_ERROR_SYSCALL: err = errno; SSL_INCREMENT_DYN_STAT(ssl_error_syscall); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, syscall"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake Syscall Error: %s", strerror(errno)); return EVENT_ERROR; break; @@ -1427,8 +1356,6 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) // FIXME -- This triggers a retry on cases of cert validation errors.... SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL"); - TraceIn(trace, get_remote_addr(), get_remote_port(), - "SSL client handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, e, buf, errno); if (e) { if (this->options.sni_servername) { Debug("ssl.error", "SSL connection failed for '%s': %s", this->options.sni_servername.get(), buf); @@ -1729,52 +1656,6 @@ SSLNetVConnection::callHooks(TSEvent eventId) return reenabled; } -bool -SSLNetVConnection::computeSSLTrace() -{ - // this has to happen before the handshake or else sni_servername will be nullptr - bool sni_trace; - if (ssl) { - const char *ssl_servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); - char *wire_trace_server_name = SSLConfigParams::ssl_wire_trace_server_name; - Debug("ssl", "for wiretrace, ssl_servername=%s, wire_trace_server_name=%s", ssl_servername, wire_trace_server_name); - sni_trace = ssl_servername && wire_trace_server_name && (0 == strcmp(wire_trace_server_name, ssl_servername)); - } else { - sni_trace = false; - } - - // count based on ip only if they set an IP value - const sockaddr *remote_addr = get_remote_addr(); - bool ip_trace = false; - if (SSLConfigParams::ssl_wire_trace_ip) { - ip_trace = (*SSLConfigParams::ssl_wire_trace_ip == remote_addr); - } - - // count based on percentage - int percentage = SSLConfigParams::ssl_wire_trace_percentage; - int random; - bool trace; - - // we only generate random numbers as needed (to maintain correct percentage) - if (SSLConfigParams::ssl_wire_trace_server_name && SSLConfigParams::ssl_wire_trace_ip) { - random = this_ethread()->generator.random() % 100; // range [0-99] - trace = sni_trace && ip_trace && (percentage > random); - } else if (SSLConfigParams::ssl_wire_trace_server_name) { - random = this_ethread()->generator.random() % 100; // range [0-99] - trace = sni_trace && (percentage > random); - } else if (SSLConfigParams::ssl_wire_trace_ip) { - random = this_ethread()->generator.random() % 100; // range [0-99] - trace = ip_trace && (percentage > random); - } else { - random = this_ethread()->generator.random() % 100; // range [0-99] - trace = percentage > random; - } - - Debug("ssl", "ssl_netvc random=%d, trace=%s", random, trace ? "TRUE" : "FALSE"); - - return trace; -} - int SSLNetVConnection::populate(Connection &con, Continuation *c, void *arg) { diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 2df2374cedf..99655287223 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -330,11 +330,6 @@ set_context_cert(SSL *ssl) int retval = 1; Debug("ssl", "set_context_cert ssl=%p server=%s handshake_complete=%d", ssl, servername, netvc->getSSLHandShakeComplete()); - if (SSLConfigParams::ssl_wire_trace_enabled) { - bool trace = netvc->computeSSLTrace(); - Debug("ssl", "sslnetvc. setting trace to=%s", trace ? "true" : "false"); - netvc->setSSLTrace(trace); - } // catch the client renegotiation early on if (SSLConfigParams::ssl_allow_client_renegotiation == false && netvc->getSSLHandShakeComplete()) { diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index e01fc57f31a..ca5e3417794 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -253,24 +253,6 @@ read_from_net(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) NET_INCREMENT_DYN_STAT(net_calls_to_read_stat); - if (vc->origin_trace) { - char origin_trace_ip[INET6_ADDRSTRLEN]; - - ats_ip_ntop(vc->origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - - if (r > 0) { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, - vc->origin_trace_port, (int)r, (int)r, (char *)tiovec[0].iov_base); - - } else if (r == 0) { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d closed connection", - origin_trace_ip, vc->origin_trace_port); - } else { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, - vc->origin_trace_port, strerror(errno)); - } - } - total_read += rattempted; } while (rattempted && r == rattempted && total_read < toread); @@ -896,10 +878,7 @@ UnixNetVConnection::UnixNetVConnection() submit_time(0), oob_ptr(nullptr), from_accept_thread(false), - accept_object(nullptr), - origin_trace(false), - origin_trace_addr(nullptr), - origin_trace_port(0) + accept_object(nullptr) { SET_HANDLER((NetVConnHandler)&UnixNetVConnection::startEvent); } @@ -998,22 +977,6 @@ UnixNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &bu r = socketManager.writev(con.fd, &tiovec[0], niov); } - if (origin_trace) { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - - if (r > 0) { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, - origin_trace_port, (int)r, (int)r, (char *)tiovec[0].iov_base); - - } else if (r == 0) { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d\tbytes=0", origin_trace_ip, origin_trace_port); - } else { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, origin_trace_port, - strerror(errno)); - } - } - if (r > 0) { buf.reader()->consume(r); total_written += r; diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 5799a0152c7..931c6e3d009 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1178,14 +1178,6 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.handshake_timeout_in", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-65535]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.wire_trace_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.ssl.wire_trace_addr", RECD_STRING, nullptr , RECU_DYNAMIC, RR_NULL, RECC_IP, R"([0-255]\.[0-255]\.[0-255]\.[0-255])", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.ssl.wire_trace_percentage", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-100]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.ssl.wire_trace_server_name", RECD_STRING, nullptr , RECU_DYNAMIC, RR_NULL, RECC_STR, ".*", RECA_NULL} - , {RECT_CONFIG, "proxy.config.ssl.cert.load_elevated", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_READ_ONLY} , {RECT_CONFIG, "proxy.config.ssl.server.groups_list", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 52d4dde2390..3d91e55abf9 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -5860,30 +5860,9 @@ HttpSM::attach_server_session(HttpServerSession *s) // Get server and client connections UnixNetVConnection *server_vc = dynamic_cast(server_session->get_netvc()); UnixNetVConnection *client_vc = (UnixNetVConnection *)(ua_txn->get_netvc()); - SSLNetVConnection *ssl_vc = dynamic_cast(client_vc); // Verifying that the user agent and server sessions/transactions are operating on the same thread. ink_release_assert(!server_vc || !client_vc || server_vc->thread == client_vc->thread); - bool associated_connection = false; - if (server_vc) { // if server_vc isn't a PluginVC - if (ssl_vc) { // if incoming connection is SSL - bool client_trace = ssl_vc->getSSLTrace(); - if (client_trace) { - // get remote address and port to mark corresponding traces - const sockaddr *remote_addr = ssl_vc->get_remote_addr(); - uint16_t remote_port = ssl_vc->get_remote_port(); - server_vc->setOriginTrace(true); - server_vc->setOriginTraceAddr(remote_addr); - server_vc->setOriginTracePort(remote_port); - associated_connection = true; - } - } - } - if (!associated_connection && server_vc) { - server_vc->setOriginTrace(false); - server_vc->setOriginTraceAddr(nullptr); - server_vc->setOriginTracePort(0); - } // set flag for server session is SSL SSLNetVConnection *server_ssl_vc = dynamic_cast(server_vc); From 07fb9130cdd458965a8631ff7cd2f1376d6ac689 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 29 Nov 2018 21:40:12 +0000 Subject: [PATCH 069/526] Add forward_route action --- .../files/ssl_server_name.yaml.en.rst | 10 ++ iocore/net/P_SNIActionPerformer.h | 11 +- iocore/net/P_SSLNetVConnection.h | 12 +- iocore/net/SSLSNIConfig.cc | 2 +- iocore/net/SSLUtils.cc | 2 +- iocore/net/YamlSNIConfig.cc | 6 + iocore/net/YamlSNIConfig.h | 3 + proxy/http/HttpSM.cc | 15 +- tests/gold_tests/tls/test-nc-s_client.sh | 19 +++ .../tls/tls_forward_nonhttp.test.py | 77 ++++++++++ .../gold_tests/tls/tls_tunnel_forward.test.py | 132 ++++++++++++++++++ 11 files changed, 276 insertions(+), 13 deletions(-) create mode 100644 tests/gold_tests/tls/test-nc-s_client.sh create mode 100644 tests/gold_tests/tls/tls_forward_nonhttp.test.py create mode 100644 tests/gold_tests/tls/tls_tunnel_forward.test.py diff --git a/doc/admin-guide/files/ssl_server_name.yaml.en.rst b/doc/admin-guide/files/ssl_server_name.yaml.en.rst index 32ab6af2a40..66fdd558e8f 100644 --- a/doc/admin-guide/files/ssl_server_name.yaml.en.rst +++ b/doc/admin-guide/files/ssl_server_name.yaml.en.rst @@ -91,6 +91,16 @@ disable_h2 :code:`true` or :code:`false`. for proxy ports on which HTTP/2 is not enabled. tunnel_route Destination as an FQDN and port, separated by a colon ``:``. + + + This will forward all traffic to the specified destination without first terminating + the incoming TLS connection. + +forward_route Destination as an FQDN and port, separated by a colon ``:``. + + This is similar to tunnel_route, but it terminates the TLS connection and forwards the + decrypted traffic. |TS| will not interpret the decrypted data, so the contents do not + need to be HTTP. ========================= ============================================================================== Client verification, via ``verify_client``, correponds to setting diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 8622ff2a033..d515fdcfdce 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -38,15 +38,17 @@ #include extern std::unordered_map snpsMap; -// enum of all the actions + +/*// enum of all the actions enum AllActions { TS_DISABLE_H2 = 0, TS_VERIFY_CLIENT, // this applies to server side vc only TS_TUNNEL_ROUTE, // blind tunnel action }; +*/ /** action for setting next hop properties should be listed in the following enum*/ -enum PropertyActions { TS_VERIFY_SERVER = 200, TS_CLIENT_CERT }; +/* enum PropertyActions { TS_VERIFY_SERVER = 200, TS_CLIENT_CERT }; */ class ActionItem { @@ -78,7 +80,7 @@ class DisableH2 : public ActionItem class TunnelDestination : public ActionItem { public: - TunnelDestination(const std::string_view &dest) : destination(dest) {} + TunnelDestination(const std::string_view &dest, bool decrypt) : destination(dest), tunnel_decrypt(decrypt) {} ~TunnelDestination() {} int @@ -87,11 +89,12 @@ class TunnelDestination : public ActionItem // Set the netvc option? SSLNetVConnection *ssl_netvc = dynamic_cast(cont); if (ssl_netvc) { - ssl_netvc->set_tunnel_destination(destination); + ssl_netvc->set_tunnel_destination(destination, tunnel_decrypt); } return SSL_TLSEXT_ERR_OK; } std::string destination; + bool tunnel_decrypt = false; }; class VerifyClient : public ActionItem diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index f2eadb3d9ac..19976be2a08 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -317,8 +317,16 @@ class SSLNetVConnection : public UnixNetVConnection return tunnel_port; } + /* Returns true if this vc was configured for forward_route + */ + bool + decrypt_tunnel() + { + return has_tunnel_destination() && tunnel_decrypt; + } + void - set_tunnel_destination(const std::string_view &destination) + set_tunnel_destination(const std::string_view &destination, bool decrypt) { auto pos = destination.find(":"); if (nullptr != tunnel_host) { @@ -331,6 +339,7 @@ class SSLNetVConnection : public UnixNetVConnection tunnel_port = 0; tunnel_host = ats_strndup(destination.data(), destination.length()); } + tunnel_decrypt = decrypt; } int populate_protocol(std::string_view *results, int n) const override; @@ -400,6 +409,7 @@ class SSLNetVConnection : public UnixNetVConnection int64_t redoWriteSize = 0; char *tunnel_host = nullptr; in_port_t tunnel_port = 0; + bool tunnel_decrypt = false; }; typedef int (SSLNetVConnection::*SSLNetVConnHandler)(int, void *); diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 21986b7d47a..524d2ab9228 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -73,7 +73,7 @@ SNIConfigParams::loadSNIConfig() ai->actions.push_back(std::make_unique(item.verify_client_level)); } if (item.tunnel_destination.length() > 0) { - ai->actions.push_back(std::make_unique(item.tunnel_destination)); + ai->actions.push_back(std::make_unique(item.tunnel_destination, item.tunnel_decrypt)); } ai->actions.push_back(std::make_unique(item.ip_allow, item.fqdn)); diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 99655287223..5e53b0bdd28 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -475,7 +475,7 @@ ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) if (ret != SSL_TLSEXT_ERR_OK) { return SSL_TLSEXT_ERR_ALERT_FATAL; } - if (netvc->has_tunnel_destination()) { + if (netvc->has_tunnel_destination() && !netvc->decrypt_tunnel()) { netvc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; } diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc index 276b271cfa7..8a542cd8706 100644 --- a/iocore/net/YamlSNIConfig.cc +++ b/iocore/net/YamlSNIConfig.cc @@ -63,6 +63,7 @@ std::set valid_sni_config_keys = {TS_fqdn, TS_disable_h2, TS_verify_client, TS_tunnel_route, + TS_forward_route, TS_verify_origin_server, TS_verify_server_policy, TS_verify_server_properties, @@ -104,7 +105,12 @@ template <> struct convert { if (node[TS_tunnel_route]) { item.tunnel_destination = node[TS_tunnel_route].as(); + item.tunnel_decrypt = false; + } else if (node[TS_forward_route]) { + item.tunnel_destination = node[TS_forward_route].as(); + item.tunnel_decrypt = true; } + // remove before 9.0.0 release // backwards compatibiity if (node[TS_verify_origin_server]) { diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h index e36fe9199e6..8926c29bdc2 100644 --- a/iocore/net/YamlSNIConfig.h +++ b/iocore/net/YamlSNIConfig.h @@ -31,6 +31,7 @@ TSDECL(fqdn); TSDECL(disable_h2); TSDECL(verify_client); TSDECL(tunnel_route); +TSDECL(forward_route); TSDECL(verify_server_policy); TSDECL(verify_server_properties); TSDECL(verify_origin_server); @@ -45,6 +46,7 @@ struct YamlSNIConfig { disable_h2 = start, verify_client, tunnel_route, // blind tunnel action + forward_route, // decrypt data and then blind tunnel action verify_server_policy, // this applies to server side vc only verify_server_properties, // this applies to server side vc only client_cert @@ -60,6 +62,7 @@ struct YamlSNIConfig { bool disable_h2 = false; uint8_t verify_client_level = 255; std::string tunnel_destination; + bool tunnel_decrypt = false; Policy verify_server_policy = Policy::DISABLED; Property verify_server_properties = Property::NONE; std::string client_cert; diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 3d91e55abf9..48ccfc65ca3 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -517,7 +517,7 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader http_parser_init(&http_parser); // Prepare raw reader which will live until we are sure this is HTTP indeed - if (is_transparent_passthrough_allowed()) { + if (is_transparent_passthrough_allowed() || (ssl_vc && ssl_vc->decrypt_tunnel())) { ua_raw_buffer_reader = buffer_reader->clone(); } @@ -664,7 +664,7 @@ HttpSM::state_read_client_request_header(int event, void *data) // We need to handle EOS as well as READ_READY because the client // may have sent all of the data already followed by a fIN and that // should be OK. - if (is_transparent_passthrough_allowed() && ua_raw_buffer_reader != nullptr) { + if (ua_raw_buffer_reader != nullptr) { bool do_blind_tunnel = false; // If we had a parse error and we're done reading data // blind tunnel @@ -686,7 +686,7 @@ HttpSM::state_read_client_request_header(int event, void *data) // Turn off read eventing until we get the // blind tunnel infrastructure set up if (netvc) { - netvc->do_io_read(this, 0, nullptr); + netvc->do_io_read(nullptr, 0, nullptr); } /* establish blind tunnel */ @@ -1578,14 +1578,17 @@ void HttpSM::handle_api_return() { switch (t_state.api_next_action) { - case HttpTransact::SM_ACTION_API_SM_START: - if (t_state.client_info.port_attribute == HttpProxyPort::TRANSPORT_BLIND_TUNNEL) { + case HttpTransact::SM_ACTION_API_SM_START: { + NetVConnection *netvc = ua_txn->get_netvc(); + SSLNetVConnection *ssl_vc = dynamic_cast(netvc); + bool forward_dest = ssl_vc != nullptr && ssl_vc->decrypt_tunnel(); + if (t_state.client_info.port_attribute == HttpProxyPort::TRANSPORT_BLIND_TUNNEL || forward_dest) { setup_blind_tunnel_port(); } else { setup_client_read_request_header(); } return; - + } case HttpTransact::SM_ACTION_API_CACHE_LOOKUP_COMPLETE: case HttpTransact::SM_ACTION_API_READ_CACHE_HDR: if (t_state.api_cleanup_cache_read && t_state.api_update_cached_object != HttpTransact::UPDATE_CACHED_OBJECT_PREPARE) { diff --git a/tests/gold_tests/tls/test-nc-s_client.sh b/tests/gold_tests/tls/test-nc-s_client.sh new file mode 100644 index 00000000000..a11a80c9329 --- /dev/null +++ b/tests/gold_tests/tls/test-nc-s_client.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +nc -l -p $1 -c 'echo -e "This is a reply"' -o test.out & +echo "This is a test" | openssl s_client -servername bar.com -connect localhost:$2 -ign_eof diff --git a/tests/gold_tests/tls/tls_forward_nonhttp.test.py b/tests/gold_tests/tls/tls_forward_nonhttp.test.py new file mode 100644 index 00000000000..2fb8b9052e3 --- /dev/null +++ b/tests/gold_tests/tls/tls_forward_nonhttp.test.py @@ -0,0 +1,77 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Forwarding a non-HTTP protocol out of TLS +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.connect_ports': '{0} 4444'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# foo.com should not terminate. Just tunnel to server_foo +# bar.com should terminate. Forward its tcp stream to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + "- fqdn: bar.com", + " forward_route: localhost:4444" + ]) + +tr = Test.AddTestRun("forward-non-http") +tr.Setup.Copy("test-nc-s_client.sh") +tr.Processes.Default.Command = "sh test-nc-s_client.sh 4444 4443" +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +testout_path = os.path.join(Test.RunDirectory, "test.out") +tr.Disk.File(testout_path, id = "testout") +tr.Processes.Default.Streams.All += Testers.IncludesExpression("This is a reply", "s_client should get response") + diff --git a/tests/gold_tests/tls/tls_tunnel_forward.test.py b/tests/gold_tests/tls/tls_tunnel_forward.test.py new file mode 100644 index 00000000000..ce031516495 --- /dev/null +++ b/tests/gold_tests/tls/tls_tunnel_forward.test.py @@ -0,0 +1,132 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tunneling and forwarding based on SNI +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True) +server_bar = Test.MakeOriginServer("server_bar", ssl=False) +server_random = Test.MakeOriginServer("server_random", ssl=False) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_random_header = {"headers": "GET / HTTP/1.1\r\nHost: random.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok foo"} +response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok bar"} +response_random_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok random"} +server_foo.addResponse("sessionlog_foo.json", request_foo_header, response_foo_header) +server_bar.addResponse("sessionlog_bar.json", request_bar_header, response_bar_header) +server_random.addResponse("sessionlog_random.json", request_random_header, response_random_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.connect_ports': '{0} {1} {2} {3}'.format(ts.Variables.ssl_port,server_foo.Variables.Port,server_bar.Variables.Port,server_random.Variables.Port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# foo.com should not terminate. Just tunnel to server_foo +# bar.com should terminate. Forward its tcp stream to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + "- fqdn: 'foo.com'", + " tunnel_route: 'localhost:{0}'".format(server_foo.Variables.Port), + "- fqdn: 'bar.com'", + " forward_route: 'localhost:{0}'".format(server_bar.Variables.Port), + "- fqdn: ''", #default case + " forward_route: 'localhost:{0}'".format(server_random.Variables.Port), + ]) + +tr = Test.AddTestRun("Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server_random) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ok foo", "Body is expected") + +tr2 = Test.AddTestRun("Forward-test") +tr2.Processes.Default.Command = "curl -v --http1.1 -H 'host:bar.com' --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server_bar +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.TimeOut = 5 +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("ok bar", "Body is expected") + +tr3 = Test.AddTestRun("no-sni-forward-test") +tr3.Processes.Default.Command = "curl --http1.1 -v -k -H 'host:random.com' https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server_random +tr3.Processes.Default.TimeOut = 5 +tr3.StillRunningAfter = ts +tr3.TimeOut = 5 +tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr3.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") +tr3.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr3.Processes.Default.Streams.All += Testers.ContainsExpression("ok random", "Body is expected") + From b2ebf57a3de0f0152e9a81c0b3e77f44951f2ca0 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 4 Dec 2018 15:53:35 -0600 Subject: [PATCH 070/526] Doc: Fix ts:cv typo in records.config documentation. --- doc/admin-guide/files/records.config.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 2936600caf9..52ee8fcb5b0 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -1418,7 +1418,7 @@ Origin Server Connect Attempts This value is used in determining when and if to prune active origin sessions. Without this value set, connections to origins can consume all the - way up to ts:cv:`proxy.config.net.connections_throttle` connections, which + way up to :ts:cv:`proxy.config.net.connections_throttle` connections, which in turn can starve incoming requests from available connections. .. ts:cv:: CONFIG proxy.config.http.per_server.connection.max INT 0 From 9c9d446c869276f75ce4061f65bd2939f0f25a38 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 4 Dec 2018 13:23:23 -0600 Subject: [PATCH 071/526] Doc: Fix "dst_ip" typo in ip_allow.config documentation. --- doc/admin-guide/files/ip_allow.config.en.rst | 4 ++-- proxy/http/remap/AclFiltering.h | 2 +- proxy/http/remap/RemapConfig.cc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/admin-guide/files/ip_allow.config.en.rst b/doc/admin-guide/files/ip_allow.config.en.rst index 537afcb6eaa..b313f8368ef 100644 --- a/doc/admin-guide/files/ip_allow.config.en.rst +++ b/doc/admin-guide/files/ip_allow.config.en.rst @@ -39,7 +39,7 @@ format:: dest_ip= action= [method=] For ``src_ip`` the remote inbound connection address, i.e. the IP address of the client, is checked -against the specified range of IP addresses. For ``dst_ip`` the outbound remote address (i.e. the IP +against the specified range of IP addresses. For ``dest_ip`` the outbound remote address (i.e. the IP address to which |TS| connects) is checked against the specified IP address range. Range specifications can be IPv4 or IPv6, but any single range must be one or the other. Ranges can @@ -67,7 +67,7 @@ For each inbound or outbound connection the applicable rule is selectd by first address. The rule is then applied (if the method matches) or its opposite is applied (if the method doesn't match). If no rule is matched access is allowed. This makes each rule both an accept and deny, one explicit and the other implicit. The ``src_ip`` rules are checked when a host connects -to |TS|. The ``dst_ip`` rules are checked when |TS| connects to another host. +to |TS|. The ``dest_ip`` rules are checked when |TS| connects to another host. By default the :file:`ip_allow.config` file contains the following lines, which allows all methods to connections from localhost and denies the ``PUSH``, ``PURGE`` and ``DELETE`` methods to all other diff --git a/proxy/http/remap/AclFiltering.h b/proxy/http/remap/AclFiltering.h index f9cb8c8d80b..a4af7264fc0 100644 --- a/proxy/http/remap/AclFiltering.h +++ b/proxy/http/remap/AclFiltering.h @@ -91,7 +91,7 @@ class acl_filter_rule src_ip_info_t src_ip_array[ACL_FILTER_MAX_SRC_IP]; // in_ip - int in_ip_cnt; // how many valid dst_ip rules we have + int in_ip_cnt; // how many valid dest_ip rules we have src_ip_info_t in_ip_array[ACL_FILTER_MAX_IN_IP]; acl_filter_rule(); diff --git a/proxy/http/remap/RemapConfig.cc b/proxy/http/remap/RemapConfig.cc index 0feefdf6d00..c0eab6effc9 100644 --- a/proxy/http/remap/RemapConfig.cc +++ b/proxy/http/remap/RemapConfig.cc @@ -529,7 +529,7 @@ remap_validate_filter_args(acl_filter_rule **rule_pp, const char **argv, int arg } } - if (ul & REMAP_OPTFLG_IN_IP) { /* "dst_ip=" option */ + if (ul & REMAP_OPTFLG_IN_IP) { /* "dest_ip=" option */ if (rule->in_ip_cnt >= ACL_FILTER_MAX_IN_IP) { Debug("url_rewrite", "[validate_filter_args] Too many \"in_ip=\" filters"); snprintf(errStrBuf, errStrBufSize, "Defined more than %d \"in_ip=\" filters!", ACL_FILTER_MAX_IN_IP); From ca264e48553bd60b0fa5fb7a888aaeeeb6f6a330 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 4 Dec 2018 14:48:35 -0700 Subject: [PATCH 072/526] Finish the config removal started in #4653 --- lib/perl/lib/Apache/TS/AdminClient.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/perl/lib/Apache/TS/AdminClient.pm b/lib/perl/lib/Apache/TS/AdminClient.pm index efd6b4b559e..db7818e0008 100644 --- a/lib/perl/lib/Apache/TS/AdminClient.pm +++ b/lib/perl/lib/Apache/TS/AdminClient.pm @@ -599,7 +599,6 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.ssl.client.private_key.filename proxy.config.ssl.client.private_key.path proxy.config.ssl.client.verify.server - proxy.config.ssl.enabled proxy.config.ssl.server.cert_chain.filename proxy.config.ssl.server.cert.path proxy.config.ssl.server.cipher_suite From f22b06f3f7c2aba5b14d8e633b2aefc7a4a14ef7 Mon Sep 17 00:00:00 2001 From: Vijay Mamidi Date: Thu, 29 Nov 2018 11:40:24 -0700 Subject: [PATCH 073/526] Use optionally provided hash string for hashing --- doc/admin-guide/files/parent.config.en.rst | 8 ++ proxy/ParentSelection.cc | 89 +++++++++++++++++----- proxy/ParentSelection.h | 1 + 3 files changed, 80 insertions(+), 18 deletions(-) diff --git a/doc/admin-guide/files/parent.config.en.rst b/doc/admin-guide/files/parent.config.en.rst index 3a26e959b9e..ca548251206 100644 --- a/doc/admin-guide/files/parent.config.en.rst +++ b/doc/admin-guide/files/parent.config.en.rst @@ -141,6 +141,14 @@ The following list shows the possible actions and their allowed values. parent="p1.x.com:8080|2.0, 192.168.0.3:80|3.0, 192.168.0.4:80|5.0" + If ``round_robin`` is set to ``consistent_hash``, you may add a ``unique hash string`` + following the ``weight`` for each parent. The ``hash string`` must start with ``&`` + and is used to build both the primary and secondary rings using the ``hash string`` + for each parent insted of the parents ``hostname`` or ``ip address``. This can be + useful so that two different hosts may be used to cache the same requests. Example:: + + parent="p1.x.com:80|1.0&abcdef, p2.x.com:80|1.0&xyzl, p3.x.com:80|1.0&ldffg" round_robin=consistent_hash + .. _parent-config-format-secondary-parent: ``secondary_parent`` diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index df3f368ee95..882e57eb27b 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -413,7 +413,7 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) int numTok = 0; const char *current = nullptr; int port = 0; - char *tmp = nullptr, *tmp2 = nullptr; + char *tmp = nullptr, *tmp2 = nullptr, *tmp3 = nullptr; const char *errPtr = nullptr; float weight = 1.0; @@ -466,23 +466,27 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) } } + tmp3 = (char *)strchr(current, '&'); + // Make sure that is no garbage beyond the parent - // port or weight - char *scan; - if (tmp2) { - scan = tmp2 + 1; - } else { - scan = tmp + 1; - } - for (; *scan != '\0' && (ParseRules::is_digit(*scan) || *scan == '.'); scan++) { - ; - } - for (; *scan != '\0' && ParseRules::is_wslfcr(*scan); scan++) { - ; - } - if (*scan != '\0') { - errPtr = "Garbage trailing entry or invalid separator"; - goto MERROR; + // port or weight + if (!tmp3) { + char *scan; + if (tmp2) { + scan = tmp2 + 1; + } else { + scan = tmp + 1; + } + for (; *scan != '\0' && (ParseRules::is_digit(*scan) || *scan == '.'); scan++) { + ; + } + for (; *scan != '\0' && ParseRules::is_wslfcr(*scan); scan++) { + ; + } + if (*scan != '\0') { + errPtr = "Garbage trailing entry or invalid separator"; + goto MERROR; + } } // Check to make sure that the string will fit in the // pRecord @@ -505,6 +509,10 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) this->parents[i].name = this->parents[i].hostname; this->parents[i].available = true; this->parents[i].weight = weight; + if (tmp3) { + memcpy(this->parents[i].hash_string, tmp3 + 1, strlen(tmp3)); + this->parents[i].name = this->parents[i].hash_string; + } hs.createHostStat(this->parents[i].hostname); } else { memcpy(this->secondary_parents[i].hostname, current, tmp - current); @@ -517,8 +525,13 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) this->secondary_parents[i].name = this->secondary_parents[i].hostname; this->secondary_parents[i].available = true; this->secondary_parents[i].weight = weight; + if (tmp3) { + memcpy(this->secondary_parents[i].hash_string, tmp3 + 1, strlen(tmp3)); + this->secondary_parents[i].name = this->secondary_parents[i].hash_string; + } hs.createHostStat(this->secondary_parents[i].hostname); } + tmp3 = nullptr; } if (isPrimary) { @@ -803,7 +816,7 @@ ParentRecord::Print() { printf("\t\t"); for (int i = 0; i < num_parents; i++) { - printf(" %s:%d ", parents[i].hostname, parents[i].port); + printf(" %s:%d|%f&%s ", parents[i].hostname, parents[i].port, parents[i].weight, parents[i].name); } printf(" direct=%s\n", (go_direct == true) ? "true" : "false"); printf(" parent_is_proxy=%s\n", (parent_is_proxy == true) ? "true" : "false"); @@ -1719,6 +1732,46 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, sleep(1); RE(verify(result, PARENT_FAIL, nullptr, 80), 208); + // Tests 209 through 211 test that host selection is based upon the hash_string + + // Test 209 + // fuzzy { curly larry, moe } fluffy + tbl[0] = '\0'; + ST(209); + T("dest_domain=stooges.net parent=curly:80|1.0&myhash;joe:80|1.0&hishash;larry:80|1.0&ourhash " + "round_robin=consistent_hash go_direct=false\n"); + REBUILD; + REINIT; + br(request, "i.am.stooges.net"); + FP; + RE(verify(result, PARENT_SPECIFIED, "larry", 80), 209); + + // Test 210 + // fuzzy { curly larry, moe } fluffy + tbl[0] = '\0'; + ST(210); + T("dest_domain=stooges.net parent=curly:80|1.0&ourhash;joe:80|1.0&hishash;larry:80|1.0&myhash " + "round_robin=consistent_hash go_direct=false\n"); + REBUILD; + REINIT; + br(request, "i.am.stooges.net"); + FP; + RE(verify(result, PARENT_SPECIFIED, "curly", 80), 210); + + // Test 211 + // fuzzy { curly larry, moe } fluffy + tbl[0] = '\0'; + ST(211); + T("dest_domain=stooges.net parent=curly:80|1.0&ourhash;joe:80|1.0&hishash;larry:80|1.0&myhash " + "secondary_parent=carol:80|1.0&ourhash;betty:80|1.0&hishash;donna:80|1.0&myhash " + "round_robin=consistent_hash go_direct=false\n"); + REBUILD; + REINIT; + _st.setHostStatus("curly", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + br(request, "i.am.stooges.net"); + FP; + RE(verify(result, PARENT_SPECIFIED, "carol", 80), 211); + delete request; delete result; delete params; diff --git a/proxy/ParentSelection.h b/proxy/ParentSelection.h index 3b8fbd22b38..c0f3802df8c 100644 --- a/proxy/ParentSelection.h +++ b/proxy/ParentSelection.h @@ -102,6 +102,7 @@ struct pRecord : ATSConsistentHashNode { const char *scheme; // for which parent matches (if any) int idx; float weight; + char hash_string[MAXDNAME + 1]; }; typedef ControlMatcher P_table; From 37028f9785c79834b2e4dbe72de4093f5cc54543 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Tue, 13 Nov 2018 18:38:19 +0000 Subject: [PATCH 074/526] Adding cdniip and cdnistd claims to uri signing --- plugins/experimental/uri_signing/jwt.c | 14 +++++++++++++- plugins/experimental/uri_signing/jwt.h | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index da4e804f25f..0942c84dd0a 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -61,9 +61,11 @@ parse_jwt(json_t *raw) jwt->iat = parse_number(json_object_get(raw, "iat")); jwt->jti = json_string_value(json_object_get(raw, "jti")); jwt->cdniv = parse_integer_default(json_object_get(raw, "cdniv"), 1); + jwt->cdnicrit = json_string_value(json_object_get(raw, "cdnicrit")); + jwt->cdniip = json_string_value(json_object_get(raw, "cdniip")); jwt->cdniets = json_integer_value(json_object_get(raw, "cdniets")); jwt->cdnistt = json_integer_value(json_object_get(raw, "cdnistt")); - jwt->cdnicrit = json_string_value(json_object_get(raw, "cdnicrit")); + jwt->cdnistd = parse_integer_default(json_object_get(raw, "cdnistd"), 0); return jwt; } @@ -132,6 +134,11 @@ jwt_validate(struct jwt *jwt) return false; } + if (!unsupported_string_claim(jwt->cdniip)) { + PluginDebug("Initial JWT Failure: cdniip unsupported"); + return false; + } + if (!unsupported_string_claim(jwt->jti)) { PluginDebug("Initial JWT Failure: nonse unsupported"); return false; @@ -147,6 +154,11 @@ jwt_validate(struct jwt *jwt) return false; } + if (jwt->cdnistd != 0) { + PluginDebug("Initial JWT Failure: unsupported value for cdnistd: %d", jwt->cdnistd); + return false; + } + return true; } diff --git a/plugins/experimental/uri_signing/jwt.h b/plugins/experimental/uri_signing/jwt.h index 1604eeac1b2..f0bd67aa2fa 100644 --- a/plugins/experimental/uri_signing/jwt.h +++ b/plugins/experimental/uri_signing/jwt.h @@ -28,10 +28,12 @@ struct jwt { double nbf; double iat; const char *jti; - const char *cdnicrit; int cdniv; + const char *cdnicrit; + const char *cdniip; int cdniets; int cdnistt; + int cdnistd; }; struct jwt *parse_jwt(json_t *raw); void jwt_delete(struct jwt *jwt); From ae094ab660eb67fc9ec019c086de8a4b07c914f4 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Fri, 30 Nov 2018 16:21:44 +0000 Subject: [PATCH 075/526] Avoid the auto-reschedule with dispatchEvent --- .../api/functions/TSContCall.en.rst | 25 ++++------- iocore/eventsystem/Continuation.cc | 42 ------------------- iocore/eventsystem/I_Continuation.h | 14 ------- iocore/eventsystem/Makefile.am | 1 - iocore/net/UnixNetVConnection.cc | 11 ++++- proxy/http/HttpConfig.cc | 6 ++- proxy/http/HttpSM.cc | 10 ++--- proxy/http/HttpTunnel.cc | 4 +- src/traffic_server/InkAPI.cc | 22 +++++++++- 9 files changed, 50 insertions(+), 85 deletions(-) delete mode 100644 iocore/eventsystem/Continuation.cc diff --git a/doc/developer-guide/api/functions/TSContCall.en.rst b/doc/developer-guide/api/functions/TSContCall.en.rst index 9da6a319447..5453bb3cd45 100644 --- a/doc/developer-guide/api/functions/TSContCall.en.rst +++ b/doc/developer-guide/api/functions/TSContCall.en.rst @@ -44,24 +44,17 @@ As a result :func:`TSContCall` will effectively do:: return CallbackHandler(contp, event, edata); -:func:`TSContCall` will check :arg:`contp` for a mutex. If there is a mutex an attempt will be made -to lock that mutex. If either there is no mutex, or the mutex lock was acquired, the handler will be -called directly. Otherwise there is a mutex and it was not successfully locked. In that case an -event will be scheduled to dispatch as soon as possible, but not in the current call stack. The -nature of event dispatch means the event will not be dispatched until the mutext can be locked. In -all cases the handler in :arg:`contp` will be called with the same arguments. :func:`TSContCall` -will return 0 if a mutex was present but the lock was not acquired. Otherwise it will return the +If there is a mutex associated with :arg:`contp`, :func:`TSComtCall` assumes that mutex is held already. +:func:`TSContCall` will directly call the handler associated with the continuation. It will return the value returned by the handler in :arg:`contp`. -If the scheduling behavior of :func:`TSContCall` isn't appropriate, either :arg:`contp` must not -have a mutex, or the plugin must acquire the lock on the mutex for :arg:`contp` before calling -:func:`TSContCall`. See :func:`TSContMutexGet` and :func:`TSMutexLockTry` for mechanisms for doing -the latter. This is what :func:`TSContCall` does internally, and should be done by the plugin only -if a different approach for waiting for the lock is needed. The most common case is the code called -by :func:`TSContCall` must complete before further code is executed at the call site. An alternative -approach to handling the locking directly would be to split the call site in to two continuations, -one of which is signalled (possibly via :func:`TSContCall`) from the original :func:`TSContCall` -target. +If :arg:`contp` has a mutex, the plugin must acquire the lock on the mutex for :arg:`contp` before calling +:func:`TSContCall`. See :func:`TSContMutexGet` and :func:`TSMutexLockTry` for mechanisms for doing this. + +The most common case is the code called by :func:`TSContCall` must complete before further code is executed +at the call site. An alternative approach to handling the locking directly would be to split the call site +into two continuations, one of which is signalled (possibly via :func:`TSContCall`) from the original +:func:`TSContCall` target. Note mutexes returned by :func:`TSMutexCreate` are recursive mutexes, therefore if the lock is already held on the thread of execution acquiring the lock again is very fast. Mutexes are also diff --git a/iocore/eventsystem/Continuation.cc b/iocore/eventsystem/Continuation.cc deleted file mode 100644 index c6936322dca..00000000000 --- a/iocore/eventsystem/Continuation.cc +++ /dev/null @@ -1,42 +0,0 @@ -/** @file - - Contination.cc - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "I_EventSystem.h" -#include "I_Continuation.h" - -int -Continuation::dispatchEvent(int event, void *data) -{ - if (mutex) { - EThread *t = this_ethread(); - MUTEX_TRY_LOCK(lock, this->mutex, t); - if (!lock.is_locked()) { - t->schedule_imm(this, event, data); - return 0; - } else { - return (this->*handler)(event, data); - } - } else { - return (this->*handler)(event, data); - } -} diff --git a/iocore/eventsystem/I_Continuation.h b/iocore/eventsystem/I_Continuation.h index bd73f715492..140a11da841 100644 --- a/iocore/eventsystem/I_Continuation.h +++ b/iocore/eventsystem/I_Continuation.h @@ -165,20 +165,6 @@ class Continuation : private force_VFPT_to_top return (this->*handler)(event, data); } - /** - Receives the event code and data for an Event. - - It will attempt to get the lock for the continuation, and reschedule - the event if the lock cannot be obtained. If the lock can be obtained - dispatchEvent acts like handleEvent. - - @param event Event code to be passed at callback (Processor specific). - @param data General purpose data related to the event code (Processor specific). - @return State machine and processor specific return code. - - */ - int dispatchEvent(int event = CONTINUATION_EVENT_NONE, void *data = nullptr); - protected: /** Constructor of the Continuation object. It should not be used diff --git a/iocore/eventsystem/Makefile.am b/iocore/eventsystem/Makefile.am index d481f40b82b..6f44a16b983 100644 --- a/iocore/eventsystem/Makefile.am +++ b/iocore/eventsystem/Makefile.am @@ -59,7 +59,6 @@ libinkevent_a_SOURCES = \ P_UnixSocketManager.h \ P_VConnection.h \ P_VIO.h \ - Continuation.cc \ Processor.cc \ ProtectedQueue.cc \ ProxyAllocator.cc \ diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index ca5e3417794..2e9edfcd988 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -1087,8 +1087,15 @@ UnixNetVConnection::acceptEvent(int event, Event *e) if (active_timeout_in) { UnixNetVConnection::set_active_timeout(active_timeout_in); } - - action_.continuation->dispatchEvent(NET_EVENT_ACCEPT, this); + if (action_.continuation->mutex != nullptr) { + MUTEX_TRY_LOCK(lock3, action_.continuation->mutex, t); + if (!lock3.is_locked()) { + ink_release_assert(0); + } + action_.continuation->handleEvent(NET_EVENT_ACCEPT, this); + } else { + action_.continuation->handleEvent(NET_EVENT_ACCEPT, this); + } return EVENT_DONE; } diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index 13b11258fe2..887fe5e2b86 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -1221,7 +1221,11 @@ HttpConfig::startup() OutboundConnTrack::config_init(&c.outbound_conntrack, &c.oride.outbound_conntrack); - http_config_cont->dispatchEvent(EVENT_NONE, nullptr); + MUTEX_TRY_LOCK(lock, http_config_cont->mutex, this_ethread()); + if (!lock.is_locked()) { + ink_release_assert(0); + } + http_config_cont->handleEvent(EVENT_NONE, nullptr); return; } diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 48ccfc65ca3..514e9f9d814 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -555,7 +555,7 @@ HttpSM::setup_client_read_request_header() ua_entry->read_vio = ua_txn->do_io_read(this, INT64_MAX, ua_buffer_reader->mbuf); // The header may already be in the buffer if this // a request from a keep-alive connection - dispatchEvent(VC_EVENT_READ_READY, ua_entry->read_vio); + handleEvent(VC_EVENT_READ_READY, ua_entry->read_vio); } void @@ -888,7 +888,7 @@ HttpSM::state_watch_for_client_abort(int event, void *data) "[%" PRId64 "] [watch_for_client_abort] " "forwarding event %s to tunnel", sm_id, HttpDebugNames::get_event_name(event)); - tunnel.dispatchEvent(event, c->write_vio); + tunnel.handleEvent(event, c->write_vio); return 0; } else { tunnel.kill_tunnel(); @@ -4015,7 +4015,7 @@ HttpSM::do_remap_request(bool run_inline) if (!ret) { SMDebug("url_rewrite", "Could not find a valid remapping entry for this request [%" PRId64 "]", sm_id); if (!run_inline) { - dispatchEvent(EVENT_REMAP_COMPLETE, nullptr); + handleEvent(EVENT_REMAP_COMPLETE, nullptr); } return; } @@ -5425,13 +5425,13 @@ HttpSM::handle_server_setup_error(int event, void *data) ua_producer->alive = false; ua_producer->handler_state = HTTP_SM_POST_SERVER_FAIL; - tunnel.dispatchEvent(VC_EVENT_ERROR, c->write_vio); + tunnel.handleEvent(VC_EVENT_ERROR, c->write_vio); return; } } else { // c could be null here as well if (c != nullptr) { - tunnel.dispatchEvent(event, c->write_vio); + tunnel.handleEvent(event, c->write_vio); return; } } diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 86ca78be6b8..84634c37bb2 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -776,7 +776,7 @@ HttpTunnel::tunnel_run(HttpTunnelProducer *p_arg) // back to say we are done if (!is_tunnel_alive()) { active = false; - sm->dispatchEvent(HTTP_TUNNEL_EVENT_DONE, this); + sm->handleEvent(HTTP_TUNNEL_EVENT_DONE, this); } } @@ -1640,7 +1640,7 @@ HttpTunnel::main_handler(int event, void *data) if (reentrancy_count == 1) { reentrancy_count = 0; active = false; - sm->dispatchEvent(HTTP_TUNNEL_EVENT_DONE, this); + sm->handleEvent(HTTP_TUNNEL_EVENT_DONE, this); return EVENT_DONE; } else { call_sm = true; diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index fc18b721c08..810435a7183 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -1295,7 +1295,16 @@ APIHook::invoke(int event, void *edata) ink_assert(!"not reached"); } } - return m_cont->dispatchEvent(event, edata); + if (m_cont->mutex != nullptr) { + MUTEX_TRY_LOCK(lock, m_cont->mutex, this_ethread()); + if (!lock.is_locked()) { + // If we cannot get the lock, the caller needs to restructure to handle rescheduling + ink_release_assert(0); + } + return m_cont->handleEvent(event, edata); + } else { + return m_cont->handleEvent(event, edata); + } } APIHook * @@ -4510,7 +4519,16 @@ int TSContCall(TSCont contp, TSEvent event, void *edata) { Continuation *c = (Continuation *)contp; - return c->dispatchEvent((int)event, edata); + if (c->mutex != nullptr) { + MUTEX_TRY_LOCK(lock, c->mutex, this_ethread()); + if (!lock.is_locked()) { + // If we cannot get the lock, the caller needs to restructure to handle rescheduling + ink_release_assert(0); + } + return c->handleEvent((int)event, edata); + } else { + return c->handleEvent((int)event, edata); + } } TSMutex From 0b80592bc472d5394c777d0fca66f72779276cf2 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 6 Dec 2018 08:58:00 -0700 Subject: [PATCH 076/526] Deals better with Cc: max-age=0 There's a small window where we can still serve cached responses even when they have a Cache-Control: max-age=0. This fixes the cache-tests checks for id=freshness-max-age-0 and id=freshness-max-age-0-expires. --- proxy/http/HttpTransact.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 3c92d0265ce..2c06b77c909 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -7324,7 +7324,7 @@ HttpTransact::what_is_document_freshness(State *s, HTTPHdr *client_request, HTTP // now, see if the age is "fresh enough" // /////////////////////////////////////////// - if (do_revalidate || current_age > age_limit) { // client-modified limit + if (do_revalidate || !age_limit || current_age > age_limit) { // client-modified limit TxnDebug("http_match", "[..._document_freshness] document needs revalidate/too old; " "returning FRESHNESS_STALE"); return (FRESHNESS_STALE); From 5f69cd63bb309fdab72518c357e22efe730eaa0c Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 5 Dec 2018 13:13:10 -0800 Subject: [PATCH 077/526] Cleans up some plugin READMEs Fixes doc URLs and removes references to tsxs builds --- plugins/experimental/cache_range_requests/README | 11 ----------- plugins/experimental/collapsed_forwarding/README | 16 +--------------- plugins/header_rewrite/README | 2 +- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/plugins/experimental/cache_range_requests/README b/plugins/experimental/cache_range_requests/README index 12ea4d03608..39cd0fcd11d 100644 --- a/plugins/experimental/cache_range_requests/README +++ b/plugins/experimental/cache_range_requests/README @@ -16,17 +16,6 @@ object is written to cache using the new cache key url. The response code sent to the client will be changed back to a 206 and all requests to the origin server will contain the range header so that the correct response is received. -Installation: - - make - sudo make install - -If you don't have the traffic server binaries in your path, then you will need -to specify the path to tsxs manually: - - make TSXS=/opt/trafficserver/bin/tsxs - sudo make TSXS=/opt/trafficserver/bin/tsxs install - Configuration: Add @plugin=cache_range_requests.so to your remap.config rules. diff --git a/plugins/experimental/collapsed_forwarding/README b/plugins/experimental/collapsed_forwarding/README index 49ee7f882f3..ca0a0025972 100644 --- a/plugins/experimental/collapsed_forwarding/README +++ b/plugins/experimental/collapsed_forwarding/README @@ -25,24 +25,10 @@ // contention and so can not work. The setting proxy.config.http.wait_for_cache // may be enabled which allows blocking incoming connections from being // accepted until cache is ready. -//////////////////////////////////////////////////////////////////////////////////// -// This plugin currently supports only per-remap mode activation. -//////////////////////////////////////////////////////////////////////////////////// More details are available at -https://docs.trafficserver.apache.org/en/6.0.x/admin/http-proxy-caching.en.html#reducing-origin-server-requests-avoiding-the-thundering-herd - -Installation: - - make - sudo make install - -If you don't have the traffic server binaries in your path, then you will need -to specify the path to tsxs manually: - - make TSXS=/opt/trafficserver/bin/tsxs - sudo make TSXS=/opt/trafficserver/bin/tsxs install +https://docs.trafficserver.apache.org/en/latest/admin-guide/configuration/cache-basics.en.html#reducing-origin-server-requests-avoiding-the-thundering-herd Configuration: diff --git a/plugins/header_rewrite/README b/plugins/header_rewrite/README index 40ea174625b..62fb930b486 100644 --- a/plugins/header_rewrite/README +++ b/plugins/header_rewrite/README @@ -24,4 +24,4 @@ during the origin response header parsing, using READ_RESPONSE_HDR_HOOK. For more information on how to use this plugin, please see - http://docs.trafficserver.apache.org/en/latest/reference/plugins/header_rewrite.en.html + https://docs.trafficserver.apache.org/en/latest/admin-guide/plugins/header_rewrite.en.html From 0ec4f643af03b34c3f7dfd81f69ef0ae7b69c254 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 6 Dec 2018 12:33:54 -0600 Subject: [PATCH 078/526] Doc: Cleanup warnings. --- doc/admin-guide/files/parent.config.en.rst | 2 +- doc/admin-guide/plugins/header_rewrite.en.rst | 36 ++++++++++--------- .../api/functions/TSContCall.en.rst | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/doc/admin-guide/files/parent.config.en.rst b/doc/admin-guide/files/parent.config.en.rst index ca548251206..4f4de28f117 100644 --- a/doc/admin-guide/files/parent.config.en.rst +++ b/doc/admin-guide/files/parent.config.en.rst @@ -143,7 +143,7 @@ The following list shows the possible actions and their allowed values. If ``round_robin`` is set to ``consistent_hash``, you may add a ``unique hash string`` following the ``weight`` for each parent. The ``hash string`` must start with ``&`` - and is used to build both the primary and secondary rings using the ``hash string`` + and is used to build both the primary and secondary rings using the ``hash string`` for each parent insted of the parents ``hostname`` or ``ip address``. This can be useful so that two different hosts may be used to cache the same requests. Example:: diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index edb39359735..e0eeec37418 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -1,18 +1,14 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under the License. .. include:: ../../common.defs @@ -820,9 +816,9 @@ String concatenation is not yet supported in condition testing. Note: In versions prior to ATS v9.0.0, an alternative string expansion was available. those expansions are no longer available, but the following table can help migrations: -======================== ================================================================================== +======================== ========================================================================== Old expansion variable Condition variable to use with concatenatinos -======================== ================================================================================== +======================== ========================================================================== % %{CLIENT-URL:SCHEME} % %{CLIENT-URL:PORT} % %{IP:CLIENT}, %{INBOUND:REMOTE-ADDR} or e.g. %{CIDR:24,48} @@ -830,6 +826,7 @@ Old expansion variable Condition variable to use with concatenatinos % %{METHOD} % %[CLIENT-URL} % %{CLIENT-URL:PATH} +======================== ========================================================================== Header Values ------------- @@ -865,13 +862,18 @@ The URL part names which may be used for these conditions and actions are: Part Description ======== ====================================================================== HOST Full hostname. -PATH URL substring beginning with (but not including) the first ``/`` after the hostname up to, - but not including, the query string. + +PATH URL substring beginning with (but not including) the first ``/`` after + the hostname up to, but not including, the query string. + PORT Port number. + QUERY URL substring from the ``?``, signifying the beginning of the query parameters, until the end of the URL. Empty string if there were no - quuery parameters. + query parameters. + SCHEME URL scheme in use (e.g. ``http`` and ``https``). + URL The complete URL. ======== ====================================================================== diff --git a/doc/developer-guide/api/functions/TSContCall.en.rst b/doc/developer-guide/api/functions/TSContCall.en.rst index 5453bb3cd45..8f99332ac27 100644 --- a/doc/developer-guide/api/functions/TSContCall.en.rst +++ b/doc/developer-guide/api/functions/TSContCall.en.rst @@ -44,7 +44,7 @@ As a result :func:`TSContCall` will effectively do:: return CallbackHandler(contp, event, edata); -If there is a mutex associated with :arg:`contp`, :func:`TSComtCall` assumes that mutex is held already. +If there is a mutex associated with :arg:`contp`, :func:`TSContCall` assumes that mutex is held already. :func:`TSContCall` will directly call the handler associated with the continuation. It will return the value returned by the handler in :arg:`contp`. From a7d147d7b4696ea760284fba2ce5e37181eedbfa Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Thu, 15 Nov 2018 17:18:34 +0900 Subject: [PATCH 079/526] Doc: parent config has more features than balancer plugin The balancer plugin has consistent hashing, and round robin (not weighted). As a remap plugin, it is limited to url attribnutes. It neither has passive nor active health checks. Really, Balancer plugin should be deprecated. --- doc/admin-guide/introduction.en.rst | 154 ++++++++++++------------ doc/admin-guide/plugins/balancer.en.rst | 4 + 2 files changed, 79 insertions(+), 79 deletions(-) diff --git a/doc/admin-guide/introduction.en.rst b/doc/admin-guide/introduction.en.rst index f6b40ba3797..06effa3c9ae 100644 --- a/doc/admin-guide/introduction.en.rst +++ b/doc/admin-guide/introduction.en.rst @@ -30,102 +30,98 @@ nightmare for IT professionals as they struggle with overloaded servers and congested networks. It can be challenging to consistently and reliably accommodate society’s growing data demands. -Traffic Server is a high-performance web proxy cache that improves +|TS| is a high-performance web proxy cache that improves network efficiency and performance by caching frequently-accessed information at the edge of the network. This brings content physically closer to end users, while enabling faster delivery and reduced -bandwidth use. Traffic Server is designed to improve content delivery +bandwidth use. |TS| is designed to improve content delivery for enterprises, Internet service providers (ISPs), backbone providers, and large intranets by maximizing existing and available bandwidth. -Traffic Server Deployment Options -================================= +|TS| Deployment Options +======================= -To best suit your needs, Traffic Server can be deployed in several ways: +To best suit your needs, |TS| can be deployed in several ways: - As a web proxy cache - As a reverse proxy - In a cache hierarchy -The following sections provide a summary of these Traffic Server +The following sections provide a summary of these |TS| deployment options. -Traffic Server as a Web Proxy Cache ------------------------------------ +|TS| as a Web Proxy Cache +------------------------- -As a web proxy cache, Traffic Server receives user requests for web +As a web proxy cache, |TS| receives user requests for web content as those requests travel to the destined web server (origin -server). If Traffic Server contains the requested content, then it +server). If |TS| contains the requested content, then it serves the content directly. If the requested content is not available -from cache, then Traffic Server acts as a proxy: it obtains the content +from cache, then |TS| acts as a proxy: it obtains the content from the origin server on the user’s behalf and also keeps a copy to satisfy future requests. -Traffic Server provides explicit proxy caching, in which the user’s +|TS| provides explicit proxy caching, in which the user’s client software must be configured to send requests directly to Traffic Server. Explicit proxy caching is described in the :ref:`explicit-proxy-caching` chapter. -Traffic Server can also be employed as a transparent caching proxy server, in +|TS| can also be employed as a transparent caching proxy server, in which the client software needs no special configuration or even knowledge of the proxy's existence. This setup is described in the :ref:`transparent-proxy` section. -Traffic Server as a Reverse Proxy ---------------------------------- +|TS| as a Reverse Proxy +----------------------- -As a reverse proxy, Traffic Server is configured to be the origin server +As a reverse proxy, |TS| is configured to be the origin server to which the user is trying to connect (typically, the origin server’s -advertised hostname resolves to Traffic Server, which acts as the real +advertised hostname resolves to |TS|, which acts as the real origin server). The reverse proxy feature is also called server acceleration. Reverse proxy is described in more detail in :ref:`reverse-proxy-and-http-redirects`. -Traffic Server in a Cache Hierarchy ------------------------------------ +|TS| in a Cache Hierarchy +------------------------- -Traffic Server can participate in flexible cache hierarchies, in which +|TS| can participate in flexible cache hierarchies, in which Internet requests not fulfilled from one cache are routed to other regional caches, thereby leveraging the contents and proximity of nearby -caches. In a hierarchy of proxy servers, Traffic Server can act either -as a parent or a child cache to other Traffic Server systems or to +caches. In a hierarchy of proxy servers, |TS| can act either +as a parent or a child cache to other |TS| systems or to similar caching products. -Deployment Limitations ----------------------- +|TS| as a Load Balancer +----------------------- -There are a number of deployment options that Traffic Server does not support -right out of the box. Such functionality may be implemented in a plugin, but -in some cases Traffic Server's internal APIs or architectural restrictions -won't make it easy: +|TS| can act as a layer 7 HTTP load balancer distributing requests across +several servers. It can choose the next hop server using request attributes +like the Host: header, URL attributes, scheme, method, and client IP address. +It has a few selection strategies in place like weighted round robin, and +URL consistent hashing. -* Load Balancing - note that there is an experimental plugin for this, - :ref:`admin-plugins-balancer`. +|TS| Components +=============== -.. XXX needs re-work: the leadin states there's "a number" of scenarios, and then only lists one -- one's a number, but not much of a list - -Traffic Server Components -========================= - -Traffic Server consists of several components that work together to form +|TS| consists of several components that work together to form a web proxy cache you can easily monitor and configure. -The Traffic Server Cache ------------------------- +The |TS| Cache +-------------- -The Traffic Server cache consists of a high-speed object database called +The |TS| cache consists of a high-speed object database called the *object store*. The object store indexes objects according to URLs and associated headers. Using sophisticated object management, the object store can cache alternate versions of the same object (perhaps in a different language or encoding type). It can also efficiently store very small and very large objects, thereby minimizing wasted space. When the -cache is full, Traffic Server removes stale data to ensure that the most +cache is full, |TS| removes stale data to ensure that the most requested objects are readily available and fresh. -Traffic Server is designed to tolerate total disk failures on any of the -cache disks. If the disk fails completely, then Traffic Server marks the +|TS| is designed to tolerate total disk failures on any of the +cache disks. If the disk fails completely, then |TS| marks the entire disk as corrupt and continues to use remaining disks. If all of -the cache disks fail, then Traffic Server switches to proxy-only mode. +the cache disks fail, then |TS| switches to proxy-only mode. You can partition the cache to reserve a certain amount of disk space for storing data for specific protocols and origin servers. For more information about the cache, see :ref:`http-proxy-caching`. @@ -133,7 +129,7 @@ information about the cache, see :ref:`http-proxy-caching`. The RAM Cache ------------- -Traffic Server maintains a small RAM cache that contains extremely +|TS| maintains a small RAM cache that contains extremely popular objects. This RAM cache serves the most popular objects as fast as possible and reduces load on disks, especially during temporary traffic peaks. You can configure the RAM cache size to suit your needs. @@ -142,8 +138,8 @@ For detailed information, refer to :ref:`changing-the-size-of-the-ram-cache`. The Host Database ----------------- -The Traffic Server host database stores the domain name server (DNS) -entries of origin servers to which Traffic Server connects to fulfill +The |TS| host database stores the domain name server (DNS) +entries of origin servers to which |TS| connects to fulfill user requests. This information is used to adapt future protocol interactions and optimize performance. Along with other information, the host database tracks: @@ -159,26 +155,26 @@ host database tracks: The DNS Resolver ---------------- -Traffic Server includes a fast, asynchronous DNS resolver to streamline -conversion of hostnames to IP addresses. Traffic Server implements the +|TS| includes a fast, asynchronous DNS resolver to streamline +conversion of hostnames to IP addresses. |TS| implements the DNS resolver natively by directly issuing DNS command packets rather than relying on slower, conventional resolver libraries. Since many DNS queries can be issued in parallel and a fast DNS cache maintains popular bindings in memory, DNS traffic is reduced. -Traffic Server Processes ------------------------- +|TS| Processes +-------------- -Traffic Server contains three processes that work together to serve +|TS| contains three processes that work together to serve requests and manage, control, and monitor the health of the system. - The :program:`traffic_server` process is the transaction processing engine - of Traffic Server. It is responsible for accepting connections, + of |TS|. It is responsible for accepting connections, processing protocol requests, and serving documents from the cache or origin server. - The :program:`traffic_manager` process is the command and control facility - of the Traffic Server, responsible for launching, monitoring, and + of the |TS|, responsible for launching, monitoring, and reconfiguring the :program:`traffic_server` process. The :program:`traffic_manager` process is also responsible for the proxy autoconfiguration port, the statistics interface, and virtual IP failover. @@ -191,51 +187,51 @@ requests and manage, control, and monitor the health of the system. first-served order. This connection queueing shields users from any server restart downtime. -The figure below illustrates the Traffic Server processes. +The figure below illustrates the |TS| processes. .. figure:: ../static/images/admin/process.jpg :align: center - :alt: Illustration of the Traffic Server Processes + :alt: Illustration of the |TS| Processes - Illustration of the Traffic Server Processes + Illustration of the |TS| Processes Administration Tools -------------------- -Traffic Server offers the following administration options: +|TS| offers the following administration options: - The :program:`traffic_ctl` command-line interface is a - text-based interface from which you can monitor Traffic Server performance - and network traffic, as well as configure the Traffic Server system. + text-based interface from which you can monitor |TS| performance + and network traffic, as well as configure the |TS| system. -- Various configuration files enable you to configure Traffic Server +- Various configuration files enable you to configure |TS| through a simple file-editing and signal-handling interface. Any changes you make through :program:`traffic_ctl` are automatically made to the configuration files as well. - Finally, there is a clean C API which can be put to good use from a - multitude of languages. The Traffic Server Admin Client demonstrates + multitude of languages. The |TS| Admin Client demonstrates this for Perl. Traffic Analysis Options ======================== -Traffic Server provides several options for network traffic analysis and +|TS| provides several options for network traffic analysis and monitoring: - :program:`traffic_ctl` enables you to collect and process statistics obtained from network traffic information. - Transaction logging enables you to record information (in a log file) - about every request Traffic Server receives and every error it + about every request |TS| receives and every error it detects. By analyzing the log files, you can determine how many - clients used the Traffic Server cache, how much information each of + clients used the |TS| cache, how much information each of them requested, and what pages were most popular. You can also see why a particular transaction was in error and what state the Traffic Server was in at a particular time. For example, you can see that - Traffic Server was restarted. + |TS| was restarted. - Traffic Server supports several standard log file formats, such as + |TS| supports several standard log file formats, such as Squid and Netscape, and its own custom format. You can analyze the standard format log files with off-the-shelf analysis packages. To help with log file analysis, you can separate log files so that they @@ -244,37 +240,37 @@ monitoring: |TS| event and error logging, monitoring, and analysis is covered in greater detail in :ref:`admin-monitoring`. -Traffic Server Security Options -=============================== +|TS| Security Options +===================== -Traffic Server provides numerous options that enable you to establish -secure communication between the Traffic Server system and other +|TS| provides numerous options that enable you to establish +secure communication between the |TS| system and other computers on the network. Using the security options, you can do the following: -- Control client access to the Traffic Server proxy cache. +- Control client access to the |TS| proxy cache. -- Configure Traffic Server to use multiple DNS servers to match your - site's security configuration. For example, Traffic Server can use +- Configure |TS| to use multiple DNS servers to match your + site's security configuration. For example, |TS| can use different DNS servers, depending on whether it needs to resolve hostnames located inside or outside a firewall. This enables you to keep your internal network configuration secure while continuing to provide transparent access to external sites on the Internet. -- Configure Traffic Server to verify that clients are authenticated - before they can access content from the Traffic Server cache. +- Configure |TS| to verify that clients are authenticated + before they can access content from the |TS| cache. - Secure connections in reverse proxy mode between a client and Traffic - Server, and Traffic Server and the origin server, using the SSL + Server, and |TS| and the origin server, using the SSL termination option. - Control access via SSL (Secure Sockets Layer). -Traffic Server security options are described in more detail in +|TS| security options are described in more detail in :ref:`admin-security`. -Tuning Traffic Server -===================== +Tuning |TS| +=========== Finally, this last chapter on :ref:`performance-tuning` discusses the vast number of options that allow administrators to optimally tune Apache Traffic diff --git a/doc/admin-guide/plugins/balancer.en.rst b/doc/admin-guide/plugins/balancer.en.rst index 48b9a17dadf..8283c3818a6 100644 --- a/doc/admin-guide/plugins/balancer.en.rst +++ b/doc/admin-guide/plugins/balancer.en.rst @@ -20,6 +20,10 @@ Balancer Plugin specific language governing permissions and limitations under the License. +.. note:: + + All of the the features in this plugin (and more) are found in + :file:`parent.config`. As a result, this plugin is likely to be deprecated. The ``balancer`` balances requests across multiple origin servers. To use this plugin, configure it in a :file:`remap.config` rule, specifying From 2ff2866ba57a59c3c46b0eab98c9f8d0eca79f86 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 28 Nov 2018 19:51:08 +0000 Subject: [PATCH 080/526] Add SNI-based cert selection test. --- .../tls/tls_check_cert_selection.test.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/gold_tests/tls/tls_check_cert_selection.test.py diff --git a/tests/gold_tests/tls/tls_check_cert_selection.test.py b/tests/gold_tests/tls/tls_check_cert_selection.test.py new file mode 100644 index 00000000000..01684d69ffc --- /dev/null +++ b/tests/gold_tests/tls/tls_check_cert_selection.test.py @@ -0,0 +1,106 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test ATS offering different certificates based on SNI +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server", ssl=True) +dns = Test.MakeDNServer("dns") + +request_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed2-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map / https://foo.com:{1}'.format(ts.Variables.ssl_port, server.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLines([ + 'ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key', + 'ssl_cert_name=signed2-bar.pem ssl_key_name=signed-bar.key', + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +]) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|ssl|dns|hostdb', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL' +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + +# Should receive a bar.com cert +tr = Test.AddTestRun("bar.com cert") +tr.Setup.Copy("ssl/signer.pem") +tr.Setup.Copy("ssl/signer2.pem") +tr.Processes.Default.Command = "curl -v --cacert signer2.pem --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=bar.com", "Cert should contain bar.com") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=foo.com", "Cert should not contain foo.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") +tr.TimeOut = 5 + +# Should receive a foo.com cert +tr2 = Test.AddTestRun("foo.com cert") +tr2.Processes.Default.Command = "curl -v --cacert signer.pem --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=foo.com", "Cert should contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") +tr2.TimeOut = 5 + From f606676613ee233efd959b8f93b66b0696b1f294 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 21 Nov 2018 12:06:58 -0700 Subject: [PATCH 081/526] Adds a reasonable rip-grep config file For this to be used though, you have to do e.g. export RIPGREP_CONFIG_PATH=.ripgreprc or make a rip-grep alias that sets it for a specific command only --- .ripgreprc | 7 +++++++ ci/rat-regex.txt | 1 + 2 files changed, 8 insertions(+) create mode 100644 .ripgreprc diff --git a/.ripgreprc b/.ripgreprc new file mode 100644 index 00000000000..5286a8a9c5a --- /dev/null +++ b/.ripgreprc @@ -0,0 +1,7 @@ +--color=auto +--no-heading +--smart-case +--line-number +--multiline +--trim +--one-file-system diff --git a/ci/rat-regex.txt b/ci/rat-regex.txt index 1fda181301f..93a24d5b087 100644 --- a/ci/rat-regex.txt +++ b/ci/rat-regex.txt @@ -28,6 +28,7 @@ ^\.indent.pro$ ^\.vimrc$ ^\.clang-.*$ +^\.ripgreprc$ ^Doxyfile$ ^CHANGES$ ^CHANGELOG.*$ From f4fb8fe3e5fd6639f69afbc2db1dd655a5c787d1 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 30 Nov 2018 14:31:27 -0600 Subject: [PATCH 082/526] Cleanup: change RemapPlugins::run_single_remap to return bool. --- proxy/http/remap/RemapPlugins.cc | 18 ++++++++---------- proxy/http/remap/RemapPlugins.h | 2 +- proxy/http/remap/RemapProcessor.cc | 6 ++---- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/proxy/http/remap/RemapPlugins.cc b/proxy/http/remap/RemapPlugins.cc index cd71cb77917..8900e545bcb 100644 --- a/proxy/http/remap/RemapPlugins.cc +++ b/proxy/http/remap/RemapPlugins.cc @@ -78,13 +78,13 @@ RemapPlugins::run_plugin(remap_plugin_info *plugin) there actually *is* something to do). */ -int +bool RemapPlugins::run_single_remap() { url_mapping *map = _s->url_map.getMapping(); remap_plugin_info *plugin = map->get_plugin(_cur); // get the nth plugin in our list of plugins TSRemapStatus plugin_retcode = TSREMAP_NO_REMAP; - int zret = 1; + bool zret = true; // default - last iteration. Debug("url_rewrite", "running single remap rule id %d for the %d%s time", map->map_id, _cur, _cur == 1 ? "st" : _cur == 2 ? "nd" : _cur == 3 ? "rd" : "th"); @@ -108,7 +108,7 @@ RemapPlugins::run_single_remap() Debug("url_rewrite", "completed all remap plugins for rule id %d, changed by %d plugins", map->map_id, _rewritten); } else { Debug("url_rewrite", "completed single remap, attempting another via immediate callback"); - zret = 0; // not done yet. + zret = false; // not done yet. } // If the chain is finished, and the URL hasn't been rewritten, do the rule remap. @@ -128,8 +128,6 @@ RemapPlugins::run_remap(int event, Event *e) ink_assert(action.continuation); ink_assert(action.continuation); - int ret = 0; - /* make sure we weren't cancelled */ if (action.cancelled) { mutex.clear(); @@ -140,14 +138,14 @@ RemapPlugins::run_remap(int event, Event *e) switch (event) { case EVENT_IMMEDIATE: Debug("url_rewrite", "handling immediate event inside RemapPlugins::run_remap"); - ret = run_single_remap(); /** - * If ret !=0 then we are done with this processor and we call back into the SM; - * otherwise, we call this function again immediately (which really isn't immediate) - * thru the eventProcessor, thus forcing another run of run_single_remap() which will + * If @c run_single_remap returns @c true then we are done with this processor and we call back + * into the SM; otherwise, we call this function again immediately (which really isn't + * immediate) thru the eventProcessor, thus forcing another run of run_single_remap() which will * then operate on _request_url, etc performing additional remaps (mainly another plugin run) + * **/ - if (ret) { + if (run_single_remap()) { action.continuation->handleEvent(EVENT_REMAP_COMPLETE, nullptr); mutex.clear(); action.mutex.clear(); diff --git a/proxy/http/remap/RemapPlugins.h b/proxy/http/remap/RemapPlugins.h index fad40976e12..eafff41c402 100644 --- a/proxy/http/remap/RemapPlugins.h +++ b/proxy/http/remap/RemapPlugins.h @@ -67,7 +67,7 @@ struct RemapPlugins : public Continuation { } int run_remap(int event, Event *e); - int run_single_remap(); + bool run_single_remap(); TSRemapStatus run_plugin(remap_plugin_info *plugin); Action action; diff --git a/proxy/http/remap/RemapProcessor.cc b/proxy/http/remap/RemapProcessor.cc index e22b710f882..b8776df27c9 100644 --- a/proxy/http/remap/RemapProcessor.cc +++ b/proxy/http/remap/RemapProcessor.cc @@ -318,11 +318,9 @@ RemapProcessor::perform_remap(Continuation *cont, HttpTransact::State *s) return &plugins->action; } else { RemapPlugins plugins(s, request_url, request_header, hh_info); - int ret = 0; - do { - ret = plugins.run_single_remap(); - } while (ret == 0); + while (!plugins.run_single_remap()) + ; // EMPTY return ACTION_RESULT_DONE; } From c4619bba50fa3dd3c48bbbc69f8d3a242e1ba39e Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 14 Nov 2018 16:10:27 -0600 Subject: [PATCH 083/526] BWF: Change timestamp global name to be consistent with logging time stamps. --- src/tscore/BufferWriterFormat.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/tscore/BufferWriterFormat.cc b/src/tscore/BufferWriterFormat.cc index e3bf9a7843c..7efd5e04eae 100644 --- a/src/tscore/BufferWriterFormat.cc +++ b/src/tscore/BufferWriterFormat.cc @@ -965,12 +965,14 @@ namespace void BWF_Timestamp(ts::BufferWriter &w, ts::BWFSpec const &spec) { - // Unfortunately need to write to a temporary buffer or the sizing isn't correct if @a w is clipped - // because @c strftime returns 0 if the buffer isn't large enough. - char buff[32]; - std::time_t t = std::time(nullptr); - auto n = strftime(buff, sizeof(buff), "%Y %b %d %H:%M:%S", std::localtime(&t)); - w.write(buff, n); + auto now = std::chrono::system_clock::now(); + auto epoch = std::chrono::system_clock::to_time_t(now); + ts::LocalBufferWriter<48> lw; + + ctime_r(&epoch, lw.auxBuffer()); + lw.fill(19); // clip trailing stuff, do not want. + lw.print(".{:03}", std::chrono::time_point_cast(now).time_since_epoch().count() % 1000); + w.write(lw.view().substr(4)); } void From cb2727667511b803d0bf664da42a660de0411fe7 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Sat, 20 Oct 2018 15:30:56 -0500 Subject: [PATCH 084/526] YAML: LibSWOC prep - update BufferWriter logic in HttpTransactHeaders.cc --- proxy/http/HttpTransactHeaders.cc | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/proxy/http/HttpTransactHeaders.cc b/proxy/http/HttpTransactHeaders.cc index 95f662626c7..5f18843b4d8 100644 --- a/proxy/http/HttpTransactHeaders.cc +++ b/proxy/http/HttpTransactHeaders.cc @@ -1155,25 +1155,19 @@ HttpTransactHeaders::add_forwarded_field_to_request(HttpTransact::State *s, HTTP if (n_proto > 0) { auto Conn = [&](HttpForwarded::Option opt, HttpTransactHeaders::ProtocolStackDetail detail) -> void { - if (optSet[opt]) { - int revert = hdr.size(); + if (optSet[opt] && hdr.remaining() > 0) { + ts::FixedBufferWriter lw{hdr.auxBuffer(), hdr.remaining()}; if (hdr.size()) { - hdr << ';'; + lw << ';'; } - hdr << "connection="; + lw << "connection="; int numChars = - HttpTransactHeaders::write_hdr_protocol_stack(hdr.auxBuffer(), hdr.remaining(), detail, protoBuf.data(), n_proto, '-'); - if (numChars > 0) { - hdr.fill(size_t(numChars)); - } - - if ((numChars <= 0) or (hdr.size() >= hdr.capacity())) { - // Remove parameter with potentially incomplete value. - // - hdr.reduce(revert); + HttpTransactHeaders::write_hdr_protocol_stack(lw.auxBuffer(), lw.remaining(), detail, protoBuf.data(), n_proto, '-'); + if (numChars > 0 && !lw.fill(size_t(numChars)).error()) { + hdr.fill(lw.size()); } } }; From 5350c6541856f70f61b901ddd3a8e37341b7f522 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Thu, 6 Dec 2018 19:05:29 -0800 Subject: [PATCH 085/526] remove ignoring unused variable warning from webp plugin --- plugins/experimental/webp_transform/Makefile.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/experimental/webp_transform/Makefile.inc b/plugins/experimental/webp_transform/Makefile.inc index 15ed0446259..fd24dabcaa4 100644 --- a/plugins/experimental/webp_transform/Makefile.inc +++ b/plugins/experimental/webp_transform/Makefile.inc @@ -16,7 +16,7 @@ # limitations under the License. experimental_webp_transform_WebpTransform_la_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMAGICKCPP_CFLAGS) -experimental_webp_transform_WebpTransform_la_CXXFLAGS = $(AM_CXXFLAGS) -Wno-unused-variable +experimental_webp_transform_WebpTransform_la_CXXFLAGS = $(AM_CXXFLAGS) pkglib_LTLIBRARIES += experimental/webp_transform/WebpTransform.la From a2dc839fc234f7697db637655d277b83fc4a5ea8 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 7 Dec 2018 14:09:18 -0700 Subject: [PATCH 086/526] Updates the CI build scripts, adds a new build for HTTP cache-tests --- ci/jenkins/bin/cache-tests.sh | 50 +++++++++++++++++++++++++++++++++++ ci/jenkins/bin/environment.sh | 9 +++++-- ci/jenkins/bin/gh-mirror.sh | 2 ++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100755 ci/jenkins/bin/cache-tests.sh diff --git a/ci/jenkins/bin/cache-tests.sh b/ci/jenkins/bin/cache-tests.sh new file mode 100755 index 00000000000..f285b99e810 --- /dev/null +++ b/ci/jenkins/bin/cache-tests.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Change to the build area (this is previously setup in extract.sh) +set +x +PREFIX="${WORKSPACE}/${BUILD_NUMBER}/install" + +cd "${WORKSPACE}/${BUILD_NUMBER}/build" + +./configure \ + --prefix=${PREFIX} \ + --with-user=jenkins \ + --enable-debug \ + --enable-ccache + +${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= +${ATS_MAKE} install + +# Setup and start ATS with the required remap rule +echo "map http://127.0.0.1:8080 http://192.168.3.13:8000" >> ${PREFIX}/etc/trafficserver/remap.config +${PREFIX}/bin/trafficserver start + +# Get NPM v10, and run the tests +source /opt/rh/rh-nodejs10/enable +set -x + +cd /home/jenkins/cache-tests +npm run --silent cli --base=http://127.0.0.1:8080/ > /CA/cache-tests/${ATS_BRANCH}.json + +${PREFIX}/bin/trafficserver stop + +cat /CA/cache-tests/${ATS_BRANCH}.json + +# We should check the .json file here, but not yet +exit 0 diff --git a/ci/jenkins/bin/environment.sh b/ci/jenkins/bin/environment.sh index 079035f6dd1..2790694bd1d 100755 --- a/ci/jenkins/bin/environment.sh +++ b/ci/jenkins/bin/environment.sh @@ -44,13 +44,18 @@ ATS_IS_7="yes" test "${JOB_NAME#*-6.2.x}" != "${JOB_NAME}" && ATS_BRANCH=6.2.x && ATS_IS_7="no" test "${JOB_NAME#*-7.1.x}" != "${JOB_NAME}" && ATS_BRANCH=7.1.x -test "${JOB_NAME#*-7.2.x}" != "${JOB_NAME}" && ATS_BRANCH=7.2.x test "${JOB_NAME#*-8.0.x}" != "${JOB_NAME}" && ATS_BRANCH=8.0.x test "${JOB_NAME#*-8.1.x}" != "${JOB_NAME}" && ATS_BRANCH=8.1.x test "${JOB_NAME#*-8.2.x}" != "${JOB_NAME}" && ATS_BRANCH=8.2.x +test "${JOB_NAME#*-8.3.x}" != "${JOB_NAME}" && ATS_BRANCH=8.3.x test "${JOB_NAME#*-9.0.x}" != "${JOB_NAME}" && ATS_BRANCH=9.0.x test "${JOB_NAME#*-9.1.x}" != "${JOB_NAME}" && ATS_BRANCH=9.1.x -test "${JOB_NAME#*-9.2.x}" != "${JOB_NAME}" && ATS_BRANCH=9.1.x +test "${JOB_NAME#*-9.2.x}" != "${JOB_NAME}" && ATS_BRANCH=9.2.x +test "${JOB_NAME#*-9.3.x}" != "${JOB_NAME}" && ATS_BRANCH=9.3.x +test "${JOB_NAME#*-10.0.x}" != "${JOB_NAME}" && ATS_BRANCH=10.0.x +test "${JOB_NAME#*-10.1.x}" != "${JOB_NAME}" && ATS_BRANCH=10.1.x +test "${JOB_NAME#*-10.2.x}" != "${JOB_NAME}" && ATS_BRANCH=10.2.x +test "${JOB_NAME#*-10.3.x}" != "${JOB_NAME}" && ATS_BRANCH=10.3.x export ATS_BRANCH echo "Branch is $ATS_BRANCH" diff --git a/ci/jenkins/bin/gh-mirror.sh b/ci/jenkins/bin/gh-mirror.sh index 48d22f0028f..4f9fab92d0d 100755 --- a/ci/jenkins/bin/gh-mirror.sh +++ b/ci/jenkins/bin/gh-mirror.sh @@ -69,6 +69,7 @@ function checkBuild() { REF_6_2=$(getRef "6.2.x") REF_7_1=$(getRef "7.1.x") REF_8_0=$(getRef "8.0.x") +REF_8_1=$(getRef "8.1.x") REF_master=$(getRef "master") # Do the updates @@ -79,4 +80,5 @@ ${GIT} update-server-info checkBuild "$REF_6_2" "6.2.x" checkBuild "$REF_7_1" "7.1.x" checkBuild "$REF_8_0" "8.0.x" +checkBuild "$REF_8_1" "8.1.x" checkBuild "$REF_master" "master" From bb8ef7bbf023b4b64b9f97d3715ed44a0c128b3c Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Thu, 6 Dec 2018 09:54:29 -0600 Subject: [PATCH 087/526] remove unnecessary TSHandleMLocRelease --- src/tscpp/api/Request.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tscpp/api/Request.cc b/src/tscpp/api/Request.cc index 83bd1901d93..eec8244354a 100644 --- a/src/tscpp/api/Request.cc +++ b/src/tscpp/api/Request.cc @@ -204,10 +204,6 @@ Request::~Request() TSMLoc null_parent_loc = nullptr; TSHandleMLocRelease(state_->hdr_buf_, null_parent_loc, state_->url_loc_); TSMBufferDestroy(state_->hdr_buf_); - } else { - LOG_DEBUG("Destroying request object on hdr_buf=%p, hdr_loc=%p, url_loc=%p", state_->hdr_buf_, state_->hdr_loc_, - state_->url_loc_); - TSHandleMLocRelease(state_->hdr_buf_, state_->hdr_loc_, state_->url_loc_); } } delete state_; From 840d7c06e97ed7ab9191a7cf796be5062e54d929 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Wed, 12 Dec 2018 11:47:17 +0800 Subject: [PATCH 088/526] Correct the statements within ink_assert and ink_release_assert --- proxy/http/HttpSM.cc | 8 ++++---- proxy/logging/LogObject.cc | 2 +- src/tscore/CryptoHash.cc | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 514e9f9d814..e677a8e808f 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1632,7 +1632,7 @@ HttpSM::handle_api_return() state_remove_from_list(EVENT_NONE, nullptr); return; default: - ink_release_assert("! Not reached"); + ink_release_assert(!"Not reached"); break; } @@ -2563,7 +2563,7 @@ HttpSM::state_cache_open_read(int event, void *data) break; default: - ink_release_assert("!Unknown event"); + ink_release_assert(!"Unknown event"); break; } @@ -3980,7 +3980,7 @@ HttpSM::state_remap_request(int event, void * /* data ATS_UNUSED */) } default: - ink_assert("Unexpected event inside state_remap_request"); + ink_assert(!"Unexpected event inside state_remap_request"); break; } @@ -7559,7 +7559,7 @@ HttpSM::set_next_state() } default: { - ink_release_assert("!Unknown next action"); + ink_release_assert(!"Unknown next action"); } } } diff --git a/proxy/logging/LogObject.cc b/proxy/logging/LogObject.cc index c745bddb277..6ccb416f473 100644 --- a/proxy/logging/LogObject.cc +++ b/proxy/logging/LogObject.cc @@ -1338,7 +1338,7 @@ LogObjectManager::log(LogAccess *lad) } else if (likely(ret & Log::SKIP)) { RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_event_log_access_skip_stat, 1); } else { - ink_release_assert("Unexpected result"); + ink_release_assert(!"Unexpected result"); } return ret; diff --git a/src/tscore/CryptoHash.cc b/src/tscore/CryptoHash.cc index fec88fb84e0..926e5f2452b 100644 --- a/src/tscore/CryptoHash.cc +++ b/src/tscore/CryptoHash.cc @@ -55,7 +55,7 @@ CryptoContext::CryptoContext() break; #endif default: - ink_release_assert("Invalid global URL hash context"); + ink_release_assert(!"Invalid global URL hash context"); }; #if TS_ENABLE_FIPS == 0 static_assert(CryptoContext::OBJ_SIZE >= sizeof(MD5Context), "bad OBJ_SIZE"); From 9da865baad311542ecd4142a7a111b6e7e46c595 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Mon, 10 Dec 2018 12:12:24 -0600 Subject: [PATCH 089/526] sslheaders experimental plugin: fix doc typo, improve container use. --- doc/admin-guide/plugins/sslheaders.en.rst | 2 +- plugins/experimental/sslheaders/sslheaders.cc | 18 ++++++++---------- plugins/experimental/sslheaders/sslheaders.h | 8 +++++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/admin-guide/plugins/sslheaders.en.rst b/doc/admin-guide/plugins/sslheaders.en.rst index 8fc08816ca4..49a8777b5f3 100644 --- a/doc/admin-guide/plugins/sslheaders.en.rst +++ b/doc/admin-guide/plugins/sslheaders.en.rst @@ -66,7 +66,7 @@ The `client.certificate` and `server.certificate` fields emit the corresponding certificate in PEM format, with newline characters replaced by spaces. -If the ``sslheaders`` plugin activtes on non-SSL connections, it +If the ``sslheaders`` plugin activates on non-SSL connections, it will delete all the configured HTTP header names so that malicious clients cannot inject misleading information. If any of the SSL fields expand to an empty string, those headers are also deleted. diff --git a/plugins/experimental/sslheaders/sslheaders.cc b/plugins/experimental/sslheaders/sslheaders.cc index 085c7ac4b05..6c6838d5040 100644 --- a/plugins/experimental/sslheaders/sslheaders.cc +++ b/plugins/experimental/sslheaders/sslheaders.cc @@ -124,14 +124,14 @@ SslHdrExpand(SSL *ssl, const SslHdrInstance::expansion_list &expansions, TSMBuff { if (ssl == nullptr) { for (const auto &expansion : expansions) { - SslHdrRemoveHeader(mbuf, mhdr, expansion->name); + SslHdrRemoveHeader(mbuf, mhdr, expansion.name); } } else { X509 *x509; BIO *exp = BIO_new(BIO_s_mem()); for (const auto &expansion : expansions) { - switch (expansion->scope) { + switch (expansion.scope) { case SSL_HEADERS_SCOPE_CLIENT: x509 = SSL_get_peer_certificate(ssl); break; @@ -146,15 +146,15 @@ SslHdrExpand(SSL *ssl, const SslHdrInstance::expansion_list &expansions, TSMBuff continue; } - SslHdrExpandX509Field(exp, x509, expansion->field); + SslHdrExpandX509Field(exp, x509, expansion.field); if (BIO_pending(exp)) { - SslHdrSetHeader(mbuf, mhdr, expansion->name, exp); + SslHdrSetHeader(mbuf, mhdr, expansion.name, exp); } else { - SslHdrRemoveHeader(mbuf, mhdr, expansion->name); + SslHdrRemoveHeader(mbuf, mhdr, expansion.name); } // Getting the peer certificate takes a reference count, but the server certificate doesn't. - if (x509 && expansion->scope == SSL_HEADERS_SCOPE_CLIENT) { + if (x509 && expansion.scope == SSL_HEADERS_SCOPE_CLIENT) { X509_free(x509); } } @@ -199,14 +199,12 @@ SslHdrParseOptions(int argc, const char **argv) } // Pick up the remaining options as SSL header expansions. + hdr->expansions.resize(argc - optind); for (int i = optind; i < argc; ++i) { - SslHdrExpansion exp; - if (!SslHdrParseExpansion(argv[i], exp)) { + if (!SslHdrParseExpansion(argv[i], hdr->expansions[i - optind])) { // If we fail, the expansion parsing logs the error. return nullptr; } - - hdr->expansions.push_back(&exp); } return hdr.release(); diff --git a/plugins/experimental/sslheaders/sslheaders.h b/plugins/experimental/sslheaders/sslheaders.h index b7b66ddbaa1..ff72100ec03 100644 --- a/plugins/experimental/sslheaders/sslheaders.h +++ b/plugins/experimental/sslheaders/sslheaders.h @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include extern "C" { @@ -67,13 +67,15 @@ struct SslHdrExpansion { ExpansionScope scope; ExpansionField field; - // noncopyable + // noncopyable but moveable SslHdrExpansion(const SslHdrExpansion &) = delete; SslHdrExpansion &operator=(const SslHdrExpansion &) = delete; + SslHdrExpansion(SslHdrExpansion &&) = default; + SslHdrExpansion &operator=(SslHdrExpansion &&) = default; }; struct SslHdrInstance { - typedef std::list expansion_list; + typedef std::vector expansion_list; SslHdrInstance(); ~SslHdrInstance(); From a95301b390dd34f3bc35e588a3c90633c349f6c1 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 11 Dec 2018 10:47:38 -0700 Subject: [PATCH 090/526] Fixes some cache-tests, and make check should fail --- ci/jenkins/bin/build.sh | 2 +- ci/jenkins/bin/cache-tests.sh | 8 ++++++-- ci/jenkins/bin/environment.sh | 1 + ci/jenkins/bin/regression.sh | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ci/jenkins/bin/build.sh b/ci/jenkins/bin/build.sh index a1e52fc8f37..9a9fd354a01 100755 --- a/ci/jenkins/bin/build.sh +++ b/ci/jenkins/bin/build.sh @@ -48,4 +48,4 @@ set -x ${WERROR} \ ${DEBUG} -${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= +${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= || exit 1 diff --git a/ci/jenkins/bin/cache-tests.sh b/ci/jenkins/bin/cache-tests.sh index f285b99e810..9fece1111d3 100755 --- a/ci/jenkins/bin/cache-tests.sh +++ b/ci/jenkins/bin/cache-tests.sh @@ -28,8 +28,12 @@ cd "${WORKSPACE}/${BUILD_NUMBER}/build" --enable-debug \ --enable-ccache -${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= -${ATS_MAKE} install +# Not great, but these can fail on the "docs' builds for older versions, sigh +${ATS_MAKE} -i ${ATS_MAKE_FLAGS} V=1 Q= +${ATS_MAKE} -i install + +[ -x ${PREFIX}/bin/traffic_server ] || exit 1 + # Setup and start ATS with the required remap rule echo "map http://127.0.0.1:8080 http://192.168.3.13:8000" >> ${PREFIX}/etc/trafficserver/remap.config diff --git a/ci/jenkins/bin/environment.sh b/ci/jenkins/bin/environment.sh index 2790694bd1d..e76cf55522e 100755 --- a/ci/jenkins/bin/environment.sh +++ b/ci/jenkins/bin/environment.sh @@ -42,6 +42,7 @@ export TODAY=$(/bin/date +'%m%d%Y') ATS_BRANCH=master ATS_IS_7="yes" +test "${JOB_NAME#*-5.3.x}" != "${JOB_NAME}" && ATS_BRANCH=5.3.x && ATS_IS_7="no" test "${JOB_NAME#*-6.2.x}" != "${JOB_NAME}" && ATS_BRANCH=6.2.x && ATS_IS_7="no" test "${JOB_NAME#*-7.1.x}" != "${JOB_NAME}" && ATS_BRANCH=7.1.x test "${JOB_NAME#*-8.0.x}" != "${JOB_NAME}" && ATS_BRANCH=8.0.x diff --git a/ci/jenkins/bin/regression.sh b/ci/jenkins/bin/regression.sh index c21725691e1..62c1f3237a5 100755 --- a/ci/jenkins/bin/regression.sh +++ b/ci/jenkins/bin/regression.sh @@ -19,6 +19,7 @@ cd "${WORKSPACE}/${BUILD_NUMBER}/build" [ -d BUILDS ] && cd BUILDS -${ATS_MAKE} check VERBOSE=Y V=1 && ${ATS_MAKE} install +${ATS_MAKE} check VERBOSE=Y V=1 || exit 1 +${ATS_MAKE} install || exit "${WORKSPACE}/${BUILD_NUMBER}/install/bin/traffic_server" -k -K -R 1 From 7f63d1630a11cd324b8ebdd606d896a5b475c279 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Mon, 10 Dec 2018 11:36:59 -0600 Subject: [PATCH 091/526] Make sslheaders plugin better conform to documentation. --- plugins/experimental/sslheaders/expand.cc | 2 +- plugins/experimental/sslheaders/sslheaders.cc | 66 +++++++++++--- .../pluginTest/sslheaders/observer.py | 31 +++++++ .../pluginTest/sslheaders/ssl/server.key | 28 ++++++ .../pluginTest/sslheaders/ssl/server.pem | 21 +++++ .../pluginTest/sslheaders/sslheaders.gold | 1 + .../pluginTest/sslheaders/sslheaders.test.py | 91 +++++++++++++++++++ 7 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 tests/gold_tests/pluginTest/sslheaders/observer.py create mode 100644 tests/gold_tests/pluginTest/sslheaders/ssl/server.key create mode 100644 tests/gold_tests/pluginTest/sslheaders/ssl/server.pem create mode 100644 tests/gold_tests/pluginTest/sslheaders/sslheaders.gold create mode 100644 tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py diff --git a/plugins/experimental/sslheaders/expand.cc b/plugins/experimental/sslheaders/expand.cc index 5d99853ea6f..647bdbc6cdf 100644 --- a/plugins/experimental/sslheaders/expand.cc +++ b/plugins/experimental/sslheaders/expand.cc @@ -113,7 +113,7 @@ static const std::array expansions = {{ x509_expand_serial, // SSL_HEADERS_FIELD_SERIAL x509_expand_signature, // SSL_HEADERS_FIELD_SIGNATURE x509_expand_notbefore, // SSL_HEADERS_FIELD_NOTBEFORE - x509_expand_notafter, // SSL_HEADERS_FIELD_NOTBEFORE + x509_expand_notafter, // SSL_HEADERS_FIELD_NOTAFTER }}; bool diff --git a/plugins/experimental/sslheaders/sslheaders.cc b/plugins/experimental/sslheaders/sslheaders.cc index 6c6838d5040..657a9cd34ed 100644 --- a/plugins/experimental/sslheaders/sslheaders.cc +++ b/plugins/experimental/sslheaders/sslheaders.cc @@ -116,6 +116,49 @@ SslHdrSetHeader(TSMBuffer mbuf, TSMLoc mhdr, const std::string &name, BIO *value } } +namespace +{ +template class WrapX509 +{ +public: + WrapX509(SSL *ssl) : _ssl(ssl), _x509(_nonNullInvalidValue()) {} + + X509 * + get() + { + if (_x509 == _nonNullInvalidValue()) { + _set(); + } + + return _x509; + } + + ~WrapX509() + { + if (IsClient && (_x509 != _nonNullInvalidValue()) && (_x509 != nullptr)) { + X509_free(_x509); + } + } + +private: + SSL *_ssl; + X509 *_x509; + + // The address of this object can not be a valid X509 structure address. + X509 * + _nonNullInvalidValue() const + { + return reinterpret_cast(const_cast(this)); + } + + void + _set() + { + _x509 = (IsClient ? SSL_get_peer_certificate : SSL_get_certificate)(_ssl); + } +}; +} // end anonymous namespace + // Process SSL header expansions. If this is not an SSL connection, then we need to delete the SSL headers // so that malicious clients cannot inject bogus information. Otherwise, we populate the header with the // expanded value. If the value expands to something empty, we nuke the header. @@ -127,22 +170,28 @@ SslHdrExpand(SSL *ssl, const SslHdrInstance::expansion_list &expansions, TSMBuff SslHdrRemoveHeader(mbuf, mhdr, expansion.name); } } else { + WrapX509 clientX509(ssl); + WrapX509 serverX509(ssl); X509 *x509; + BIO *exp = BIO_new(BIO_s_mem()); for (const auto &expansion : expansions) { switch (expansion.scope) { case SSL_HEADERS_SCOPE_CLIENT: - x509 = SSL_get_peer_certificate(ssl); + x509 = clientX509.get(); + if (x509 == nullptr) { + SslHdrRemoveHeader(mbuf, mhdr, expansion.name); + continue; + } break; case SSL_HEADERS_SCOPE_SERVER: - x509 = SSL_get_certificate(ssl); + x509 = serverX509.get(); + if (x509 == nullptr) { + continue; + } break; default: - x509 = nullptr; - } - - if (x509 == nullptr) { continue; } @@ -152,11 +201,6 @@ SslHdrExpand(SSL *ssl, const SslHdrInstance::expansion_list &expansions, TSMBuff } else { SslHdrRemoveHeader(mbuf, mhdr, expansion.name); } - - // Getting the peer certificate takes a reference count, but the server certificate doesn't. - if (x509 && expansion.scope == SSL_HEADERS_SCOPE_CLIENT) { - X509_free(x509); - } } BIO_free(exp); diff --git a/tests/gold_tests/pluginTest/sslheaders/observer.py b/tests/gold_tests/pluginTest/sslheaders/observer.py new file mode 100644 index 00000000000..673a56eb1a2 --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/observer.py @@ -0,0 +1,31 @@ +''' +Extract ssl-* headers and store in a log file for later verification. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log = open('sslheaders.log', 'w') + +def observe(headers): + + for h in headers.items(): + if h[0].lower().startswith('ssl-'): + + log.write(h[0] + ": " + h[1] + "\n"); + log.write("-\n") + log.flush() + +Hooks.register(Hooks.ReadRequestHook, observe) diff --git a/tests/gold_tests/pluginTest/sslheaders/ssl/server.key b/tests/gold_tests/pluginTest/sslheaders/ssl/server.key new file mode 100644 index 00000000000..9cdfc36aa5f --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCZkEXSlZ+ZFKFg +CPpcDG39e73BuK6E5uE38q2PHh4DV0xcsJnIUx51viqLPwYughxfP0crHyBdXoHV +dW/3WX4gpiGrdiM/dvCouheo0DPaqUUJ2nZKVYh2M57oyeiuJidlKb7BGkfw3HWP +9TV7dVyGWok/cowjopqaLHJWxg/kh2KqvUBD0CHt9Kd1XvgXVmHwE7vCv0j5owv2 +MaExTsFb16uWmVLhl1gNHI2RqCX2yLaebH1DvtbLrit1XErjtaSYeJE9clVRaqT6 +vsvLOhyB5tA9WfZqfBYr/MHDeXQfrbIf+4Cp3aTpq5grc5InIMMH0eOk6/f/4tW+ +nq1lfszZAgMBAAECggEAYvYAqRbXRRVwca0Xel5gO2yU+tSDUw5esWlow8RK3yhR +A6KjV9+Iz6P/UsEIwMwEcLUcrgNfHgybau5Fe4dmqq+lHxQA3xNNP869FIMoB4/x +98mbVYgNau8VRztnAWOBG8ZtMZA4MFZCRMVm8+rL96E8tXCiMwzEyPo/rP/ymfhN +3GRunX+GhfIA79AYNbd7HMVL+cvWWUGUF5Bc5i1wXcLy4I7b9NYtv920BeCLzSFK +BypFB7ku/vKgTcBxe4yxThxPeXPwm4WFzGYKk/Afl1j8tVXCE2U4Y3yykfC0Qk6S +ECZbCKLO2Rxi9fclIDZBHWuKejZhdjHfjeNvZ2vLoQKBgQDJzLmkVLvWAxgl1yvF +U7gwqj/TzYqtVowbjEvTNEnPU1j/hIVI343SVV/EvJmif/iRUop6sRYfLsUjpMsH +CmPysNKL3UtgSYOxLs+0xLhG4OOQRpPSf/uvl9YyWY9G3AqiC7ScthkQjEhZa4c1 +eycYy0jr42kX0OL9MuIH9q0ENQKBgQDCzvGKMs8r5E/Qd3YB9VYB60dx+6G83AHZ +YqIelykObhCdxL9n4K+p4VKKLvgTcCOLYYIkBSWRJWR+ue3s3ey9+XWd2/q4Xvfh +TCjAuO2ibMV+y5ClNlW0fQ/doIVWSDbjO2tZW1jh7YWZ4CtuVrsEisv1sk3KltMB +MguhpTUylQKBgG6TfrncMFzxrx61C+gBmvEXqQffHfkjbnx94OKnSTaQ3jiNHhez +X9v8KhD8o1bWtpay2uyl8pA9qYqBdzqxZ9kJKSW4qd/mCIJjOy87iBpWint5IPD8 +biZmldlbF9ZlJnJq5ZnlclCN/er5r8oPZHoCkj+nieOh8294nUBt25ptAoGAMnPA +EIeaKgbmONpHgLhWPwb9KOL/f1cHT5KA5CVH58nPmdyTqcaCGCAX7Vu+ueIIApgN +SWDf2thxT3S9zuOm5YiO0oRfSZKm5f2AbHE4ciFzgKQd4PvSdH0TN9XT0oW/WVhR +NAI5YcHPIQvyk4/4vXNo4Uf9Z6NqIFwisQmFXoUCgYBK/ZI/HsFsvnR5MV0tFdGM +AdNe6bsQRSZkowoaPxuWbklE4Hn6QvwEmQg3ST2O+vCQV1f1yI6YiWYoptOYscJc +MSs/HxhhaaO5ZsiuPUO6WEPzpNb2CxuIGDDtl83VtUQyjxCmOb6pqqjwzFmZ2bsw +JNMaBCzokrJTxknvauCuSQ== +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/sslheaders/ssl/server.pem b/tests/gold_tests/pluginTest/sslheaders/ssl/server.pem new file mode 100644 index 00000000000..2b56cc83ea2 --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/ssl/server.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDZDCCAkygAwIBAgIJANod1+h9CtCaMA0GCSqGSIb3DQEBCwUAMEcxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRowGAYDVQQDDBFy +YW5kb20uc2VydmVyLmNvbTAeFw0xODExMTkxNzEwMTlaFw0yODExMTYxNzEwMTla +MEcxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRow +GAYDVQQDDBFyYW5kb20uc2VydmVyLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAJmQRdKVn5kUoWAI+lwMbf17vcG4roTm4TfyrY8eHgNXTFywmchT +HnW+Kos/Bi6CHF8/RysfIF1egdV1b/dZfiCmIat2Iz928Ki6F6jQM9qpRQnadkpV +iHYznujJ6K4mJ2UpvsEaR/DcdY/1NXt1XIZaiT9yjCOimposclbGD+SHYqq9QEPQ +Ie30p3Ve+BdWYfATu8K/SPmjC/YxoTFOwVvXq5aZUuGXWA0cjZGoJfbItp5sfUO+ +1suuK3VcSuO1pJh4kT1yVVFqpPq+y8s6HIHm0D1Z9mp8Fiv8wcN5dB+tsh/7gKnd +pOmrmCtzkicgwwfR46Tr9//i1b6erWV+zNkCAwEAAaNTMFEwHQYDVR0OBBYEFI2y +qm0+UAChDAnLrAINeFOuyUlhMB8GA1UdIwQYMBaAFI2yqm0+UAChDAnLrAINeFOu +yUlhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAA3ZNFbqxcOX +szS5A4EXCepyBJBFejEYy0CsvwQX/ai/pMrw5jqVeF0GAOTpBCVLddyY+ZV1arD2 +Pqi7Qwot9OxEZOzbCBiuMJGotruKgnWFQDHzJ9HA7KDQs270uNESAOG/xW9os9zN +MXApzqfBSR5EIQU5L3RtaiPzoKdQenGQUOj86s0Kon7snDSUzaA2VcfstMWgGvXP +JHtaVusULm0gry32cEap5G5UK+gII6DfLWgFwFGhHHmTz3mKjyGiJQ+09XBtu4lb +ENE+HGRBBA49dUKSr3kwErO4HyHnS0YrsTDnbYURCsGUDma12oijX2sCos6Q4zn8 +3svaouRrucw= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/sslheaders/sslheaders.gold b/tests/gold_tests/pluginTest/sslheaders/sslheaders.gold new file mode 100644 index 00000000000..39cdd0ded6d --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/sslheaders.gold @@ -0,0 +1 @@ +- diff --git a/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py b/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py new file mode 100644 index 00000000000..26c3a432583 --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py @@ -0,0 +1,91 @@ +''' +Test the sslheaders plugin. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test sslheaders plugin. +''' + +Test.SkipUnless( + Condition.HasATSFeature('TS_USE_TLS_ALPN'), + Condition.HasCurlFeature('http2'), +) + +Test.Disk.File('sslheaders.log').Content = 'sslheaders.gold' + +server = Test.MakeOriginServer("server", options={'--load': Test.TestDirectory + '/observer.py'}) + +request_header = { + "headers": "GET / HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts = Test.MakeATSProcess("ts", select_ports=False) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +# ts.addSSLfile("ssl/signer.pem") + +ts.Variables.ssl_port = 4443 + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.cache.http': 0, # Make sure each request is forwarded to the origin server. + 'proxy.config.proxy_name': 'Poxy_Proxy', # This will be the server name. + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': ( + 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl ipv6:{0} ipv6:{1}:proto=http2;http:ssl' + .format(ts.Variables.port, ts.Variables.ssl_port)), +# 'proxy.config.ssl.client.verify.server': 0, +# 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +# 'proxy.config.url_remap.pristine_host_hdr' : 1, +# 'proxy.config.ssl.client.certification_level': 2, +# 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), +# 'proxy.config.ssl.TLSv1_3': 0 +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map http://bar.com http://127.0.0.1:{0}'.format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + 'map https://bar.com http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: "*bar.com"', + ' verify_client: STRICT', +]) + +ts.Disk.plugin_config.AddLine( + 'sslheaders.so SSL-Client-ID=client.subject' +) + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Command = ( + 'curl -H "SSL-Client-ID: My Fake Client ID" --verbose --ipv4 --insecure --header "Host: bar.com"' + + ' https://localhost:{}'.format(ts.Variables.ssl_port) +) +tr.Processes.Default.ReturnCode = 0 From acf3297e3eb8dffd0b71b53a756d73bc76157f8f Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Thu, 13 Dec 2018 15:44:01 -0600 Subject: [PATCH 092/526] Remove the APIHooks::invoke() function. In general, it is bad to have functions that are not used, and for which there is no clear potential use. This one is particularly bad. It creates the false impression that there is a possibility of simultaneous execution of continuations that are attached to the same hook (prior to calling the reenable function). --- proxy/InkAPIInternal.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/proxy/InkAPIInternal.h b/proxy/InkAPIInternal.h index c2bd1a86ca4..cd75d64fc47 100644 --- a/proxy/InkAPIInternal.h +++ b/proxy/InkAPIInternal.h @@ -135,7 +135,6 @@ class APIHooks APIHook *get() const; void clear(); bool is_empty() const; - void invoke(int event, void *data); private: Que(APIHook, m_link) m_hooks; @@ -147,13 +146,6 @@ APIHooks::is_empty() const return nullptr == m_hooks.head; } -inline void -APIHooks::invoke(int event, void *data) -{ - for (APIHook *hook = m_hooks.head; nullptr != hook; hook = hook->next()) - hook->invoke(event, data); -} - /** Container for API hooks for a specific feature. This is an array of hook lists, each identified by a numeric identifier (id). Each array element is a list of all From 0d7076e22fe64c66c9f5062d66c7e485a5ae224f Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Thu, 13 Dec 2018 14:28:52 -0600 Subject: [PATCH 093/526] Correction to documentation for TSHttpHookAdd(). --- doc/developer-guide/api/functions/TSHttpHookAdd.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst index 6435be88955..283e7e28bae 100644 --- a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst @@ -45,7 +45,7 @@ transaction, or for specific transactions only. HTTP :term:`transaction` hooks are set on a global basis using the function :func:`TSHttpHookAdd`. This means that the continuation specified as the parameter to :func:`TSHttpHookAdd` is called for every -transaction. :func:`TSHttpHookAdd` is typically called from +transaction. :func:`TSHttpHookAdd` must only be called from :func:`TSPluginInit` or :func:`TSRemapInit`. :func:`TSHttpSsnHookAdd` adds :arg:`contp` to From eef8d136db97fc499c4a8dbd1b5a5652b5a8e30e Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Thu, 13 Dec 2018 19:27:38 +0000 Subject: [PATCH 094/526] Revert "allow regex_revalidate per remap rule, remove direct ink includes" This reverts commit a89fb08794d0ffa00cf07e0825e55dcb9616bb22. --- .../plugins/regex_revalidate.en.rst | 5 +- plugins/regex_revalidate/regex_revalidate.c | 167 +++++------------- 2 files changed, 43 insertions(+), 129 deletions(-) diff --git a/doc/admin-guide/plugins/regex_revalidate.en.rst b/doc/admin-guide/plugins/regex_revalidate.en.rst index 50748820abb..a24fbeec349 100644 --- a/doc/admin-guide/plugins/regex_revalidate.en.rst +++ b/doc/admin-guide/plugins/regex_revalidate.en.rst @@ -24,8 +24,7 @@ Regex Revalidate Plugin This plugin allows for the creation of rules which match regular expressions against mapped URLs to determine if and when a cache object revalidation should -be forced. This plugin can be used both globally and for individual remap -rules. +be forced. Purpose ======= @@ -61,7 +60,7 @@ The rule configuration file format is described below in `Revalidation Rules`_. By default The plugin regularly (every 60 seconds) checks its rules configuration file for changes and it will also check for changes when ``traffic_ctl config reload`` -is run. If the file has been modified since its last scan, the contents +is run. If the file has been modified since its last scan, the contents are read and the in-memory rules list is updated. Thus, new rules may be added and existing ones modified without requiring a service restart. diff --git a/plugins/regex_revalidate/regex_revalidate.c b/plugins/regex_revalidate/regex_revalidate.c index 56713ea4df9..4b009df2b8b 100644 --- a/plugins/regex_revalidate/regex_revalidate.c +++ b/plugins/regex_revalidate/regex_revalidate.c @@ -19,8 +19,8 @@ limitations under the License. */ -#include "ts/ts.h" -#include "ts/remap.h" +#include "tscore/ink_defs.h" +#include "tscore/ink_platform.h" #include #include @@ -59,13 +59,6 @@ ts_free(void *s) return TSfree(s); } -static void -setup_memory_allocation() -{ - pcre_malloc = &ts_malloc; - pcre_free = &ts_free; -} - typedef struct invalidate_t { const char *regex_text; pcre *regex; @@ -338,7 +331,7 @@ list_config(plugin_state_t *pstate, invalidate_t *i) } static int -free_handler(TSCont cont, TSEvent event, void *edata) +free_handler(TSCont cont, TSEvent event ATS_UNUSED, void *edata ATS_UNUSED) { invalidate_t *iptr; @@ -350,7 +343,7 @@ free_handler(TSCont cont, TSEvent event, void *edata) } static int -config_handler(TSCont cont, TSEvent event, void *edata) +config_handler(TSCont cont, TSEvent event ATS_UNUSED, void *edata ATS_UNUSED) { plugin_state_t *pstate; invalidate_t *i, *iptr; @@ -460,23 +453,44 @@ main_handler(TSCont cont, TSEvent event, void *edata) return 0; } -TSRemapStatus -TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) +static bool +check_ts_version() { - TSCont main_cont = NULL; + const char *ts_version = TSTrafficServerVersionGet(); - main_cont = TSContCreate(main_handler, NULL); - TSContDataSet(main_cont, ih); - TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_REQUEST_HDR_HOOK, main_cont); + if (ts_version) { + int major_ts_version = 0; + int minor_ts_version = 0; + int micro_ts_version = 0; - return TSREMAP_NO_REMAP; + if (sscanf(ts_version, "%d.%d.%d", &major_ts_version, &minor_ts_version, µ_ts_version) != 3) { + return false; + } + + if ((TS_VERSION_MAJOR == major_ts_version) && (TS_VERSION_MINOR == minor_ts_version) && + (TS_VERSION_MICRO == micro_ts_version)) { + return true; + } + } + + return false; } -static bool -configure_plugin_state(plugin_state_t *pstate, int argc, const char **argv, bool *const disable_timed_reload) +void +TSPluginInit(int argc, const char *argv[]) { + TSPluginRegistrationInfo info; + TSCont main_cont, config_cont; + plugin_state_t *pstate; + invalidate_t *iptr = NULL; + bool disable_timed_reload = false; + + TSDebug(LOG_PREFIX, "Starting plugin init"); + + pstate = (plugin_state_t *)TSmalloc(sizeof(plugin_state_t)); + init_plugin_state_t(pstate); + int c; - invalidate_t *iptr = NULL; static const struct option longopts[] = {{"config", required_argument, NULL, 'c'}, {"log", required_argument, NULL, 'l'}, {"disable-timed-reload", no_argument, NULL, 'd'}, @@ -486,19 +500,16 @@ configure_plugin_state(plugin_state_t *pstate, int argc, const char **argv, bool switch (c) { case 'c': pstate->config_file = TSstrdup(optarg); - TSDebug(LOG_PREFIX, "Config File: %s", pstate->config_file); break; case 'l': if (TS_SUCCESS == TSTextLogObjectCreate(optarg, TS_LOG_MODE_ADD_TIMESTAMP, &pstate->log)) { TSTextLogObjectRollingEnabledSet(pstate->log, 1); TSTextLogObjectRollingIntervalSecSet(pstate->log, LOG_ROLL_INTERVAL); TSTextLogObjectRollingOffsetHrSet(pstate->log, LOG_ROLL_OFFSET); - TSDebug(LOG_PREFIX, "Logging Mode enabled"); } break; case 'd': - *disable_timed_reload = true; - TSDebug(LOG_PREFIX, "Timed reload disabled (disable-timed-reload)"); + disable_timed_reload = true; break; default: break; @@ -507,7 +518,8 @@ configure_plugin_state(plugin_state_t *pstate, int argc, const char **argv, bool if (!pstate->config_file) { TSError("[regex_revalidate] Plugin requires a --config option along with a config file name"); - return false; + free_plugin_state_t(pstate); + return; } if (!load_config(pstate, &iptr)) { @@ -517,104 +529,6 @@ configure_plugin_state(plugin_state_t *pstate, int argc, const char **argv, bool list_config(pstate, iptr); } - return true; -} - -TSReturnCode -TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) -{ - TSCont config_cont = NULL; - plugin_state_t *pstate = NULL; - bool disable_timed_reload = false; - - TSDebug(LOG_PREFIX, "Starting remap init"); - pstate = (plugin_state_t *)TSmalloc(sizeof(plugin_state_t)); - init_plugin_state_t(pstate); - - if (!configure_plugin_state(pstate, argc - 1, (const char **)(argv + 1), &disable_timed_reload)) { - free_plugin_state_t(pstate); - TSError("[regex_revalidate] Remap plugin registration failed"); - return TS_ERROR; - } - - *ih = (void *)pstate; - - config_cont = TSContCreate(config_handler, TSMutexCreate()); - TSContDataSet(config_cont, (void *)pstate); - - TSMgmtUpdateRegister(config_cont, LOG_PREFIX); - - if (!disable_timed_reload) { - TSContSchedule(config_cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); - } - - TSDebug(LOG_PREFIX, "Remap plugin registration succeeded"); - - return TS_SUCCESS; -} - -void -TSRemapDeleteInstance(void *ih) -{ - if (NULL != ih) { - plugin_state_t *const pstate = (plugin_state_t *)ih; - free_plugin_state_t(pstate); - } -} - -static bool -check_ts_version() -{ - const char *ts_version = TSTrafficServerVersionGet(); - - if (ts_version) { - int major_ts_version = 0; - int minor_ts_version = 0; - int micro_ts_version = 0; - - if (sscanf(ts_version, "%d.%d.%d", &major_ts_version, &minor_ts_version, µ_ts_version) != 3) { - return false; - } - - if ((TS_VERSION_MAJOR == major_ts_version)) { - return true; - } - } - - return false; -} - -TSReturnCode -TSRemapInit(TSRemapInterface *api_info, char *errbug, int errbuf_size) -{ - setup_memory_allocation(); - - if (!check_ts_version()) { - TSError("[regex_revalidate] Plugin requires Traffic Server %d", TS_VERSION_MAJOR); - return TS_ERROR; - } - - return TS_SUCCESS; -} - -void -TSPluginInit(int argc, const char *argv[]) -{ - TSPluginRegistrationInfo info; - TSCont main_cont, config_cont; - plugin_state_t *pstate = NULL; - bool disable_timed_reload = false; - - TSDebug(LOG_PREFIX, "Starting plugin init"); - - pstate = (plugin_state_t *)TSmalloc(sizeof(plugin_state_t)); - init_plugin_state_t(pstate); - - if (!configure_plugin_state(pstate, argc, argv, &disable_timed_reload)) { - free_plugin_state_t(pstate); - return; - } - info.plugin_name = LOG_PREFIX; info.vendor_name = "Apache Software Foundation"; info.support_email = "dev@trafficserver.apache.org"; @@ -629,12 +543,13 @@ TSPluginInit(int argc, const char *argv[]) } if (!check_ts_version()) { - TSError("[regex_revalidate] Plugin requires Traffic Server %d", TS_VERSION_MAJOR); + TSError("[regex_revalidate] Plugin requires Traffic Server %d.%d.%d", TS_VERSION_MAJOR, TS_VERSION_MINOR, TS_VERSION_MICRO); free_plugin_state_t(pstate); return; } - setup_memory_allocation(); + pcre_malloc = &ts_malloc; + pcre_free = &ts_free; main_cont = TSContCreate(main_handler, NULL); TSContDataSet(main_cont, (void *)pstate); From 083c5256f6b3c92b3f13ee8141db3c74e23feefb Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 29 Nov 2018 16:08:05 +0000 Subject: [PATCH 095/526] Add back in the option to conf_remap the verify_server settings. --- doc/admin-guide/files/records.config.en.rst | 8 +- include/ts/apidefs.h.in | 3 + iocore/net/P_SSLSNI.h | 6 +- iocore/net/SSLNetVConnection.cc | 22 ++- iocore/net/YamlSNIConfig.h | 8 +- plugins/lua/ts_lua_http_config.c | 6 + proxy/http/HttpConfig.h | 4 + proxy/http/HttpSM.cc | 42 ++++ src/traffic_server/InkAPI.cc | 23 ++- src/traffic_server/InkAPITest.cc | 5 +- .../tls/tls_verify_override.test.py | 185 ++++++++++++++++++ 11 files changed, 297 insertions(+), 15 deletions(-) create mode 100644 tests/gold_tests/tls/tls_verify_override.test.py diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 52ee8fcb5b0..249bd25c439 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3455,11 +3455,14 @@ Client-Related Configuration .. ts:cv:: CONFIG proxy.config.ssl.client.verify.server.policy STRING DISABLED :reloadable: + :overridable: Configures |TS| to verify the origin server certificate with the Certificate Authority (CA). This configuration takes a value of :code:`DISABLED`, :code:`PERMISSIVE`, or :code:`ENFORCED` - You can override this global setting on a per domain basis in the ssl_servername.yaml file using the :ref:`verify_server_policy attribute`. + You can override this global setting on a per domain basis in the ssl_server_name.yaml file using the :ref:`verify_server_policy attribute`. + + You can also override via the conf_remap plugin. Those changes will take precedence over the changes in ssl_server_name.yaml. :code:`DISABLED` Server Certificate will not be verified @@ -3470,11 +3473,14 @@ Client-Related Configuration .. ts:cv:: CONFIG proxy.config.ssl.client.verify.server.properties STRING ALL :reloadable: + :overridable: Configures |TS| for what the default verify callback should check during origin server verification. You can override this global setting on a per domain basis in the ssl_servername.yaml file using the :ref:`verify_server_properties attribute`. + You can also override via the conf_remap plugin. Those changes will take precedence over the changes in ssl_server_name.yaml. + :code:`NONE` Check nothing in the standard callback. Rely entirely on plugins to check the certificate. :code:`SIGNATURE` diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index b8ab9629187..6c042653d40 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -782,6 +782,9 @@ typedef enum { TS_CONFIG_HTTP_ALLOW_HALF_OPEN, TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX, TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH, + TS_CONFIG_SSL_CLIENT_VERIFY_SERVER, + TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, + TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, TS_CONFIG_LAST_ENTRY } TSOverridableConfigKey; diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index 15580ff05ff..a1d75045417 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -42,9 +42,9 @@ // Properties for the next hop server struct NextHopProperty { - std::string name; // name of the server - YamlSNIConfig::Policy verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; // whether to verify the next hop - YamlSNIConfig::Property verifyServerProperties = YamlSNIConfig::Property::NONE; // what to verify on the next hop + std::string name; // name of the server + YamlSNIConfig::Policy verifyServerPolicy = YamlSNIConfig::Policy::UNSET; // whether to verify the next hop + YamlSNIConfig::Property verifyServerProperties = YamlSNIConfig::Property::UNSET; // what to verify on the next hop SSL_CTX *ctx = nullptr; // ctx generated off the certificate to present to this server NextHopProperty(); }; diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 71e1b085162..96221d49696 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -971,15 +971,27 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) SSL_CTX *clientCTX = nullptr; if (nps) { - clientCTX = nps->ctx; - options.verifyServerPolicy = nps->verifyServerPolicy; - options.verifyServerProperties = nps->verifyServerProperties; + clientCTX = nps->ctx; + } else { // Just stay with the values passed down from the SM for verify + clientCTX = params->client_ctx; + } + if (options.verifyServerPolicy != YamlSNIConfig::Policy::UNSET) { + // Stay with conf-override version as the highest priority + } else if (nps && nps->verifyServerPolicy != YamlSNIConfig::Policy::UNSET) { + options.verifyServerPolicy = nps->verifyServerPolicy; + } else { + options.verifyServerPolicy = params->verifyServerPolicy; + } + + if (options.verifyServerProperties != YamlSNIConfig::Property::UNSET) { + // Stay with conf-override version as the highest priority + } else if (nps && nps->verifyServerProperties != YamlSNIConfig::Property::UNSET) { + options.verifyServerProperties = nps->verifyServerProperties; } else { - clientCTX = params->client_ctx; - options.verifyServerPolicy = params->verifyServerPolicy; options.verifyServerProperties = params->verifyServerProperties; } + if (!clientCTX) { SSLErrorVC(this, "failed to create SSL client session"); return EVENT_ERROR; diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h index 8926c29bdc2..0acf4f21f21 100644 --- a/iocore/net/YamlSNIConfig.h +++ b/iocore/net/YamlSNIConfig.h @@ -52,8 +52,8 @@ struct YamlSNIConfig { client_cert }; enum class Level { NONE = 0, MODERATE, STRICT }; - enum class Policy : uint8_t { DISABLED = 0, PERMISSIVE, ENFORCED }; - enum class Property : uint8_t { NONE = 0, SIGNATURE_MASK = 0x1, NAME_MASK = 0x2, ALL_MASK = 0x3 }; + enum class Policy : uint8_t { DISABLED = 0, PERMISSIVE, ENFORCED, UNSET }; + enum class Property : uint8_t { NONE = 0, SIGNATURE_MASK = 0x1, NAME_MASK = 0x2, ALL_MASK = 0x3, UNSET }; YamlSNIConfig() {} @@ -63,8 +63,8 @@ struct YamlSNIConfig { uint8_t verify_client_level = 255; std::string tunnel_destination; bool tunnel_decrypt = false; - Policy verify_server_policy = Policy::DISABLED; - Property verify_server_properties = Property::NONE; + Policy verify_server_policy = Policy::UNSET; + Property verify_server_properties = Property::UNSET; std::string client_cert; std::string client_key; std::string ip_allow; diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c index d7f25c61e3e..b7c63ea9219 100644 --- a/plugins/lua/ts_lua_http_config.c +++ b/plugins/lua/ts_lua_http_config.c @@ -134,6 +134,9 @@ typedef enum { TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE = TS_CONFIG_HTTP_ALLOW_MULTI_RANGE, TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED = TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, TS_LUA_CONFIG_HTTP_ALLOW_HALF_OPEN = TS_CONFIG_HTTP_ALLOW_HALF_OPEN, + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER, + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY, } TSLuaOverridableConfigKey; @@ -258,6 +261,9 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ALLOW_HALF_OPEN), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY), diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index fd61d074243..41d5e819324 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -492,6 +492,8 @@ struct OverridableHttpConfigParams { request_buffer_enabled(0), allow_half_open(1), ssl_client_verify_server(0), + ssl_client_verify_server_policy(nullptr), + ssl_client_verify_server_properties(nullptr), redirect_use_orig_cache_key(0), number_of_redirections(0), proxy_response_hsts_max_age(-1), @@ -677,6 +679,8 @@ struct OverridableHttpConfigParams { // server verification mode// ///////////////////////////// MgmtByte ssl_client_verify_server; + char *ssl_client_verify_server_policy; + char *ssl_client_verify_server_properties; ////////////////// // Redirection // diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index e677a8e808f..435da94f07d 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -4671,6 +4671,46 @@ HttpSM::send_origin_throttled_response() call_transact_and_set_next_state(HttpTransact::HandleResponse); } +static void +set_tls_options(NetVCOptions &opt, OverridableHttpConfigParams *txn_conf) +{ + char *verify_server = nullptr; + if (txn_conf->ssl_client_verify_server_policy == nullptr) { + opt.verifyServerPolicy = YamlSNIConfig::Policy::UNSET; + } else { + verify_server = txn_conf->ssl_client_verify_server_policy; + if (strcmp(verify_server, "DISABLED") == 0) { + opt.verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; + } else if (strcmp(verify_server, "PERMISSIVE") == 0) { + opt.verifyServerPolicy = YamlSNIConfig::Policy::PERMISSIVE; + } else if (strcmp(verify_server, "ENFORCED") == 0) { + opt.verifyServerPolicy = YamlSNIConfig::Policy::ENFORCED; + } else { + Warning("%s is invalid for proxy.config.ssl.client.verify.server.policy. Should be one of DISABLED, PERMISSIVE, or ENFORCED", + verify_server); + opt.verifyServerPolicy = YamlSNIConfig::Policy::UNSET; + } + } + if (txn_conf->ssl_client_verify_server_properties == nullptr) { + opt.verifyServerProperties = YamlSNIConfig::Property::UNSET; + } else { + verify_server = txn_conf->ssl_client_verify_server_properties; + if (strcmp(verify_server, "SIGNATURE") == 0) { + opt.verifyServerProperties = YamlSNIConfig::Property::SIGNATURE_MASK; + } else if (strcmp(verify_server, "NAME") == 0) { + opt.verifyServerProperties = YamlSNIConfig::Property::NAME_MASK; + } else if (strcmp(verify_server, "ALL") == 0) { + opt.verifyServerProperties = YamlSNIConfig::Property::ALL_MASK; + } else if (strcmp(verify_server, "NONE") == 0) { + opt.verifyServerProperties = YamlSNIConfig::Property::NONE; + } else { + Warning("%s is invalid for proxy.config.ssl.client.verify.server.properties. Should be one of SIGNATURE, NAME, or ALL", + verify_server); + opt.verifyServerProperties = YamlSNIConfig::Property::NONE; + } + } +} + ////////////////////////////////////////////////////////////////////////// // // HttpSM::do_http_server_open() @@ -4959,6 +4999,8 @@ HttpSM::do_http_server_open(bool raw) t_state.txn_conf->sock_option_flag_out, t_state.txn_conf->sock_packet_mark_out, t_state.txn_conf->sock_packet_tos_out); + set_tls_options(opt, t_state.txn_conf); + opt.ip_family = ip_family; if (ua_txn) { diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 810435a7183..7b17eb05ea9 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -8204,6 +8204,13 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr case TS_CONFIG_SSL_CERT_FILEPATH: ret = _memberp_to_generic(&overridableHttpConfig->client_cert_filepath, conv); break; + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER: + ret = _memberp_to_generic(&overridableHttpConfig->ssl_client_verify_server, conv); + break; + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY: + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES: + // String, must be handled elsewhere + break; case TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB: ret = _memberp_to_generic(&overridableHttpConfig->parent_failures_update_hostdb, conv); break; @@ -8408,6 +8415,16 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char } } break; + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_verify_server_policy = const_cast(value); + } + break; + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_verify_server_properties = const_cast(value); + } + break; default: { MgmtConverter const *conv; void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); @@ -8594,7 +8611,11 @@ static const std::unordered_map SDK_Overridable_Configs = { "proxy.config.http.request_buffer_enabled", "proxy.config.http.allow_half_open", OutboundConnTrack::CONFIG_VAR_MAX, - OutboundConnTrack::CONFIG_VAR_MATCH}}; + OutboundConnTrack::CONFIG_VAR_MATCH, + "proxy.config.ssl.client.verify.server", + "proxy.config.ssl.client.verify.server.policy", + "proxy.config.ssl.client.verify.server.properties"}}; REGRESSION_TEST(SDK_API_OVERRIDABLE_CONFIGS)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus) { diff --git a/tests/gold_tests/tls/tls_verify_override.test.py b/tests/gold_tests/tls/tls_verify_override.test.py new file mode 100644 index 00000000000..6afbfd52063 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_override.test.py @@ -0,0 +1,185 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options. Exercise conf_remap +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map http://foo.com/basic https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://foo.com/override https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/basic https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overridedisabled https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overridesignature https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overrideenforced https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /basic https://127.0.0.1:{0}'.format(server.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.ssl.client.verify.server.properties': 'ALL', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# Should succeed without message +tr = Test.AddTestRun("default-permissive-success") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = 'curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/basic'.format(ts.Variables.port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +# Should succed. No message +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.TimeOut = 5 + +tr2 = Test.AddTestRun("default-permissive-fail") +tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about name mismatch +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.TimeOut = 5 + +tr2 = Test.AddTestRun("default-permissive-fail2") +tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about signature +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.TimeOut = 5 + +tr3 = Test.AddTestRun("override-foo") +tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/override".format(ts.Variables.port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +# Should succeed. No error messages +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr3.Processes.Default.TimeOut = 5 + +tr4 = Test.AddTestRun("override-bar-disabled") +tr4.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" http://127.0.0.1:{0}/overridedisabled".format(ts.Variables.port) +tr4.ReturnCode = 0 +tr4.StillRunningAfter = server +tr4.StillRunningAfter = ts +# Succeed. No error messages +tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr4.Processes.Default.TimeOut = 5 + +tr5 = Test.AddTestRun("override-bar-signature-enforced") +tr5.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overridesignature".format(ts.Variables.port) +tr5.ReturnCode = 0 +tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr5.StillRunningAfter = server +tr5.StillRunningAfter = ts +tr5.Processes.Default.TimeOut = 5 + +tr6 = Test.AddTestRun("override-bar-enforced") +tr6.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overrideenforced".format(ts.Variables.port) +tr6.ReturnCode = 0 +# Should fail +tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") +tr6.StillRunningAfter = server +tr6.StillRunningAfter = ts +tr6.Processes.Default.TimeOut = 5 + + +# Over riding the built in ERROR check since we expect some cases to fail + +# checks on random.com should fail with message only +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=127.0.0.1\(127.0.0.1\) depth=0", "Warning for self signed certificate") +# No complaints about foo +ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI \(foo.com\) not in certificate", "foo.com name requests are good") +# permissive failure for bar.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", "Warning on missing name for bar.com") +# name check failure for random.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", "Warning on missing name for randome.com") +# name check failure for bar.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=127.0.0.1\(127.0.0.1\)", "Failure on missing name for bar.com") + + From f6fe5f0079005813d33d1cddd9d127c75a1f5e0b Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 18 Dec 2018 16:00:40 +0000 Subject: [PATCH 096/526] Revert "Two more places to check whether attempting half_closed connection logic is feasible." This reverts commit 653927f9aab9a3cbe8f09521dd8c8154ccbc1614. Erroneous fix caused KEEPALIVE to fail on HTTP/1.1 over HTTPS --- proxy/ProxyClientSession.cc | 3 +-- proxy/http/HttpSM.cc | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/proxy/ProxyClientSession.cc b/proxy/ProxyClientSession.cc index 0c71d12be10..23077797564 100644 --- a/proxy/ProxyClientSession.cc +++ b/proxy/ProxyClientSession.cc @@ -202,8 +202,7 @@ ProxyClientSession::handle_api_return(int event) break; } default: - Error("received invalid session hook %s (%d)", HttpDebugNames::get_api_hook_name(hookid), hookid); - ink_release_assert(false); + Fatal("received invalid session hook %s (%d)", HttpDebugNames::get_api_hook_name(hookid), hookid); break; } } diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 435da94f07d..b372dd334b5 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -3241,8 +3241,7 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) c->write_success = true; t_state.client_info.abort = HttpTransact::DIDNOT_ABORT; if (t_state.client_info.keep_alive == HTTP_KEEPALIVE) { - if (ua_txn->allow_half_open() && - (t_state.www_auth_content != HttpTransact::CACHE_AUTH_SERVE || ua_txn->get_server_session())) { + if (t_state.www_auth_content != HttpTransact::CACHE_AUTH_SERVE || ua_txn->get_server_session()) { // successful keep-alive close_connection = false; } @@ -3303,8 +3302,8 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) is_eligible_post_request &= !vc->get_is_internal_request(); } } - if ((is_eligible_post_request || t_state.client_info.pipeline_possible == true) && ua_txn->allow_half_open() && - c->producer->vc_type != HT_STATIC && event == VC_EVENT_WRITE_COMPLETE) { + if ((is_eligible_post_request || t_state.client_info.pipeline_possible == true) && c->producer->vc_type != HT_STATIC && + event == VC_EVENT_WRITE_COMPLETE) { ua_txn->set_half_close_flag(true); } From 10b3b7d292283c863682881f2de5502220117a23 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 18 Dec 2018 14:36:22 -0600 Subject: [PATCH 097/526] BWF: Fix handling of null char * - avoid string_view constructor. --- include/tscore/BufferWriter.h | 27 ++++++++++++++++--- .../unit_tests/test_BufferWriterFormat.cc | 8 ++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/include/tscore/BufferWriter.h b/include/tscore/BufferWriter.h index 052a4de29cf..6da3e12c21b 100644 --- a/include/tscore/BufferWriter.h +++ b/include/tscore/BufferWriter.h @@ -704,12 +704,26 @@ BufferWriter::printv(BWFormat const &fmt, std::tuple const &args) return *this; } +// Must be first so that other inline formatters can use it. +BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, std::string_view sv); + // Pointers that are not specialized. inline BufferWriter & bwformat(BufferWriter &w, BWFSpec const &spec, const void *ptr) { BWFSpec ptr_spec{spec}; ptr_spec._radix_lead_p = true; + + if (ptr == nullptr) { + if (spec._type == 's' || spec._type == 'S') { + ptr_spec._type = BWFSpec::DEFAULT_TYPE; + ptr_spec._ext = ""_sv; // clear any extension. + return bwformat(w, spec, spec._type == 's' ? "null"_sv : "NULL"_sv); + } else if (spec._type == BWFSpec::DEFAULT_TYPE) { + return w; // print nothing if there is no format character override. + } + } + if (ptr_spec._type == BWFSpec::DEFAULT_TYPE || ptr_spec._type == 'p') { ptr_spec._type = 'x'; // if default or 'p;, switch to lower hex. } else if (ptr_spec._type == 'P') { @@ -723,7 +737,12 @@ BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, MemSpan const &span // -- Common formatters -- -BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, std::string_view sv); +// Capture this explicitly so it doesn't go to any other pointer type. +inline BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, std::nullptr_t) +{ + return bwformat(w, spec, static_cast(nullptr)); +} template BufferWriter & @@ -735,10 +754,12 @@ bwformat(BufferWriter &w, BWFSpec const &spec, const char (&a)[N]) inline BufferWriter & bwformat(BufferWriter &w, BWFSpec const &spec, const char *v) { - if (spec._type == 'x' || spec._type == 'X') { + if (spec._type == 'x' || spec._type == 'X' || spec._type == 'p') { bwformat(w, spec, static_cast(v)); - } else { + } else if (v != nullptr) { bwformat(w, spec, std::string_view(v)); + } else { + bwformat(w, spec, nullptr); } return w; } diff --git a/src/tscore/unit_tests/test_BufferWriterFormat.cc b/src/tscore/unit_tests/test_BufferWriterFormat.cc index e8dee671a08..1e3a70b8cca 100644 --- a/src/tscore/unit_tests/test_BufferWriterFormat.cc +++ b/src/tscore/unit_tests/test_BufferWriterFormat.cc @@ -306,6 +306,14 @@ TEST_CASE("bwstring", "[bwprint][bwstring]") ts::bwprint(out, fmt, std::string_view(), "Leif", "confused"); REQUIRE(out == "Did you know? Leif is confused"); } + + char const *null_string{nullptr}; + ts::bwprint(s, "Null {0:x}.{0}", null_string); + REQUIRE(s == "Null 0x0."); + ts::bwprint(s, "Null {0:X}.{0}", nullptr); + REQUIRE(s == "Null 0X0."); + ts::bwprint(s, "Null {0:p}.{0:P}.{0:s}.{0:S}", null_string); + REQUIRE(s == "Null 0x0.0X0.null.NULL"); } TEST_CASE("BWFormat integral", "[bwprint][bwformat]") From d9732a5a95d81d963c4b26dccc3a3622e78ac0f5 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Tue, 18 Dec 2018 17:18:43 +0800 Subject: [PATCH 098/526] Optimize: Avoid meaningless lock operations --- iocore/eventsystem/ProtectedQueue.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/iocore/eventsystem/ProtectedQueue.cc b/iocore/eventsystem/ProtectedQueue.cc index bb9466ba0bb..c4b5377de89 100644 --- a/iocore/eventsystem/ProtectedQueue.cc +++ b/iocore/eventsystem/ProtectedQueue.cc @@ -114,10 +114,15 @@ ProtectedQueue::dequeue_external() void ProtectedQueue::wait(ink_hrtime timeout) { - ink_mutex_acquire(&lock); + // If there are no external events available, don't do a cond_timedwait. if (INK_ATOMICLIST_EMPTY(al)) { - timespec ts = ink_hrtime_to_timespec(timeout); - ink_cond_timedwait(&might_have_data, &lock, &ts); + ink_mutex_acquire(&lock); + // The "al" may have new events while waiting for the mutex become available. + // We have to recheck the external queue again. + if (INK_ATOMICLIST_EMPTY(al)) { + timespec ts = ink_hrtime_to_timespec(timeout); + ink_cond_timedwait(&might_have_data, &lock, &ts); + } + ink_mutex_release(&lock); } - ink_mutex_release(&lock); } From 35a7de1b9ea0f87ceeae3be5985c687f534eb16d Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Tue, 18 Dec 2018 17:01:36 -0600 Subject: [PATCH 099/526] fix disable freelist options --- include/tscore/Allocator.h | 14 ++++++-------- include/tscore/ink_queue.h | 11 ++++------- src/tscore/ink_queue.cc | 28 ++++++++++------------------ src/tscore/test_freelist.cc | 17 ++++++++--------- 4 files changed, 28 insertions(+), 42 deletions(-) diff --git a/include/tscore/Allocator.h b/include/tscore/Allocator.h index 4ed9094e449..3de986a8c45 100644 --- a/include/tscore/Allocator.h +++ b/include/tscore/Allocator.h @@ -48,8 +48,6 @@ #define RND16(_x) (((_x) + 15) & ~15) -extern int cmd_disable_pfreelist; - /** Allocator for fixed size memory blocks. */ class Allocator { @@ -61,7 +59,7 @@ class Allocator void * alloc_void() { - return ink_freelist_new(this->fl, freelist_class_ops); + return ink_freelist_new(this->fl); } /** @@ -72,7 +70,7 @@ class Allocator void free_void(void *ptr) { - ink_freelist_free(this->fl, ptr, freelist_class_ops); + ink_freelist_free(this->fl, ptr); } /** @@ -85,7 +83,7 @@ class Allocator void free_void_bulk(void *head, void *tail, size_t num_item) { - ink_freelist_free_bulk(this->fl, head, tail, num_item, freelist_class_ops); + ink_freelist_free_bulk(this->fl, head, tail, num_item); } Allocator() { fl = nullptr; } @@ -130,7 +128,7 @@ template class ClassAllocator : public Allocator C * alloc() { - void *ptr = ink_freelist_new(this->fl, freelist_class_ops); + void *ptr = ink_freelist_new(this->fl); memcpy(ptr, (void *)&this->proto.typeObject, sizeof(C)); return (C *)ptr; @@ -144,7 +142,7 @@ template class ClassAllocator : public Allocator void free(C *ptr) { - ink_freelist_free(this->fl, ptr, freelist_class_ops); + ink_freelist_free(this->fl, ptr); } /** @@ -157,7 +155,7 @@ template class ClassAllocator : public Allocator void free_bulk(C *head, C *tail, size_t num_item) { - ink_freelist_free_bulk(this->fl, head, tail, num_item, freelist_class_ops); + ink_freelist_free_bulk(this->fl, head, tail, num_item); } /** diff --git a/include/tscore/ink_queue.h b/include/tscore/ink_queue.h index 6d242cd9f15..bc8a71efac7 100644 --- a/include/tscore/ink_queue.h +++ b/include/tscore/ink_queue.h @@ -156,12 +156,9 @@ struct _InkFreeList { typedef struct ink_freelist_ops InkFreeListOps; typedef struct _InkFreeList InkFreeList; -extern const ink_freelist_ops *freelist_global_ops; -extern const ink_freelist_ops *freelist_class_ops; - const InkFreeListOps *ink_freelist_malloc_ops(); const InkFreeListOps *ink_freelist_freelist_ops(); -void ink_freelist_init_ops(int nofl_global, int nofl_class); +void ink_freelist_init_ops(int nofl_class, int nofl_proxy); /* * alignment must be a power of 2 @@ -171,9 +168,9 @@ InkFreeList *ink_freelist_create(const char *name, uint32_t type_size, uint32_t inkcoreapi void ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment); inkcoreapi void ink_freelist_madvise_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment, int advice); -inkcoreapi void *ink_freelist_new(InkFreeList *f, const InkFreeListOps *ops); -inkcoreapi void ink_freelist_free(InkFreeList *f, void *item, const InkFreeListOps *ops); -inkcoreapi void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item, const InkFreeListOps *ops); +inkcoreapi void *ink_freelist_new(InkFreeList *f); +inkcoreapi void ink_freelist_free(InkFreeList *f, void *item); +inkcoreapi void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item); void ink_freelists_dump(FILE *f); void ink_freelists_dump_baselinerel(FILE *f); void ink_freelists_snap_baseline(); diff --git a/src/tscore/ink_queue.cc b/src/tscore/ink_queue.cc index f6e8787f177..0a6ef05d1bf 100644 --- a/src/tscore/ink_queue.cc +++ b/src/tscore/ink_queue.cc @@ -91,10 +91,8 @@ static const ink_freelist_ops malloc_ops = {malloc_new, malloc_free, malloc_bu static const ink_freelist_ops freelist_ops = {freelist_new, freelist_free, freelist_bulkfree}; static const ink_freelist_ops *default_ops = &freelist_ops; -const ink_freelist_ops *freelist_global_ops = default_ops; -const ink_freelist_ops *freelist_class_ops = default_ops; - -static ink_freelist_list *freelists = nullptr; +static ink_freelist_list *freelists = nullptr; +static const ink_freelist_ops *freelist_global_ops = default_ops; const InkFreeListOps * ink_freelist_malloc_ops() @@ -109,15 +107,13 @@ ink_freelist_freelist_ops() } void -ink_freelist_init_ops(int nofl_global, int nofl_class) +ink_freelist_init_ops(int nofl_class, int nofl_proxy) { // This *MUST* only be called at startup before any freelists allocate anything. We will certainly crash if object // allocated from the freelist are freed by malloc. ink_release_assert(freelist_global_ops == default_ops); - ink_release_assert(freelist_class_ops == default_ops); - freelist_global_ops = nofl_global ? ink_freelist_malloc_ops() : ink_freelist_freelist_ops(); - freelist_class_ops = nofl_class ? ink_freelist_malloc_ops() : ink_freelist_freelist_ops(); + freelist_global_ops = (nofl_class || nofl_proxy) ? ink_freelist_malloc_ops() : ink_freelist_freelist_ops(); } void @@ -184,12 +180,11 @@ int fake_global_for_ink_queue = 0; #endif void * -ink_freelist_new(InkFreeList *f, const InkFreeListOps *ops) +ink_freelist_new(InkFreeList *f) { - ink_assert(ops != nullptr); void *ptr; - if (likely(ptr = ops->fl_new(f))) { + if (likely(ptr = freelist_global_ops->fl_new(f))) { ink_atomic_increment((int *)&f->used, 1); } @@ -275,13 +270,11 @@ malloc_new(InkFreeList *f) } void -ink_freelist_free(InkFreeList *f, void *item, const InkFreeListOps *ops) +ink_freelist_free(InkFreeList *f, void *item) { - ink_assert(ops != nullptr); - if (likely(item != nullptr)) { ink_assert(f->used != 0); - ops->fl_free(f, item); + freelist_global_ops->fl_free(f, item); ink_atomic_decrement((int *)&f->used, 1); } } @@ -334,12 +327,11 @@ malloc_free(InkFreeList *f, void *item) } void -ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item, const InkFreeListOps *ops) +ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item) { - ink_assert(ops != nullptr); ink_assert(f->used >= num_item); - ops->fl_bulkfree(f, head, tail, num_item); + freelist_global_ops->fl_bulkfree(f, head, tail, num_item); ink_atomic_decrement((int *)&f->used, num_item); } diff --git a/src/tscore/test_freelist.cc b/src/tscore/test_freelist.cc index 159951017ea..3fdc27ab20d 100644 --- a/src/tscore/test_freelist.cc +++ b/src/tscore/test_freelist.cc @@ -37,13 +37,12 @@ test(void *d) id = (intptr_t)d; - time_t start = time(nullptr); - int count = 0; - const InkFreeListOps *ops = ink_freelist_freelist_ops(); + time_t start = time(nullptr); + int count = 0; for (;;) { - m1 = ink_freelist_new(flist, ops); - m2 = ink_freelist_new(flist, ops); - m3 = ink_freelist_new(flist, ops); + m1 = ink_freelist_new(flist); + m2 = ink_freelist_new(flist); + m3 = ink_freelist_new(flist); if ((m1 == m2) || (m1 == m3) || (m2 == m3)) { printf("0x%08" PRIx64 " 0x%08" PRIx64 " 0x%08" PRIx64 "\n", (uint64_t)(uintptr_t)m1, (uint64_t)(uintptr_t)m2, @@ -55,9 +54,9 @@ test(void *d) memset(m2, id, 64); memset(m3, id, 64); - ink_freelist_free(flist, m1, ops); - ink_freelist_free(flist, m2, ops); - ink_freelist_free(flist, m3, ops); + ink_freelist_free(flist, m1); + ink_freelist_free(flist, m2); + ink_freelist_free(flist, m3); // break out of the test if we have run more then 60 seconds if (++count % 1000 == 0 && (start + 60) < time(nullptr)) { From 8ee588238e7cc0bdebbb8e4528c95f282e3ad03f Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 18 Dec 2018 17:27:56 +0000 Subject: [PATCH 100/526] Add TLS client keep alive test. --- tests/gold_tests/tls/gold/accesslog.gold | 8 ++ tests/gold_tests/tls/tls_keepalive.test.py | 106 +++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 tests/gold_tests/tls/gold/accesslog.gold create mode 100644 tests/gold_tests/tls/tls_keepalive.test.py diff --git a/tests/gold_tests/tls/gold/accesslog.gold b/tests/gold_tests/tls/gold/accesslog.gold new file mode 100644 index 00000000000..d00fc14a51a --- /dev/null +++ b/tests/gold_tests/tls/gold/accesslog.gold @@ -0,0 +1,8 @@ +1 0 +1 1 +1 0 +1 0 +1 0 +1 1 +1 0 +1 0 diff --git a/tests/gold_tests/tls/tls_keepalive.test.py b/tests/gold_tests/tls/tls_keepalive.test.py new file mode 100644 index 00000000000..518f2bfae24 --- /dev/null +++ b/tests/gold_tests/tls/tls_keepalive.test.py @@ -0,0 +1,106 @@ +''' +Use pre-accept hook to verify that both requests are made over the same TLS session +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Verify that the client-side keep alive is honored for TLS and different versions of HTTP +''' + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_hook_test|log', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.TLSv1_3': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.log.max_secs_per_buffer': 1 +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.logging_yaml.AddLines([ + 'formats:', + '- name: testformat', + " format: '% %'", + "logs:", + "- mode: ascii", + " format: testformat", + " filename: squid" ]) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-preaccept=1') + +tr = Test.AddTestRun("Test two HTTP/1.1 requests over one TLS connection") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0} https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun("Test two HTTP/1.1 requests over two TLS connections") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}; curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun("Test two HTTP/2 requests over one TLS connection") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -v --http2 -H \'host:example.com:{0}\' https://127.0.0.1:{0} https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun("Test two HTTP/2 requests over two TLS connections") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -v --http2 -H \'host:example.com:{0}\' https://127.0.0.1:{0}; curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +# Just a check to flush out the traffic log until we have a clean shutdown for traffic_server +tr = Test.AddTestRun("Wait for the access log to write out") +tr.DelayStart = 5 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'ls' +tr.Processes.Default.ReturnCode = 0 + +ts.Disk.squid_log.Content = "gold/accesslog.gold" + +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 From 92ab7ef135e890380eb270fcc20ee27c57061e93 Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Mon, 17 Dec 2018 16:41:53 -0600 Subject: [PATCH 101/526] task threads ready lifecycle hook --- .../api/functions/TSLifecycleHookAdd.en.rst | 7 +++++++ include/ts/apidefs.h.in | 10 +++++++++- iocore/eventsystem/I_EventProcessor.h | 1 + iocore/eventsystem/I_Tasks.h | 1 + iocore/eventsystem/Tasks.cc | 9 ++++++++- iocore/eventsystem/UnixEventProcessor.cc | 3 +++ proxy/http/HttpDebugNames.cc | 2 ++ src/traffic_server/traffic_server.cc | 13 +++++++++++++ 8 files changed, 44 insertions(+), 2 deletions(-) diff --git a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst index efdcee1f226..bac552b2218 100644 --- a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst @@ -98,6 +98,13 @@ Types Called after every SSL context initialization used by |TS| for inbound connections (|TS| as the server). + .. cpp:enumerator:: TS_LIFECYCLE_TASK_THREADS_READY_HOOK + + Called after |TS| task threads have been started. + + Invoked with the event :c:data:`TS_EVENT_LIFECYCLE_TASK_THREADS_READY` and ``NULL`` + data. + .. c:type:: TSPluginMsg The format of the data for the plugin message event :c:data:`TS_EVENT_LIFECYCLE_MSG`. diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 6c042653d40..8b749690960 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -324,7 +324,7 @@ typedef enum { accept connections. This is *not* guaranteed to be called before the first connection is accepted. - Event: TS_EVENT_LIFECYCLE_PORTS_READY_HOOK + Event: TS_EVENT_LIFECYCLE_PORTS_READY TS_LIFECYCLE_CACHE_READY_HOOK @@ -354,6 +354,12 @@ typedef enum { Event: TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED + TS_LIFECYCLE_TASK_THREADS_READY_HOOK + + called once, after the task threads have been started. + + Event: TS_EVENT_LIFECYCLE_TASK_THREADS_READY + Ordering guarantees: - TS_LIFECYCLE_PORTS_INITIALIZED_HOOK before TS_LIFECYCLE_PORTS_READY_HOOK. @@ -368,6 +374,7 @@ typedef enum { TS_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED_HOOK, TS_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED_HOOK, TS_LIFECYCLE_MSG_HOOK, + TS_LIFECYCLE_TASK_THREADS_READY_HOOK, TS_LIFECYCLE_LAST_HOOK } TSLifecycleHookID; @@ -453,6 +460,7 @@ typedef enum { TS_EVENT_LIFECYCLE_CACHE_READY = 60020, TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60021, TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60022, + TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60027, TS_EVENT_VCONN_START = 60023, TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible TS_EVENT_VCONN_CLOSE = 60026, diff --git a/iocore/eventsystem/I_EventProcessor.h b/iocore/eventsystem/I_EventProcessor.h index 28dbd012ba9..3730b4915cf 100644 --- a/iocore/eventsystem/I_EventProcessor.h +++ b/iocore/eventsystem/I_EventProcessor.h @@ -308,6 +308,7 @@ class EventProcessor : public Processor int _next_round_robin = 0; ///< Index of thread to use for events assigned to this group. Que(Event, link) _spawnQueue; ///< Events to dispatch when thread is spawned. EThread *_thread[MAX_THREADS_IN_EACH_TYPE] = {}; ///< The actual threads in this group. + std::function _afterStartCallback = nullptr; }; /// Storage for per group data. diff --git a/iocore/eventsystem/I_Tasks.h b/iocore/eventsystem/I_Tasks.h index 0484f366bea..160ffe8f87a 100644 --- a/iocore/eventsystem/I_Tasks.h +++ b/iocore/eventsystem/I_Tasks.h @@ -31,6 +31,7 @@ extern EventType ET_TASK; class TasksProcessor : public Processor { public: + EventType register_event_type(); int start(int task_threads, size_t stacksize = DEFAULT_STACKSIZE) override; }; diff --git a/iocore/eventsystem/Tasks.cc b/iocore/eventsystem/Tasks.cc index 91a5942c2db..7de34e2747d 100644 --- a/iocore/eventsystem/Tasks.cc +++ b/iocore/eventsystem/Tasks.cc @@ -27,11 +27,18 @@ EventType ET_TASK = ET_CALL; TasksProcessor tasksProcessor; +EventType +TasksProcessor::register_event_type() +{ + ET_TASK = eventProcessor.register_event_type("ET_TASK"); + return ET_TASK; +} + // Note that if the number of task_threads is 0, all continuations scheduled for // ET_TASK ends up running on ET_CALL (which is the net-threads). int TasksProcessor::start(int task_threads, size_t stacksize) { - ET_TASK = eventProcessor.spawn_event_threads("ET_TASK", std::max(1, task_threads), stacksize); + eventProcessor.spawn_event_threads(ET_TASK, std::max(1, task_threads), stacksize); return 0; } diff --git a/iocore/eventsystem/UnixEventProcessor.cc b/iocore/eventsystem/UnixEventProcessor.cc index 32b9ed63c44..f2bcf4ee9fc 100644 --- a/iocore/eventsystem/UnixEventProcessor.cc +++ b/iocore/eventsystem/UnixEventProcessor.cc @@ -396,6 +396,9 @@ EventProcessor::initThreadState(EThread *t) for (int i = 0; i < MAX_EVENT_TYPES; ++i) { if (t->is_event_type(i)) { // that event type done here, roll thread start events of that type. ++thread_group[i]._started; + if (thread_group[i]._started == thread_group[i]._count && thread_group[i]._afterStartCallback != nullptr) { + thread_group[i]._afterStartCallback(); + } // To avoid race conditions on the event in the spawn queue, create a local one to actually send. // Use the spawn queue event as a read only model. Event *nev = eventAllocator.alloc(); diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index 59beb0157c9..576e75a4876 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -345,6 +345,8 @@ HttpDebugNames::get_event_name(int event) return "TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED"; case TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED: return "TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED"; + case TS_EVENT_LIFECYCLE_TASK_THREADS_READY: + return "TS_EVENT_LIFECYCLE_TASK_THREADS_READY"; case TS_EVENT_VCONN_START: return "TS_EVENT_VCONN_START"; case TS_EVENT_VCONN_CLOSE: diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index a69aeec49b9..2d5f3373389 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -121,6 +121,7 @@ static void *mgmt_lifecycle_msg_callback(void *x, char *data, int len); static void init_ssl_ctx_callback(void *ctx, bool server); static void load_ssl_file_callback(const char *ssl_file, unsigned int options); static void load_remap_file_callback(const char *remap_file); +static void task_threads_started_callback(); // We need these two to be accessible somewhere else now int num_of_net_threads = ink_number_of_processors(); @@ -1957,6 +1958,8 @@ main(int /* argc ATS_UNUSED */, const char **argv) RecConfigWarnIfUnregistered(); // "Task" processor, possibly with its own set of task threads + tasksProcessor.register_event_type(); + eventProcessor.thread_group[ET_TASK]._afterStartCallback = task_threads_started_callback; tasksProcessor.start(num_task_threads, stacksize); if (netProcessor.socks_conf_stuff->accept_enabled) { @@ -2100,3 +2103,13 @@ load_remap_file_callback(const char *remap_file) { pmgmt->signalConfigFileChild("remap.config", remap_file, CONFIG_FLAG_UNVERSIONED); } + +static void +task_threads_started_callback() +{ + APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_TASK_THREADS_READY_HOOK); + while (hook) { + hook->invoke(TS_EVENT_LIFECYCLE_TASK_THREADS_READY, nullptr); + hook = hook->next(); + } +} From a9a10247c4979753d9b9f9f4a69d2cc92d1f796c Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Thu, 20 Dec 2018 13:27:18 +0900 Subject: [PATCH 102/526] Remove unused Http2Stream destructor --- proxy/http2/Http2Stream.h | 1 - 1 file changed, 1 deletion(-) diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index bc74c72bad9..8286acc1f75 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -59,7 +59,6 @@ class Http2Stream : public ProxyClientTransaction response_header.create(HTTP_TYPE_RESPONSE); } - ~Http2Stream() { this->destroy(); } int main_event_handler(int event, void *edata); void destroy() override; From b9a9fc4c5730e5af5e36db5a1a43f30bde51ab26 Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Wed, 19 Dec 2018 15:14:51 -0600 Subject: [PATCH 103/526] shutdown hook --- .../api/functions/TSLifecycleHookAdd.en.rst | 6 ++++++ include/ts/apidefs.h.in | 8 ++++++++ proxy/http/HttpDebugNames.cc | 2 ++ src/traffic_server/traffic_server.cc | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst index bac552b2218..82b22d46cb6 100644 --- a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst @@ -105,6 +105,12 @@ Types Invoked with the event :c:data:`TS_EVENT_LIFECYCLE_TASK_THREADS_READY` and ``NULL`` data. + .. cpp:enumerator:: TS_LIFECYCLE_SHUTDOWN_HOOK + + Called after |TS| receiving a shutdown signal, such as SIGTERM. + + Invoked with the event :c:data:`TS_EVENT_LIFECYCLE_SHUTDOWN` and ``NULL`` data. + .. c:type:: TSPluginMsg The format of the data for the plugin message event :c:data:`TS_EVENT_LIFECYCLE_MSG`. diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 8b749690960..ad5be0bc541 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -360,6 +360,12 @@ typedef enum { Event: TS_EVENT_LIFECYCLE_TASK_THREADS_READY + TS_LIFECYCLE_SHUTDOWN_HOOK + + called once, after receiving a shutdown signal, such as SIGTERM. + + Event: TS_EVENT_LIFECYCLE_SHUTDOWN + Ordering guarantees: - TS_LIFECYCLE_PORTS_INITIALIZED_HOOK before TS_LIFECYCLE_PORTS_READY_HOOK. @@ -375,6 +381,7 @@ typedef enum { TS_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED_HOOK, TS_LIFECYCLE_MSG_HOOK, TS_LIFECYCLE_TASK_THREADS_READY_HOOK, + TS_LIFECYCLE_SHUTDOWN_HOOK, TS_LIFECYCLE_LAST_HOOK } TSLifecycleHookID; @@ -461,6 +468,7 @@ typedef enum { TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60021, TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60022, TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60027, + TS_EVENT_LIFECYCLE_SHUTDOWN = 60028, TS_EVENT_VCONN_START = 60023, TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible TS_EVENT_VCONN_CLOSE = 60026, diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index 576e75a4876..c76ef0e8973 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -347,6 +347,8 @@ HttpDebugNames::get_event_name(int event) return "TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED"; case TS_EVENT_LIFECYCLE_TASK_THREADS_READY: return "TS_EVENT_LIFECYCLE_TASK_THREADS_READY"; + case TS_EVENT_LIFECYCLE_SHUTDOWN: + return "TS_EVENT_LIFECYCLE_SHUTDOWN"; case TS_EVENT_VCONN_START: return "TS_EVENT_VCONN_START"; case TS_EVENT_VCONN_CLOSE: diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 2d5f3373389..b2753c83331 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -219,6 +219,12 @@ struct AutoStopCont : public Continuation { int mainEvent(int /* event */, Event * /* e */) { + APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_SHUTDOWN_HOOK); + while (hook) { + hook->invoke(TS_EVENT_LIFECYCLE_SHUTDOWN, nullptr); + hook = hook->next(); + } + pmgmt->stop(); shutdown_event_system = true; delete this; From edccb98c97f15abce60ddb795e8429ce940c1414 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 19 Dec 2018 20:25:00 +0000 Subject: [PATCH 104/526] Fix documentation on client certificates in ssl_server_name --- doc/admin-guide/files/ssl_server_name.yaml.en.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/admin-guide/files/ssl_server_name.yaml.en.rst b/doc/admin-guide/files/ssl_server_name.yaml.en.rst index 66fdd558e8f..e95c3dd2d33 100644 --- a/doc/admin-guide/files/ssl_server_name.yaml.en.rst +++ b/doc/admin-guide/files/ssl_server_name.yaml.en.rst @@ -66,20 +66,25 @@ verify_origin_server Deprecated. Use verify_server_policy and verify_serve By default this is :ts:cv:`proxy.config.ssl.client.verify.server`. verify_client One of the values :code:`NONE`, :code:`MODERATE`, or :code:`STRICT`. + If ``NONE`` is specified, |TS| requests no certificate. If ``MODERATE`` is specified + |TS| will verify a certificate that is presented by the client, but it will not + fail the TLS handshake if new certificate is presented. If ``STRICT`` is specified + the client must resent a certificate during the TLS handshake. + By default this is :ts:cv:`proxy.config.ssl.client.certification_level`. client_cert The file containing the client certificate to use for the outbound connection. - If this is relative it is relative to the path in - :ts:cv:`proxy.config.ssl.server.cert.path`. If not set + If this is relative, it is relative to the path in + :ts:cv:`proxy.config.ssl.client.cert.path`. If not set :ts:cv:`proxy.config.ssl.client.cert.filename` is used. client_key The file containing the client private key that corresponds to the certificate for the outbound connection. - If this is relative it is relative to the path in - :ts:cv:`proxy.config.ssl.server.private_key.path`. If not set, + If this is relative, it is relative to the path in + :ts:cv:`proxy.config.ssl.client.private_key.path`. If not set, |TS| tries to use a private key in client_cert. Otherwise, :ts:cv:`proxy.config.ssl.client.private_key.filename` is used. From 149195e7fff0bd195a6e72de96c15354c7fcfd4e Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 19 Dec 2018 19:04:04 +0000 Subject: [PATCH 105/526] Add control for how outbound SNI is selected. --- doc/admin-guide/files/records.config.en.rst | 7 ++ .../functions/TSHttpOverridableConfig.en.rst | 5 +- include/ts/apidefs.h.in | 1 + mgmt/RecordsConfig.cc | 2 + plugins/lua/ts_lua_http_config.c | 2 + proxy/http/HttpConfig.h | 2 + proxy/http/HttpSM.cc | 13 ++-- src/traffic_server/InkAPI.cc | 10 ++- src/traffic_server/InkAPITest.cc | 3 +- .../tls/tls_verify_override.test.py | 72 ++++++++++++++++--- 10 files changed, 98 insertions(+), 19 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 249bd25c439..a62235da65d 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3541,6 +3541,13 @@ Client-Related Configuration Specifies the location of the certificate authority file against which the origin server will be verified. +.. ts:cv:: CONFIG proxy.config.ssl.client.sni_policy STRING NULL + :overridable: + + Indicate how the SNI value for the TLS connection to the origin is selected. By default it is + `host` which means the host header field value is used for the SNI. If `remap` is specified, the + remapped origin name is used for the SNI value. + .. ts:cv:: CONFIG proxy.config.ssl.client.SSLv3 INT 0 Enables (``1``) or disables (``0``) SSLv3 in the ATS client context. Disabled by default diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index 003ee33f10d..941686186c0 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -175,7 +175,10 @@ TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED proxy.config :c:macro:`TS_CONFIG_SRV_ENABLED` :ts:cv:`proxy.config.srv_enabled` :c:macro:`TS_CONFIG_SSL_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` :c:macro:`TS_CONFIG_SSL_CERT_FILEPATH` :ts:cv:`proxy.config.ssl.client.cert.path` -TS_CONFIG_SSL_CLIENT_VERIFY_SERVER :ts:cv:`proxy.config.ssl.client.verify.server` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER` :ts:cv:`proxy.config.ssl.client.verify.server` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES` :ts:cv:`proxy.config.ssl.client.verify.server,properties` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY` :ts:cv:`proxy.config.ssl.client.verify.server.policy` +:c:macro:`TS_CONFIG_SSL_CLIENT_SNI_POLICY` :ts:cv:`proxy.config.ssl.client.sni_policy` :c:macro:`TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS` :ts:cv:`proxy.config.ssl.hsts_include_subdomains` :c:macro:`TS_CONFIG_SSL_HSTS_MAX_AGE` :ts:cv:`proxy.config.ssl.hsts_max_age` :c:macro:`TS_CONFIG_URL_REMAP_PRISTINE_HOST_HDR` :ts:cv:`proxy.config.url_remap.pristine_host_hdr` diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index ad5be0bc541..76189d1eb6b 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -801,6 +801,7 @@ typedef enum { TS_CONFIG_SSL_CLIENT_VERIFY_SERVER, TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, + TS_CONFIG_SSL_CLIENT_SNI_POLICY, TS_CONFIG_LAST_ENTRY } TSOverridableConfigKey; diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 931c6e3d009..e9cb40fda08 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1154,6 +1154,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.client.sni_policy", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , {RECT_CONFIG, "proxy.config.ssl.session_cache", RECD_INT, "2", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.session_cache.size", RECD_INT, "102400", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c index b7c63ea9219..74bc20169f3 100644 --- a/plugins/lua/ts_lua_http_config.c +++ b/plugins/lua/ts_lua_http_config.c @@ -137,6 +137,7 @@ typedef enum { TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER, TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, + TS_LUA_CONFIG_SSL_CLIENT_SNI_POLICY = TS_CONFIG_SSL_CLIENT_SNI_POLICY, TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY, } TSLuaOverridableConfigKey; @@ -264,6 +265,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_SNI_POLICY), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY), diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index 41d5e819324..f274168cc7f 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -494,6 +494,7 @@ struct OverridableHttpConfigParams { ssl_client_verify_server(0), ssl_client_verify_server_policy(nullptr), ssl_client_verify_server_properties(nullptr), + ssl_client_sni_policy(nullptr), redirect_use_orig_cache_key(0), number_of_redirections(0), proxy_response_hsts_max_age(-1), @@ -681,6 +682,7 @@ struct OverridableHttpConfigParams { MgmtByte ssl_client_verify_server; char *ssl_client_verify_server_policy; char *ssl_client_verify_server_properties; + char *ssl_client_sni_policy; ////////////////// // Redirection // diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index b372dd334b5..75095d9d698 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -5048,10 +5048,15 @@ HttpSM::do_http_server_open(bool raw) if (scheme_to_use == URL_WKSIDX_HTTPS) { SMDebug("http", "calling sslNetProcessor.connect_re"); - int len = 0; - const char *host = t_state.hdr_info.server_request.host_get(&len); - if (host && len > 0) { - opt.set_sni_servername(host, len); + int len = 0; + if (t_state.txn_conf->ssl_client_sni_policy != nullptr && !strcmp(t_state.txn_conf->ssl_client_sni_policy, "remap")) { + len = strlen(t_state.server_info.name); + opt.set_sni_servername(t_state.server_info.name, len); + } else { // Do the default of host header for SNI + const char *host = t_state.hdr_info.server_request.host_get(&len); + if (host && len > 0) { + opt.set_sni_servername(host, len); + } } if (t_state.server_info.name) { opt.set_ssl_servername(t_state.server_info.name); diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 7b17eb05ea9..c537e166292 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -8209,6 +8209,7 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr break; case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY: case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES: + case TS_CONFIG_SSL_CLIENT_SNI_POLICY: // String, must be handled elsewhere break; case TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB: @@ -8425,6 +8426,11 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char s->t_state.txn_conf->ssl_client_verify_server_properties = const_cast(value); } break; + case TS_CONFIG_SSL_CLIENT_SNI_POLICY: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_sni_policy = const_cast(value); + } + break; default: { MgmtConverter const *conv; void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); @@ -8614,8 +8620,8 @@ static const std::unordered_map SDK_Overridable_Configs = { OutboundConnTrack::CONFIG_VAR_MATCH, "proxy.config.ssl.client.verify.server", "proxy.config.ssl.client.verify.server.policy", - "proxy.config.ssl.client.verify.server.properties"}}; + "proxy.config.ssl.client.verify.server.properties", + "proxy.config.ssl.client.sni_policy"}}; REGRESSION_TEST(SDK_API_OVERRIDABLE_CONFIGS)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus) { diff --git a/tests/gold_tests/tls/tls_verify_override.test.py b/tests/gold_tests/tls/tls_verify_override.test.py index 6afbfd52063..fd568194ed9 100644 --- a/tests/gold_tests/tls/tls_verify_override.test.py +++ b/tests/gold_tests/tls/tls_verify_override.test.py @@ -32,6 +32,8 @@ server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) +dns = Test.MakeDNServer("dns") + request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -54,23 +56,31 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map http://foo.com/basic https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.Port)) ts.Disk.remap_config.AddLine( - 'map http://foo.com/override https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/basic https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridedisabled https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridesignature https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overrideenforced https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) ts.Disk.remap_config.AddLine( 'map /basic https://127.0.0.1:{0}'.format(server.Variables.Port)) ts.Disk.remap_config.AddLine( 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.Port)) ts.Disk.remap_config.AddLine( 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' @@ -91,9 +101,14 @@ 'proxy.config.ssl.client.verify.server.properties': 'ALL', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', - 'proxy.config.url_remap.pristine_host_hdr': 1 + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL' }) +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + # Should succeed without message tr = Test.AddTestRun("default-permissive-success") tr.Setup.Copy("ssl/signed-foo.key") @@ -102,6 +117,7 @@ tr.Setup.Copy("ssl/signed-bar.pem") tr.Processes.Default.Command = 'curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/basic'.format(ts.Variables.port) tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(dns) tr.Processes.Default.StartBefore(server_foo) tr.Processes.Default.StartBefore(server_bar) tr.Processes.Default.StartBefore(server) @@ -168,18 +184,52 @@ tr6.StillRunningAfter = ts tr6.Processes.Default.TimeOut = 5 +# Should succeed +tr = Test.AddTestRun("foo-to-bar-sni-policy-remap") +tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/snipolicybarremap".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") + +# Should fail +tr = Test.AddTestRun("foo-to-bar-sni-policy-host") +tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/snipolicybarhost".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") + +# Should fail +tr = Test.AddTestRun("bar-to-foo-sni-policy-remap") +tr.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/snipolicyfooremap".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") + +# Should succeed +tr = Test.AddTestRun("bar-to-foo-sni-policy-host") +tr.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/snipolicyfoohost".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") + # Over riding the built in ERROR check since we expect some cases to fail # checks on random.com should fail with message only ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=127.0.0.1\(127.0.0.1\) depth=0", "Warning for self signed certificate") -# No complaints about foo -ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI \(foo.com\) not in certificate", "foo.com name requests are good") # permissive failure for bar.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", "Warning on missing name for bar.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", "Warning on missing name for bar.com") # name check failure for random.com ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", "Warning on missing name for randome.com") # name check failure for bar.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=127.0.0.1\(127.0.0.1\)", "Failure on missing name for bar.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", "Failure on missing name for bar.com") From 21ba72547fa76f3311da7e202c528d9c04373cb8 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 18 Dec 2018 21:02:15 +0000 Subject: [PATCH 106/526] Add config reload to tls_tunnel test. --- tests/gold_tests/tls/tls_tunnel.test.py | 61 +++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index 2c61a788bc1..30b976072e0 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -27,7 +27,7 @@ ) # Define default ATS -ts = Test.MakeATSProcess("ts", select_ports=False) +ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=False) server_foo = Test.MakeOriginServer("server_foo", ssl=True) server_bar = Test.MakeOriginServer("server_bar", ssl=True) @@ -86,11 +86,11 @@ ]) tr = Test.AddTestRun("foo.com Tunnel-test") -tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) -tr.ReturnCode = 0 tr.Processes.Default.StartBefore(server_foo) tr.Processes.Default.StartBefore(server_bar) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 tr.StillRunningAfter = ts tr.Processes.Default.TimeOut = 5 tr.TimeOut = 5 @@ -99,7 +99,7 @@ tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") -tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from bar") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from foo") tr = Test.AddTestRun("bob.bar.com Tunnel-test") tr.Processes.Default.Command = "curl -v --resolve 'bob.bar.com:{0}:127.0.0.1' -k https://bob.bar.com:{0}".format(ts.Variables.ssl_port) @@ -112,7 +112,7 @@ tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") -tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from bar") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from foo") tr = Test.AddTestRun("bar.com no Tunnel-test") tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) @@ -136,6 +136,57 @@ tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("bar ok", "Should get a response from bar") +# Update ssl_server_name file and reload +tr = Test.AddTestRun("Update config files") +# Update the SNI config +snipath = ts.Disk.ssl_server_name_yaml.AbsPath +recordspath = ts.Disk.records_config.AbsPath +tr.Disk.File(snipath, id = "ssl_server_name_yaml", typename="ats:config"), +tr.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + " tunnel_route: localhost:{0}".format(server_bar.Variables.Port), +]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server_foo +tr.StillRunningAfter = server_bar +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = 'echo Updated configs' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +trreload = Test.AddTestRun("Reload config") +trreload.StillRunningAfter = ts +trreload.StillRunningAfter = server_foo +trreload.StillRunningAfter = server_bar +trreload.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +trreload.Processes.Default.Env = ts.Env +trreload.Processes.Default.ReturnCode = 0 +trreload.Processes.Default.TimeOut = 5 +trreload.TimeOut = 5 + +# Should termimate on traffic_server (not tunnel) +tr = Test.AddTestRun("foo.com no Tunnel-test") +tr.StillRunningAfter = ts +# Wait for the reload to complete +tr.DelayStart = 5 +tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +# Should tunnel to server_bar +tr = Test.AddTestRun("bar.com Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Terminates on on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("bar ok", "Should get a response from bar") From 786a38beba6a48aebcbc564d6ec05c02da5cb28c Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 14 Nov 2018 16:43:41 -0600 Subject: [PATCH 107/526] BWF: Clean up diags log entry header construction. --- src/tscore/Diags.cc | 74 +++++++++------------------------------------ 1 file changed, 15 insertions(+), 59 deletions(-) diff --git a/src/tscore/Diags.cc b/src/tscore/Diags.cc index 179be9ec8af..b34db02566c 100644 --- a/src/tscore/Diags.cc +++ b/src/tscore/Diags.cc @@ -34,6 +34,8 @@ ****************************************************************************/ +#include "tscore/BufferWriter.h" +#include "tscore/bwf_std_format.h" #include "tscore/ink_platform.h" #include "tscore/ink_memory.h" #include "tscore/ink_defs.h" @@ -215,75 +217,29 @@ Diags::print_va(const char *debug_tag, DiagsLevel diags_level, const SourceLocat va_list ap) const { ink_release_assert(diags_level < DiagsLevel_Count); - - using ts::LocalBufferWriter; - LocalBufferWriter<1024> format_writer; + ts::LocalBufferWriter<1024> format_writer; // Save room for optional newline and terminating NUL bytes. format_writer.clip(2); - ////////////////////// - // append timestamp // - ////////////////////// - { - struct timeval tp = ink_gettimeofday(); - time_t cur_clock = (time_t)tp.tv_sec; - char timestamp_buf[48]; - char *buffer = ink_ctime_r(&cur_clock, timestamp_buf); - - int num_bytes_written = snprintf(&(timestamp_buf[19]), (sizeof(timestamp_buf) - 20), ".%03d", (int)(tp.tv_usec / 1000)); - - if (num_bytes_written > 0) { - format_writer.write('['); - format_writer.write(buffer + 4, strlen(buffer + 4)); - format_writer.write("] ", 2); - } - } - - size_t timestamp_end_offset = format_writer.size(); + format_writer.print("[{timestamp}] "); + auto timestamp_offset = format_writer.size(); - /////////////////////// - // add the thread name // - /////////////////////// - format_writer.print("{thread-name} "); - - ////////////////////////////////// - // append the diag level prefix // - ////////////////////////////////// - - format_writer.write(level_name(diags_level), strlen(level_name(diags_level))); - format_writer.write(": ", 2); - - ///////////////////////////// - // append location, if any // - ///////////////////////////// + format_writer.print("{thread-name}"); + format_writer.print(" {}: ", level_name(diags_level)); if (location(loc, show_location, diags_level)) { - char *lp, buf[256]; - lp = loc->str(buf, sizeof(buf)); - if (lp) { - format_writer.write('<'); - format_writer.write(lp, std::min(strlen(lp), sizeof(buf))); - format_writer.write("> ", 2); - } + format_writer.print("<{}> ", *loc); } - ////////////////////////// - // append debugging tag // - ////////////////////////// - if (debug_tag != nullptr) { - format_writer.write('('); - format_writer.write(debug_tag, strlen(debug_tag)); - format_writer.write(") ", 2); + if (debug_tag) { + format_writer.print("({}) ", debug_tag); } - ////////////////////////////////////////////////////// - // append original format string, ensure there is a // - // newline, and NUL terminate // - ////////////////////////////////////////////////////// - format_writer.write(format_string, strlen(format_string)); - format_writer.extend(2); - if (format_writer.data()[format_writer.size() - 1] != '\n') { + format_writer.print("{}", format_string); + + format_writer.extend(2); // restore the space for required termination. + if (format_writer.view().back() != '\n') { // safe because always some chars in the buffer. format_writer.write('\n'); } format_writer.write('\0'); @@ -359,7 +315,7 @@ Diags::print_va(const char *debug_tag, DiagsLevel diags_level, const SourceLocat priority = LOG_NOTICE; break; } - vsnprintf(syslog_buffer, sizeof(syslog_buffer), format_writer.data() + timestamp_end_offset, ap); + vsnprintf(syslog_buffer, sizeof(syslog_buffer), format_writer.data() + timestamp_offset, ap); syslog(priority, "%s", syslog_buffer); } From 6ac53e6487e7ccd691530a59393491d6baf64228 Mon Sep 17 00:00:00 2001 From: Shinya Kawano Date: Wed, 26 Dec 2018 15:19:21 +0900 Subject: [PATCH 108/526] Fix typo in HdrHeap --- proxy/hdrs/HdrHeap.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index 38fe92fe955..27d4b3a7b5e 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -951,7 +951,7 @@ HdrHeap::unmarshal(int buf_length, int obj_type, HdrHeapObjImpl **found_obj, Ref // Nothing to do break; default: - fprintf(stderr, "WARNING: Unmarshal failed due to unknow obj type %d after %d bytes", (int)obj->m_type, + fprintf(stderr, "WARNING: Unmarshal failed due to unknown obj type %d after %d bytes", (int)obj->m_type, (int)(obj_data - (char *)this)); dump_heap(unmarshal_size); return -1; From f373e24471ac44d2639b315b9e493fe93cce8243 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Fri, 28 Dec 2018 19:20:13 -0600 Subject: [PATCH 109/526] Slight clarification to documentation of TLS User Agent Hooks. --- .../plugins/hooks-and-transactions/ssl-hooks.en.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst b/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst index 902be513e59..a21e6365e05 100644 --- a/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst +++ b/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst @@ -22,9 +22,9 @@ TLS User Agent Hooks ******************** -In addition to the HTTP oriented hooks, a plugin can add hooks to trigger code -during the TLS handshake with the user agent. This TLS handshake occurs well before -the HTTP transaction is available, so a separate state machine is required to track the +In addition to the HTTP oriented hooks, a plugin can add hooks (by calling :c:func:`TSHttpHookAdd`) +to trigger code during the TLS handshake with the user agent. This TLS handshake occurs well +before the HTTP transaction is available, so a separate state machine is required to track the TLS hooks. TLS Hooks From 8ae5611ae1c555198a4ddeda68055ac4ff20afd5 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Wed, 19 Dec 2018 20:40:24 +0800 Subject: [PATCH 110/526] Optimize: Do not signal EThreads which are not blocked on cond_timedwait. From the man page of `pthread_cond_timedwait()`: ``` The pthread_cond_timedwait() function atomically blocks the current thread waiting on the condition variable specified by cond, and releases the mutex specified by mutex. The waiting thread unblocks only after another thread calls pthread_cond_signal(3), or pthread_cond_broadcast(3) with the same condition variable, or if the system time reaches the time specified in abstime, and the current thread reacquires the lock on mutex. ``` Refer to the code of `ProtectedQueue::wait()`, ``` ink_mutex_acquire(&lock); if (INK_ATOMICLIST_EMPTY(al)) { timespec ts = ink_hrtime_to_timespec(timeout); ink_cond_timedwait(&might_have_data, &lock, &ts); } ink_mutex_release(&lock); ``` The `EThread::lock` is acquired and then released immediatelly by `ink_cond_timedwait()`. When the thread unblocks, the thread reacquires the lock on `EThread::lock`, and then released immediatelly. Refer to the code of `ProtectedQueue::signal()`, ``` TS_INLINE void ProtectedQueue::signal() { // Need to get the lock before you can signal the thread ink_mutex_acquire(&lock); ink_cond_signal(&might_have_data); ink_mutex_release(&lock); } ``` The threads that try to send signal to the same target Event Thread are blocked on the mutex and send meaningless wakeup signals one by one. It is not necessory, - To acquire and release the `EThread::lock` again and again in the event loop, - To send wakeup signal to an Event Thread that does not blocked on `pthread_cond_timedwait()`. Changes of the commit: The Event Thread has two status: busy and sleep, - Keep `EThread::lock` locked while Event Thread is busy, - The `EThread::lock` is released while Event Thread is sleep. When other threads try to acquire the `EThread::lock` of the target Event Thread, - Acquired, indicating that the target Event Thread is sleep, must send a wakeup signal to the Event Thread. - Failed, indicating that the target Event Thread is busy, do nothing. Backports: If you backport the commit to the branch without PR#2541, - It is safe to remove `flush_signals()` from the event loop. - Also safe to remove any code related with `ethreads_to_be_signalled[]` and `n_ethreads_to_be_signalled`. - Always do `try_signal()` within `ProtectedQueue::enqueue()`. - Suggest to remove `ProtectedQueue::signal()`. --- iocore/eventsystem/I_EThread.h | 7 ++++++- iocore/eventsystem/ProtectedQueue.cc | 17 ++++++++--------- iocore/eventsystem/UnixEThread.cc | 9 +++++++++ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index 7f625451245..8ce926ab1c8 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -371,7 +371,12 @@ class EThread : public Thread void signalActivity() override { - _q.signal(); + /* Try to acquire the `EThread::lock` of the Event Thread: + * - Acquired, indicating that the Event Thread is sleep, + * must send a wakeup signal to the Event Thread. + * - Failed, indicating that the Event Thread is busy, do nothing. + */ + (void)_q.try_signal(); } ProtectedQueue &_q; diff --git a/iocore/eventsystem/ProtectedQueue.cc b/iocore/eventsystem/ProtectedQueue.cc index c4b5377de89..bd434e77325 100644 --- a/iocore/eventsystem/ProtectedQueue.cc +++ b/iocore/eventsystem/ProtectedQueue.cc @@ -114,15 +114,14 @@ ProtectedQueue::dequeue_external() void ProtectedQueue::wait(ink_hrtime timeout) { - // If there are no external events available, don't do a cond_timedwait. + /* If there are no external events available, will do a cond_timedwait. + * + * - The `EThread::lock` will be released, + * - And then the Event Thread goes to sleep and waits for the wakeup signal of `EThread::might_have_data`, + * - The `EThread::lock` will be locked again when the Event Thread wakes up. + */ if (INK_ATOMICLIST_EMPTY(al)) { - ink_mutex_acquire(&lock); - // The "al" may have new events while waiting for the mutex become available. - // We have to recheck the external queue again. - if (INK_ATOMICLIST_EMPTY(al)) { - timespec ts = ink_hrtime_to_timespec(timeout); - ink_cond_timedwait(&might_have_data, &lock, &ts); - } - ink_mutex_release(&lock); + timespec ts = ink_hrtime_to_timespec(timeout); + ink_cond_timedwait(&might_have_data, &lock, &ts); } } diff --git a/iocore/eventsystem/UnixEThread.cc b/iocore/eventsystem/UnixEThread.cc index a5c72c591a8..06920dbe5b7 100644 --- a/iocore/eventsystem/UnixEThread.cc +++ b/iocore/eventsystem/UnixEThread.cc @@ -323,7 +323,16 @@ EThread::execute() switch (tt) { case REGULAR: { + /* The Event Thread has two status: busy and sleep: + * - Keep `EThread::lock` locked while Event Thread is busy, + * - The `EThread::lock` is released while Event Thread is sleep. + * When other threads try to acquire the `EThread::lock` of the target Event Thread: + * - Acquired, indicating that the target Event Thread is sleep, + * - Failed, indicating that the target Event Thread is busy. + */ + ink_mutex_acquire(&EventQueueExternal.lock); this->execute_regular(); + ink_mutex_release(&EventQueueExternal.lock); break; } case DEDICATED: { From 9daeb81ccc46b62ba340898319a0cadd96b2326c Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Fri, 28 Dec 2018 18:07:11 +0800 Subject: [PATCH 111/526] Optimize: tighten the logic of the PluginVC::process_read/write_side() According to the comment on the head of `process_read/write_side`: - To call the `process_read_side()`, the mutexes must be obtained: - PluginVC::mutex - PluginVC::read_state.vio.mutex - To call the `process_write_side()`, the mutexes must be obtained: - PluginVC::mutex - PluginVC::write_state.vio.mutex But it violates the rules when `process_read/write_side()` is called with `other_side_call == true`. The commit makes the code to be compatible with the rules when `other_side_call` is true. This is an update to TS-3339 (0f9dda6). --- proxy/PluginVC.cc | 87 +++++++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc index 85dd7fd9ccc..44d40765e7e 100644 --- a/proxy/PluginVC.cc +++ b/proxy/PluginVC.cc @@ -310,6 +310,7 @@ PluginVC::reenable(VIO *vio) { ink_assert(!closed); ink_assert(magic == PLUGIN_VC_MAGIC_ALIVE); + ink_assert(vio->mutex->thread_holding == this_ethread()); Ptr sm_mutex = vio->mutex; SCOPED_MUTEX_LOCK(lock, sm_mutex, this_ethread()); @@ -332,6 +333,7 @@ PluginVC::reenable_re(VIO *vio) { ink_assert(!closed); ink_assert(magic == PLUGIN_VC_MAGIC_ALIVE); + ink_assert(vio->mutex->thread_holding == this_ethread()); Debug("pvc", "[%u] %s: reenable_re %s", core_obj->id, PVC_TYPE, (vio->op == VIO::WRITE) ? "Write" : "Read"); @@ -473,25 +475,13 @@ PluginVC::process_write_side(bool other_side_call) MIOBuffer *core_buffer = (vc_type == PLUGIN_VC_ACTIVE) ? core_obj->a_to_p_buffer : core_obj->p_to_a_buffer; + Debug("pvc", "[%u] %s: process_write_side", core_obj->id, PVC_TYPE); need_write_process = false; + // Check write_state if (write_state.vio.op != VIO::WRITE || closed || write_state.shutdown) { return; } - // Acquire the lock of the write side continuation - EThread *my_ethread = mutex->thread_holding; - ink_assert(my_ethread != nullptr); - MUTEX_TRY_LOCK(lock, write_state.vio.mutex, my_ethread); - if (!lock.is_locked()) { - Debug("pvc_event", "[%u] %s: process_write_side lock miss, retrying", core_obj->id, PVC_TYPE); - - need_write_process = true; - setup_event_cb(PVC_LOCK_RETRY_TIME, &core_lock_retry_event); - return; - } - - Debug("pvc", "[%u] %s: process_write_side", core_obj->id, PVC_TYPE); - need_write_process = false; // Check the state of our write buffer as well as ntodo int64_t ntodo = write_state.vio.ntodo(); @@ -552,6 +542,30 @@ PluginVC::process_write_side(bool other_side_call) // Wake up the read side on the other side to process these bytes if (!other_side->closed) { if (!other_side_call) { + /* To clear the `need_read_process`, the mutexes must be obtained: + * + * - PluginVC::mutex + * - PluginVC::read_state.vio.mutex + * + */ + if (other_side->read_state.vio.op != VIO::READ || other_side->closed || other_side->read_state.shutdown) { + // Just return, no touch on `other_side->need_read_process`. + return; + } + // Acquire the lock of the read side continuation + EThread *my_ethread = mutex->thread_holding; + ink_assert(my_ethread != nullptr); + MUTEX_TRY_LOCK(lock, other_side->read_state.vio.mutex, my_ethread); + if (!lock.is_locked()) { + Debug("pvc_event", "[%u] %s: process_read_side from other side lock miss, retrying", other_side->core_obj->id, + ((other_side->vc_type == PLUGIN_VC_ACTIVE) ? "Active" : "Passive")); + + // set need_read_process to enforce the read processing + other_side->need_read_process = true; + other_side->setup_event_cb(PVC_LOCK_RETRY_TIME, &other_side->core_lock_retry_event); + return; + } + other_side->process_read_side(true); } else { other_side->read_state.vio.reenable(); @@ -587,28 +601,11 @@ PluginVC::process_read_side(bool other_side_call) core_reader = core_obj->a_to_p_reader; } - need_read_process = false; - - if (read_state.vio.op != VIO::READ || closed) { - return; - } - // Acquire the lock of the read side continuation - EThread *my_ethread = mutex->thread_holding; - ink_assert(my_ethread != nullptr); - MUTEX_TRY_LOCK(lock, read_state.vio.mutex, my_ethread); - if (!lock.is_locked()) { - Debug("pvc_event", "[%u] %s: process_read_side lock miss, retrying", core_obj->id, PVC_TYPE); - - need_read_process = true; - setup_event_cb(PVC_LOCK_RETRY_TIME, &core_lock_retry_event); - return; - } - Debug("pvc", "[%u] %s: process_read_side", core_obj->id, PVC_TYPE); need_read_process = false; - // Check read_state.shutdown after the lock has been obtained. - if (read_state.shutdown) { + // Check read_state + if (read_state.vio.op != VIO::READ || closed || read_state.shutdown) { return; } @@ -668,6 +665,30 @@ PluginVC::process_read_side(bool other_side_call) // intermediate buffer if (!other_side->closed) { if (!other_side_call) { + /* To clear the `need_write_process`, the mutexes must be obtained: + * + * - PluginVC::mutex + * - PluginVC::write_state.vio.mutex + * + */ + if (other_side->write_state.vio.op != VIO::WRITE || other_side->closed || other_side->write_state.shutdown) { + // Just return, no touch on `other_side->need_write_process`. + return; + } + // Acquire the lock of the write side continuation + EThread *my_ethread = mutex->thread_holding; + ink_assert(my_ethread != nullptr); + MUTEX_TRY_LOCK(lock, other_side->write_state.vio.mutex, my_ethread); + if (!lock.is_locked()) { + Debug("pvc_event", "[%u] %s: process_write_side from other side lock miss, retrying", other_side->core_obj->id, + ((other_side->vc_type == PLUGIN_VC_ACTIVE) ? "Active" : "Passive")); + + // set need_write_process to enforce the write processing + other_side->need_write_process = true; + other_side->setup_event_cb(PVC_LOCK_RETRY_TIME, &other_side->core_lock_retry_event); + return; + } + other_side->process_write_side(true); } else { other_side->write_state.vio.reenable(); From af0ad4a1880a21743e98331855bb78e15d5406ef Mon Sep 17 00:00:00 2001 From: Jean Baptiste Favre Date: Fri, 4 Jan 2019 10:34:06 +0100 Subject: [PATCH 112/526] Fix spelling errors reported by lintian --- build/pkg.m4 | 2 +- doc/admin-guide/files/ip_allow.config.en.rst | 4 ++-- doc/admin-guide/files/parent.config.en.rst | 6 +++--- doc/admin-guide/files/records.config.en.rst | 18 +++++++++--------- doc/admin-guide/files/remap.config.en.rst | 2 +- doc/admin-guide/files/storage.config.en.rst | 2 +- doc/admin-guide/plugins/header_rewrite.en.rst | 2 +- doc/appendices/command-line/traffic_ctl.en.rst | 2 +- .../api/functions/TSCacheRemove.en.rst | 2 +- .../api/functions/TSContSchedule.en.rst | 2 +- .../functions/TSHttpConnectWithPluginId.en.rst | 2 +- .../functions/TSHttpOverridableConfig.en.rst | 2 +- .../api/functions/TSHttpTxnErrorBodySet.en.rst | 2 +- .../api/functions/TSHttpTxnMilestoneGet.en.rst | 2 +- .../functions/TSHttpTxnServerIntercept.en.rst | 2 +- .../api/functions/TSIOBufferReader.en.rst | 2 +- .../api/functions/TSSslContext.en.rst | 4 ++-- .../api/functions/TSSslSession.en.rst | 2 +- .../api/functions/TSStat.en.rst | 4 ++-- .../api/functions/TSVConnReenable.en.rst | 2 +- .../api/functions/TSfwrite.en.rst | 2 +- include/tscpp/util/TextView.h | 4 ++-- iocore/cache/CacheHosting.cc | 2 +- iocore/cache/CacheVol.cc | 2 +- iocore/dns/SplitDNS.cc | 2 +- iocore/eventsystem/IOBuffer.cc | 2 +- iocore/eventsystem/I_IOBuffer.h | 2 +- iocore/net/OCSPStapling.cc | 2 +- iocore/net/Socks.cc | 2 +- iocore/net/UnixUDPNet.cc | 2 +- lib/records/RecHttp.cc | 2 +- mgmt/Alarms.cc | 2 +- mgmt/Rollback.cc | 2 +- plugins/esi/lib/EsiProcessor.cc | 8 ++++---- plugins/esi/lib/Variables.cc | 6 +++--- .../collapsed_forwarding.cc | 2 +- plugins/experimental/fq_pacing/fq_pacing.c | 2 +- .../header_normalize/header_normalize.cc | 2 +- plugins/experimental/prefetch/plugin.cc | 2 +- plugins/experimental/uri_signing/parse.c | 2 +- plugins/experimental/uri_signing/uri_signing.c | 2 +- plugins/experimental/url_sig/url_sig.c | 2 +- plugins/generator/generator.cc | 2 +- plugins/header_rewrite/header_rewrite.cc | 4 ++-- plugins/lua/ts_lua_transform.c | 2 +- plugins/s3_auth/s3_auth.cc | 2 +- proxy/ParentSelection.cc | 2 +- proxy/http/HttpSessionManager.cc | 2 +- proxy/logging/LogBuffer.cc | 2 +- proxy/logging/LogObject.cc | 2 +- src/traffic_cache_tool/CacheTool.cc | 2 +- src/traffic_crashlog/traffic_crashlog.cc | 2 +- src/traffic_logstats/logstats.cc | 2 +- src/traffic_manager/traffic_manager.cc | 2 +- src/traffic_server/CoreUtils.h | 8 ++++---- src/traffic_server/InkAPITest.cc | 2 +- src/tscore/ArgParser.cc | 2 +- src/tscore/HostLookup.cc | 2 +- 58 files changed, 81 insertions(+), 81 deletions(-) diff --git a/build/pkg.m4 b/build/pkg.m4 index b7ca359a88a..dd5a6313a57 100644 --- a/build/pkg.m4 +++ b/build/pkg.m4 @@ -53,7 +53,7 @@ fi[]dnl # to PKG_CHECK_MODULES(), but does not set variables or print errors. # # Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) -# only at the first occurence in configure.ac, so if the first place +# only at the first occurrence in configure.ac, so if the first place # it's called might be skipped (such as if it is within an "if", you # have to call PKG_CHECK_EXISTS manually # -------------------------------------------------------------- diff --git a/doc/admin-guide/files/ip_allow.config.en.rst b/doc/admin-guide/files/ip_allow.config.en.rst index b313f8368ef..ce7c632ad69 100644 --- a/doc/admin-guide/files/ip_allow.config.en.rst +++ b/doc/admin-guide/files/ip_allow.config.en.rst @@ -55,7 +55,7 @@ range with the lower and upper values equal to that IP address). The value of ``method`` is a string which must consist of either HTTP method names separated by the character '|' or the keyword literal ``ALL``. This keyword may omitted in which case it is treated as if it were ``method=ALL``. Methods can also be specified by having multiple instances of the -``method`` keyword, each specifiying a single method. E.g., ``method=GET|HEAD`` is the same as +``method`` keyword, each specifying a single method. E.g., ``method=GET|HEAD`` is the same as ``method=GET method=HEAD``. The method names are not validated which means non-standard method names can be specified. @@ -104,7 +104,7 @@ If the entire subnet were to be denied, that would be:: src_ip=123.45.6.0/24 action=ip_deny -The following example allows to any upstream servers:: +The following example allows one to any upstream servers:: dest_ip=0.0.0.0-255.255.255.255 action=ip_allow diff --git a/doc/admin-guide/files/parent.config.en.rst b/doc/admin-guide/files/parent.config.en.rst index 4f4de28f117..725b0d2f99e 100644 --- a/doc/admin-guide/files/parent.config.en.rst +++ b/doc/admin-guide/files/parent.config.en.rst @@ -210,7 +210,7 @@ The following list shows the possible actions and their allowed values. - ``simple_retry`` - If the parent origin server returns a 404 response on a request a new parent is selected and the request is retried. The number of retries is controlled by ``max_simple_retries`` which is set to 1 by default. - - ``unavailable_server_retry`` - If the parent returns a 503 response or if the reponse matches + - ``unavailable_server_retry`` - If the parent returns a 503 response or if the response matches a list of http 5xx responses defined in ``unavailable_server_retry_responses``, the currently selected parent is marked down and a new parent is selected to retry the request. The number of retries is controlled by ``max_unavailable_server_retries`` which is set to 1 by default. @@ -228,7 +228,7 @@ The following list shows the possible actions and their allowed values. ``max_simple_retries`` By default the value for ``max_simple_retries`` is 1. It may be set to any value in the range 1 to 5. - If ``parent_retry`` is set to ``simple_retry`` or ``both`` a 404 reponse + If ``parent_retry`` is set to ``simple_retry`` or ``both`` a 404 response from a parent origin server will cause the request to be retried using a new parent at most 1 to 5 times as configured by ``max_simple_retries``. @@ -236,7 +236,7 @@ The following list shows the possible actions and their allowed values. ``max_unavailable_server_retries`` By default the value for ``max_unavailable_server_retries`` is 1. It may be set to any value in the range 1 to 5. - If ``parent_retry`` is set to ``unavailable_server_retries`` or ``both`` a 503 reponse + If ``parent_retry`` is set to ``unavailable_server_retries`` or ``both`` a 503 response by default or any http 5xx response listed in the list ``unavailable_server_retry_responses`` from a parent origin server will cause the request to be retried using a new parent after first marking the current parent down. The request will be retried at most 1 to 5 times as configured by ``max_unavailable_server_retries``. diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index a62235da65d..34d9e3a7635 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -803,7 +803,7 @@ mptcp ===== ====================================================================== Value Description ===== ====================================================================== - ``0`` |TS| will buffer the request until the post body has been recieved and + ``0`` |TS| will buffer the request until the post body has been received and then send the request to the origin server. ``1`` Immediately return a ``100 Continue`` from |TS| without waiting for the post body. @@ -1724,7 +1724,7 @@ Proxy User Variables connection=full Full user agent connection :ref:`protocol tags ` ================== =============================================================== - Each paramater in the list must be separated by ``|`` or ``:``. For example, ``for|by=uuid|proto`` is + Each parameter in the list must be separated by ``|`` or ``:``. For example, ``for|by=uuid|proto`` is a valid value for this variable. Note that the ``connection`` parameter is a non-standard extension to RFC 7239. Also note that, while |TS| allows multiple ``by`` parameters for the same proxy, this is prohibited by RFC 7239. Currently, for the ``host`` parameter to provide the original host from the @@ -2478,7 +2478,7 @@ DNS .. ts:cv:: CONFIG proxy.config.dns.resolv_conf STRING /etc/resolv.conf - Allows to specify which ``resolv.conf`` file to use for finding resolvers. While the format of this file must be the same as the + Allows one to specify which ``resolv.conf`` file to use for finding resolvers. While the format of this file must be the same as the standard ``resolv.conf`` file, this option allows an administrator to manage the set of resolvers in an external configuration file, without affecting how the rest of the operating system uses DNS. @@ -2496,7 +2496,7 @@ DNS :reloadable: :overridable: - Indicates whether to use SRV records for orgin server lookup. + Indicates whether to use SRV records for origin server lookup. .. ts:cv:: CONFIG proxy.config.dns.dedicated_thread INT 0 @@ -2729,7 +2729,7 @@ HostDB Set the frequency (in seconds) to sync hostdb to disk. Note: hostdb is syncd to disk on a per-partition basis (of which there are 64). - This means that the minumum time to sync all data to disk is :ts:cv:`proxy.config.cache.hostdb.sync_frequency` * 64 + This means that the minimum time to sync all data to disk is :ts:cv:`proxy.config.cache.hostdb.sync_frequency` * 64 Logging Configuration ===================== @@ -3050,7 +3050,7 @@ Diagnostic Logging Configuration .. ts:cv:: CONFIG proxy.config.diags.debug.tags STRING http|dns - Each |TS| `diag` and `debug` level message is annotated with a subsytem tag. This configuration + Each |TS| `diag` and `debug` level message is annotated with a subsystem tag. This configuration contains an anchored regular expression that filters the messages based on the tag. The expressions are prefix matched which creates an implicit ``.*`` at the end. Therefore the default value ``http|dns`` will match tags such as ``http``, ``http_hdrs``, ``dns``, and ``dns_recv``. @@ -3058,7 +3058,7 @@ Diagnostic Logging Configuration Some commonly used debug tags are: ============ ===================================================== - Tag Subsytem usage + Tag Subsystem usage ============ ===================================================== dns DNS query resolution http_hdrs Logs the headers for HTTP requests and responses @@ -3221,7 +3221,7 @@ SSL Termination .. ts:cv:: CONFIG proxy.config.ssl.client.groups_list STRING Configures the list of supported groups provided by OpenSSL which - |TS| will use for the "key_share" and "supported groups" extention + |TS| will use for the "key_share" and "supported groups" extension of TLSv1.3 connections. The value is a colon separated list of group NIDs or names, for example "P-521:P-384:P-256". For instructions, see "Groups" section of `TLS1.3 - OpenSSLWiki `_. @@ -3364,7 +3364,7 @@ SSL Termination ``0`` Disables the session cache entirely. ``1`` Enables the session cache using OpenSSL's implementation. ``2`` Default. Enables the session cache using |TS|'s implementation. This - implentation should perform much better than the OpenSSL + implementation should perform much better than the OpenSSL implementation. ===== ====================================================================== diff --git a/doc/admin-guide/files/remap.config.en.rst b/doc/admin-guide/files/remap.config.en.rst index bbd5f6dfccd..bd7ee3d21da 100644 --- a/doc/admin-guide/files/remap.config.en.rst +++ b/doc/admin-guide/files/remap.config.en.rst @@ -415,7 +415,7 @@ Acl Filters Acl filters can be created to control access of specific remap lines. The markup is very similar to that of :file:`ip_allow.config`, with slight changes to -accomodate remap markup +accommodate remap markup Examples -------- diff --git a/doc/admin-guide/files/storage.config.en.rst b/doc/admin-guide/files/storage.config.en.rst index 943dc94e22a..91b72b5e775 100644 --- a/doc/admin-guide/files/storage.config.en.rst +++ b/doc/admin-guide/files/storage.config.en.rst @@ -92,7 +92,7 @@ which will effectively clear most of the cache. This can be problem when drives reboot causes the path names to change. The :arg:`id` option can be used to create a fixed string that an administrator can use to keep the -assignment table consistent by maintaing the mapping from physical device to base string even in the presence of hardware changes and failures. +assignment table consistent by maintaining the mapping from physical device to base string even in the presence of hardware changes and failures. Examples ======== diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index e0eeec37418..17f3d7f79df 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -326,7 +326,7 @@ The data that can be checked is :: %{INBOUND:REMOTE-PORT} The client port for the connection. %{INBOUND:TLS} The TLS protocol if the connection is over TLS, otherwise the empty string. %{INBOUND:H2} The string "h2" if the connection is HTTP/2, otherwise the empty string. - %{INBOUND:IPV4} The string "ipv4" if the connection is IPv4, otherwise the emtpy string. + %{INBOUND:IPV4} The string "ipv4" if the connection is IPv4, otherwise the empty string. %{INBOUND:IPV6} The string "ipv6" if the connection is IPv6, otherwise the empty string. %{INBOUND:IP-FAMILY} The IP family, either "ipv4" or "ipv6". %{INBOUND:STACK} The full protocol stack separated by ','. diff --git a/doc/appendices/command-line/traffic_ctl.en.rst b/doc/appendices/command-line/traffic_ctl.en.rst index 5304b6ef6c6..6539eda726d 100644 --- a/doc/appendices/command-line/traffic_ctl.en.rst +++ b/doc/appendices/command-line/traffic_ctl.en.rst @@ -255,7 +255,7 @@ traffic_ctl host .. program:: traffic_ctl host .. option:: status HOSTNAME [HOSTNAME ...] - Get the current status of the hosts used in parent.config as a next hop in a multi-tiered cache heirarchy. The value 0 or 1 is returned indicating that the host is marked as down '0' or marked as up '1'. If a host is marked as down, it will not be used as the next hop parent, another host marked as up will be chosen. + Get the current status of the hosts used in parent.config as a next hop in a multi-tiered cache hierarchy. The value 0 or 1 is returned indicating that the host is marked as down '0' or marked as up '1'. If a host is marked as down, it will not be used as the next hop parent, another host marked as up will be chosen. .. program:: traffic_ctl host .. option:: down --time seconds --reason 'manual|active|local' HOSTNAME [HOSTNAME ...] diff --git a/doc/developer-guide/api/functions/TSCacheRemove.en.rst b/doc/developer-guide/api/functions/TSCacheRemove.en.rst index 73cfd7d20b7..f5647d59a32 100644 --- a/doc/developer-guide/api/functions/TSCacheRemove.en.rst +++ b/doc/developer-guide/api/functions/TSCacheRemove.en.rst @@ -41,4 +41,4 @@ the cache calls :arg:`contp` back with the event In both of these callbacks, the user (:arg:`contp`) does not have to do anything. The user does not get any vconnection from the cache, since no data needs to be transferred. When the cache calls :arg:`contp` back with -:data:`TS_EVENT_CACHE_REMOVE`, the remove has already been commited. +:data:`TS_EVENT_CACHE_REMOVE`, the remove has already been committed. diff --git a/doc/developer-guide/api/functions/TSContSchedule.en.rst b/doc/developer-guide/api/functions/TSContSchedule.en.rst index 27820cdcb5d..944a481402b 100644 --- a/doc/developer-guide/api/functions/TSContSchedule.en.rst +++ b/doc/developer-guide/api/functions/TSContSchedule.en.rst @@ -32,7 +32,7 @@ Description =========== Schedules :arg:`contp` to run :arg:`delay` milliseconds in the future. This is approximate. The delay -will be at least :arg:`delay` but possibly more. Resultions finer than roughly 5 milliseconds will +will be at least :arg:`delay` but possibly more. Resolutions finer than roughly 5 milliseconds will not be effective. :arg:`contp` is required to have a mutex, which is provided to :func:`TSContCreate`. diff --git a/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst b/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst index 88e805cb328..0f6186f5ec9 100644 --- a/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst +++ b/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst @@ -79,7 +79,7 @@ virtual connection. The combination of :arg:`tag` and :arg:`id` is intended to enable correlation in log post processing. The :arg:`tag` identifies the connection as related -to the plugin and the :arg:`id` can be used in conjuction with plugin +to the plugin and the :arg:`id` can be used in conjunction with plugin generated logs to correlate the log records. Notes diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index 941686186c0..91eaaff7d8a 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -43,7 +43,7 @@ Description Some of the values that are set in :file:`records.config` can be changed for a specific transaction. It is important to note that these functions change the -configuration values stored for the transation, which is not quite the same as +configuration values stored for the transaction, which is not quite the same as changing the actual operating values of the transaction. The critical effect is the value must be changed before it is used by the transaction - after that, changes will not have any effect. diff --git a/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst index cee7207fbd2..52fc4c01c71 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst @@ -36,4 +36,4 @@ Description Note that both string arguments must be allocated with :c:func:`TSmalloc` or :c:func:`TSstrdup`. The :arg:`mimetype` is optional, and if not provided it defaults to :literal:`text/html`. Sending an empty string would prevent setting -a content type header (but that is not adviced). +a content type header (but that is not advised). diff --git a/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst index 08fb5d34ee2..1c0ee733053 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst @@ -138,7 +138,7 @@ is successful. .. macro:: TS_MILESTONE_LAST_ENTRY - A psuedo index which is set to be one more than the last valid index. This is useful for looping over the data. + A pseudo index which is set to be one more than the last valid index. This is useful for looping over the data. * The server connect times predate the transmission of the :literal:`SYN` diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst index a546aa33403..bc70ecc5883 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst @@ -53,7 +53,7 @@ The response from the plugin is cached subject to standard and configured HTTP caching rules. Should the plugin wish the response not be cached, the plugin must use appropriate HTTP response headers to prevent caching. The primary purpose of :func:`TSHttpTxnServerIntercept` is allow plugins to provide gateways -to other protocols or to allow to plugin to its own transport for the next hop +to other protocols or to allow one to plugin to its own transport for the next hop to the server. :func:`TSHttpTxnServerIntercept` overrides parent cache configuration. diff --git a/doc/developer-guide/api/functions/TSIOBufferReader.en.rst b/doc/developer-guide/api/functions/TSIOBufferReader.en.rst index 95919620c1d..a5c5702f19e 100644 --- a/doc/developer-guide/api/functions/TSIOBufferReader.en.rst +++ b/doc/developer-guide/api/functions/TSIOBufferReader.en.rst @@ -59,7 +59,7 @@ has two very important consequences -- * Conversely keeping a reader around unused will pin the buffer data in memory. This can be useful or harmful. A buffer has a fixed amount of possible readers (currently 5) which is determined at compile -time. Reader allocation is fast and cheap until this maxium is reached at which point it fails. +time. Reader allocation is fast and cheap until this maximum is reached at which point it fails. :func:`TSIOBufferReaderAlloc` allocates a reader for the IO buffer :arg:`bufp`. This should only be called on a newly allocated buffer. If not the location of the reader in the buffer will be diff --git a/doc/developer-guide/api/functions/TSSslContext.en.rst b/doc/developer-guide/api/functions/TSSslContext.en.rst index e921d5bacf6..9337895a525 100644 --- a/doc/developer-guide/api/functions/TSSslContext.en.rst +++ b/doc/developer-guide/api/functions/TSSslContext.en.rst @@ -36,11 +36,11 @@ Description =========== :func:`TSSslContextFindByName` searches for a SSL server context -created from :file:`ssl_multicert.config`, matching against the +created from :file:`ssl_multicert.config`, matchingg against the server :arg:`name`. :func:`TSSslContextFindByAddr` searches for a SSL server context -created from :file:`ssl_multicert.config` matchin against the server +created from :file:`ssl_multicert.config` matching against the server :arg:`address`. diff --git a/doc/developer-guide/api/functions/TSSslSession.en.rst b/doc/developer-guide/api/functions/TSSslSession.en.rst index ebf9fccd30a..321f34fc942 100644 --- a/doc/developer-guide/api/functions/TSSslSession.en.rst +++ b/doc/developer-guide/api/functions/TSSslSession.en.rst @@ -46,7 +46,7 @@ The functions also work with the :type:`TSSslSession` object which can be cast t These functions perform the appropriate locking on the session cache to avoid errors. -The :func:`TSSslSessionGet` and :func:`TSSslSessionGetBuffer` functions retreive the :type:`TSSslSession` object that is identifed by the +The :func:`TSSslSessionGet` and :func:`TSSslSessionGetBuffer` functions retrieve the :type:`TSSslSession` object that is identifed by the :type:`TSSslSessionID` object. If there is no matching sesion object, :func:`TSSslSessionGet` returns NULL and :func:`TSSslSessionGetBuffer` returns 0. diff --git a/doc/developer-guide/api/functions/TSStat.en.rst b/doc/developer-guide/api/functions/TSStat.en.rst index 43d585fa2d7..64b539f2bbc 100644 --- a/doc/developer-guide/api/functions/TSStat.en.rst +++ b/doc/developer-guide/api/functions/TSStat.en.rst @@ -46,9 +46,9 @@ Description A plugin statistic is created by :func:`TSStatCreate`. The :arg:`name` must be globally unique and should follow the standard dotted tag form. To avoid collisions and for easy of use the first tag -should be the plugin name or something easily derived from it. Currently only integers are suppored +should be the plugin name or something easily derived from it. Currently only integers are supported therefore :arg:`type` must be :macro:`TS_RECORDDATATYPE_INT`. The return value is the index of the -statistic. In general thsi should work but if it doesn't it will :code:`assert`. In particular, +statistic. In general this should work but if it doesn't it will :code:`assert`. In particular, creating the same statistic twice will fail in this way, which can happen if statistics are created as part of or based on configuration files and |TS| is reloaded. diff --git a/doc/developer-guide/api/functions/TSVConnReenable.en.rst b/doc/developer-guide/api/functions/TSVConnReenable.en.rst index 6c8f8c14596..ff3aba08142 100644 --- a/doc/developer-guide/api/functions/TSVConnReenable.en.rst +++ b/doc/developer-guide/api/functions/TSVConnReenable.en.rst @@ -32,7 +32,7 @@ Description =========== Reenable the SSL connection :arg:`svc`. If a plugin hook is called, ATS -processing on that connnection will not resume until this is invoked for that +processing on that connection will not resume until this is invoked for that connection. If the server is running OpenSSL 1.0.2, the plugin writer can pause SSL handshake diff --git a/doc/developer-guide/api/functions/TSfwrite.en.rst b/doc/developer-guide/api/functions/TSfwrite.en.rst index 30b33aec8ac..3b064ace9db 100644 --- a/doc/developer-guide/api/functions/TSfwrite.en.rst +++ b/doc/developer-guide/api/functions/TSfwrite.en.rst @@ -44,4 +44,4 @@ The behavior is undefined if length is greater than SSIZE_MAX. Return Value ============ -Returns the number of bytes actually written, or -1 if an error occured. +Returns the number of bytes actually written, or -1 if an error occurred. diff --git a/include/tscpp/util/TextView.h b/include/tscpp/util/TextView.h index 0ac740d6517..890ef3071ee 100644 --- a/include/tscpp/util/TextView.h +++ b/include/tscpp/util/TextView.h @@ -263,7 +263,7 @@ class TextView : public std::string_view self_type prefix(size_t n) const; /// Convenience overload to avoid ambiguity for literal numbers. self_type prefix(int n) const; - /** Get the prefix delimited by the first occurence of the character @a c. + /** Get the prefix delimited by the first occurrence of the character @a c. If @a c is not found the entire view is returned. The delimiter character is not included in the returned view. @@ -271,7 +271,7 @@ class TextView : public std::string_view @return A view of the prefix. */ self_type prefix(char c) const; - /** Get the prefix delimited by the first occurence of a character in @a delimiters. + /** Get the prefix delimited by the first occurrence of a character in @a delimiters. If no such character is found the entire view is returned. The delimiter character is not included in the returned view. diff --git a/iocore/cache/CacheHosting.cc b/iocore/cache/CacheHosting.cc index e7895b37f54..08573d7cdfc 100644 --- a/iocore/cache/CacheHosting.cc +++ b/iocore/cache/CacheHosting.cc @@ -715,7 +715,7 @@ ConfigVolumes::BuildListFromString(char *config_file_path, char *file_buf) // added by YTS Team, yamsat for bug id 59632 total += size; if (size > 100 || total > 100) { - err = "Total volume size added upto more than 100 percent, No volumes created"; + err = "Total volume size added up to more than 100 percent, No volumes created"; break; } // ends here diff --git a/iocore/cache/CacheVol.cc b/iocore/cache/CacheVol.cc index 98009a459f7..113ca523eb7 100644 --- a/iocore/cache/CacheVol.cc +++ b/iocore/cache/CacheVol.cc @@ -400,7 +400,7 @@ CacheVC::scanOpenWrite(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) // get volume lock if (writer_lock_retry > SCAN_WRITER_LOCK_MAX_RETRY) { int r = _action.continuation->handleEvent(CACHE_EVENT_SCAN_OPERATION_BLOCKED, nullptr); - Debug("cache_scan", "still havent got the writer lock, asking user.."); + Debug("cache_scan", "still haven't got the writer lock, asking user.."); switch (r) { case CACHE_SCAN_RESULT_RETRY: writer_lock_retry = 0; diff --git a/iocore/dns/SplitDNS.cc b/iocore/dns/SplitDNS.cc index e435ba05296..9834a8be6b9 100644 --- a/iocore/dns/SplitDNS.cc +++ b/iocore/dns/SplitDNS.cc @@ -341,7 +341,7 @@ SplitDNSRecord::ProcessDNSHosts(char *val) if (tmp - current > (MAXDNAME - 1)) { return "DNS server name (ip) is too long"; } else if (tmp - current == 0) { - return "server string is emtpy"; + return "server string is empty"; } *tmp = 0; } diff --git a/iocore/eventsystem/IOBuffer.cc b/iocore/eventsystem/IOBuffer.cc index ff8658dc0a6..1fcd758de89 100644 --- a/iocore/eventsystem/IOBuffer.cc +++ b/iocore/eventsystem/IOBuffer.cc @@ -189,7 +189,7 @@ MIOBuffer::puts(char *s, int64_t len) } if (!*pb || *pb == '\n') { int64_t n = (int64_t)(pb - s); - memcpy(end(), s, n + 1); // Upto and including '\n' + memcpy(end(), s, n + 1); // Up to and including '\n' end()[n + 1] = 0; fill(n + 1); return n + 1; diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h index 0b0c56a1c67..dd1255bd376 100644 --- a/iocore/eventsystem/I_IOBuffer.h +++ b/iocore/eventsystem/I_IOBuffer.h @@ -813,7 +813,7 @@ class IOBufferReader /** Perform a memchr() across the list of IOBufferBlocks. Returns the offset from the current start point of the reader to the first - occurence of character 'c' in the buffer. + occurrence of character 'c' in the buffer. @param c character to look for. @param len number of characters to check. If len exceeds the number diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index ee0534c5869..d5867b78b52 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -92,7 +92,7 @@ stapling_get_issuer(SSL_CTX *ssl_ctx, X509 *x) #ifdef SSL_CTX_select_current_cert if (!SSL_CTX_select_current_cert(ssl_ctx, x)) { - Warning("OCSP: could not select current certifcate chain %p", x); + Warning("OCSP: could not select current certificate chain %p", x); } #endif diff --git a/iocore/net/Socks.cc b/iocore/net/Socks.cc index e1915f2dc95..a1a59999b96 100644 --- a/iocore/net/Socks.cc +++ b/iocore/net/Socks.cc @@ -694,7 +694,7 @@ socks5PasswdAuthHandler(int event, unsigned char *p, void (**h_ptr)(void)) // NEC thinks it is 5 RFC seems to indicate 1. switch (p[1]) { case 0: - Debug("Socks", "Username/Passwd succeded"); + Debug("Socks", "Username/Passwd succeeded"); *h_ptr = nullptr; break; diff --git a/iocore/net/UnixUDPNet.cc b/iocore/net/UnixUDPNet.cc index 529a38a23d8..e6951c6a54d 100644 --- a/iocore/net/UnixUDPNet.cc +++ b/iocore/net/UnixUDPNet.cc @@ -641,7 +641,7 @@ UDPNetProcessor::CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action } if ((res = safe_getsockname(fd, &local_addr.sa, &local_addr_len)) < 0) { - Debug("udpnet", "CreateUdpsocket: getsockname didnt' work"); + Debug("udpnet", "CreateUdpsocket: getsockname didn't work"); goto HardError; } } diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index f3da6f70d73..c7acb5e0f03 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -406,7 +406,7 @@ HttpProxyPort::processOptions(const char *opts) if (in_ip_set_p && m_family != m_inbound_ip.family()) { std::string_view iname{ats_ip_family_name(m_inbound_ip.family())}; std::string_view fname{ats_ip_family_name(m_family)}; - Warning("Invalid port descriptor '%s' - the inbound adddress family [%.*s] is not the same type as the explicit family value " + Warning("Invalid port descriptor '%s' - the inbound address family [%.*s] is not the same type as the explicit family value " "[%.*s]", opts, static_cast(iname.size()), iname.data(), static_cast(fname.size()), fname.data()); zret = false; diff --git a/mgmt/Alarms.cc b/mgmt/Alarms.cc index 07441f95f4b..2240b025781 100644 --- a/mgmt/Alarms.cc +++ b/mgmt/Alarms.cc @@ -289,7 +289,7 @@ Alarms::signalAlarm(alarm_t a, const char *desc, const char *ip) (*(func))(a, ip, desc); } - /* Priority 2 alarms get signalled if they are the first unsolved occurence. */ + /* Priority 2 alarms get signalled if they are the first unsolved occurrence. */ if (priority == 2 && !ip) { execAlarmBin(desc); } diff --git a/mgmt/Rollback.cc b/mgmt/Rollback.cc index e29910190e4..0fcc8e702c7 100644 --- a/mgmt/Rollback.cc +++ b/mgmt/Rollback.cc @@ -143,7 +143,7 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc mgmt_log("[RollBack::Rollback] Automatic Rollback to prior version failed for %s : %s\n", fileName, strerror(errno)); needZeroLength = true; } else { - mgmt_log("[RollBack::Rollback] Automatic Rollback to version succeded for %s\n", fileName, strerror(errno)); + mgmt_log("[RollBack::Rollback] Automatic Rollback to version succeeded for %s\n", fileName, strerror(errno)); needZeroLength = false; highestSeen--; // Since we've made the highestVersion active diff --git a/plugins/esi/lib/EsiProcessor.cc b/plugins/esi/lib/EsiProcessor.cc index bb74e5a634a..272c3ef102a 100644 --- a/plugins/esi/lib/EsiProcessor.cc +++ b/plugins/esi/lib/EsiProcessor.cc @@ -305,7 +305,7 @@ EsiProcessor::process(const char *&data, int &data_len) /* FAILURE CACHE */ FailureData *data = static_cast(pthread_getspecific(threadKey)); - _debugLog("plugin_esi_failureInfo", "[%s]Fetched data related to thread specfic %p", __FUNCTION__, data); + _debugLog("plugin_esi_failureInfo", "[%s]Fetched data related to thread specific %p", __FUNCTION__, data); for (iter = try_iter->attempt_nodes.begin(); iter != try_iter->attempt_nodes.end(); ++iter) { if ((iter->type == DocNode::TYPE_INCLUDE) || iter->type == DocNode::TYPE_SPECIAL_INCLUDE) { @@ -342,7 +342,7 @@ EsiProcessor::process(const char *&data, int &data_len) } } if (attempt_succeeded) { - _debugLog(_debug_tag, "[%s] attempt section succeded; using attempt section", __FUNCTION__); + _debugLog(_debug_tag, "[%s] attempt section succeeded; using attempt section", __FUNCTION__); _node_list.splice(try_iter->pos, try_iter->attempt_nodes); } else { _debugLog(_debug_tag, "[%s] attempt section errored; trying except section", __FUNCTION__); @@ -436,7 +436,7 @@ EsiProcessor::flush(string &data, int &overall_len) /* FAILURE CACHE */ FailureData *fdata = static_cast(pthread_getspecific(threadKey)); - _debugLog("plugin_esi_failureInfo", "[%s]Fetched data related to thread specfic %p", __FUNCTION__, fdata); + _debugLog("plugin_esi_failureInfo", "[%s]Fetched data related to thread specific %p", __FUNCTION__, fdata); for (iter = try_iter->attempt_nodes.begin(); iter != try_iter->attempt_nodes.end(); ++iter) { if ((iter->type == DocNode::TYPE_INCLUDE) || iter->type == DocNode::TYPE_SPECIAL_INCLUDE) { @@ -473,7 +473,7 @@ EsiProcessor::flush(string &data, int &overall_len) } } if (attempt_succeeded) { - _debugLog(_debug_tag, "[%s] attempt section succeded; using attempt section", __FUNCTION__); + _debugLog(_debug_tag, "[%s] attempt section succeeded; using attempt section", __FUNCTION__); _n_prescanned_nodes = _n_prescanned_nodes + try_iter->attempt_nodes.size(); _node_list.splice(try_iter->pos, try_iter->attempt_nodes); } else { diff --git a/plugins/esi/lib/Variables.cc b/plugins/esi/lib/Variables.cc index ca304857715..adec87e72e1 100644 --- a/plugins/esi/lib/Variables.cc +++ b/plugins/esi/lib/Variables.cc @@ -437,18 +437,18 @@ Variables::_parseDictVariable(const std::string &variable, const char *&header, for (int i = 0; i < (var_size - 1); ++i) { if (variable[i] == '{') { if (paranth_index != -1) { - _debugLog(_debug_tag, "[%s] Cannot have multiple paranthesis in dict variable [%.*s]", __FUNCTION__, var_size, var_ptr); + _debugLog(_debug_tag, "[%s] Cannot have multiple parenthesis in dict variable [%.*s]", __FUNCTION__, var_size, var_ptr); return false; } paranth_index = i; } if (variable[i] == '}') { - _debugLog(_debug_tag, "[%s] Cannot have multiple paranthesis in dict variable [%.*s]", __FUNCTION__, var_size, var_ptr); + _debugLog(_debug_tag, "[%s] Cannot have multiple parenthesis in dict variable [%.*s]", __FUNCTION__, var_size, var_ptr); return false; } } if (paranth_index == -1) { - _debugLog(_debug_tag, "[%s] Could not find opening paranthesis in variable [%.*s]", __FUNCTION__, var_size, var_ptr); + _debugLog(_debug_tag, "[%s] Could not find opening parenthesis in variable [%.*s]", __FUNCTION__, var_size, var_ptr); return false; } if (paranth_index == 0) { diff --git a/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc b/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc index 9e4d41ed2ba..d9165bed6ff 100644 --- a/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc +++ b/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc @@ -353,7 +353,7 @@ TSRemapInit(TSRemapInterface * /* api_info */, char * /* errbuf */, int /* errbu TSError("Cannot initialize %s as both global and remap plugin", DEBUG_TAG); return TS_ERROR; } else { - TSDebug(DEBUG_TAG, "plugin is succesfully initialized for remap"); + TSDebug(DEBUG_TAG, "plugin is successfully initialized for remap"); return TS_SUCCESS; } } diff --git a/plugins/experimental/fq_pacing/fq_pacing.c b/plugins/experimental/fq_pacing/fq_pacing.c index 297e23fbd7c..b38a433fb28 100644 --- a/plugins/experimental/fq_pacing/fq_pacing.c +++ b/plugins/experimental/fq_pacing/fq_pacing.c @@ -118,7 +118,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_ERROR; } - TSDebug(PLUGIN_NAME, "plugin is succesfully initialized"); + TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); return TS_SUCCESS; } diff --git a/plugins/experimental/header_normalize/header_normalize.cc b/plugins/experimental/header_normalize/header_normalize.cc index 1dddf3565be..d408ed5fdfe 100644 --- a/plugins/experimental/header_normalize/header_normalize.cc +++ b/plugins/experimental/header_normalize/header_normalize.cc @@ -158,7 +158,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_ERROR; } buildHdrMap(); - TSDebug(PLUGIN_NAME, "plugin is succesfully initialized"); + TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); return TS_SUCCESS; } diff --git a/plugins/experimental/prefetch/plugin.cc b/plugins/experimental/prefetch/plugin.cc index 1c16ae082b3..e71d039695a 100644 --- a/plugins/experimental/prefetch/plugin.cc +++ b/plugins/experimental/prefetch/plugin.cc @@ -202,7 +202,7 @@ evaluate(const String &v) } else { stmt.assign(v); } - PrefetchDebug("statement: '%s', formating length: %zu", stmt.c_str(), len); + PrefetchDebug("statement: '%s', formatting length: %zu", stmt.c_str(), len); int result = 0; pos = stmt.find_first_of("+-"); diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index ade5706a42d..6f7d421319d 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -142,7 +142,7 @@ validate_jws(cjose_jws_t *jws, struct config *cfg, const char *uri, size_t uri_c PluginDebug("Initial validation of JWT failed for %16p", jws); goto jwt_fail; } - TimerDebug("inital validation of jwt"); + TimerDebug("initial validation of jwt"); cjose_header_t *hdr = cjose_jws_get_protected(jws); TimerDebug("getting header of jws"); diff --git a/plugins/experimental/uri_signing/uri_signing.c b/plugins/experimental/uri_signing/uri_signing.c index c648d183c76..55ba1173045 100644 --- a/plugins/experimental/uri_signing/uri_signing.c +++ b/plugins/experimental/uri_signing/uri_signing.c @@ -46,7 +46,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_ERROR; } - TSDebug(PLUGIN_NAME, "plugin is succesfully initialized"); + TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); return TS_SUCCESS; } diff --git a/plugins/experimental/url_sig/url_sig.c b/plugins/experimental/url_sig/url_sig.c index 6aa1fe0afca..762a7da3ad4 100644 --- a/plugins/experimental/url_sig/url_sig.c +++ b/plugins/experimental/url_sig/url_sig.c @@ -98,7 +98,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_ERROR; } - TSDebug(PLUGIN_NAME, "plugin is succesfully initialized"); + TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); return TS_SUCCESS; } diff --git a/plugins/generator/generator.cc b/plugins/generator/generator.cc index 276313a75a2..5f282cd6891 100644 --- a/plugins/generator/generator.cc +++ b/plugins/generator/generator.cc @@ -609,7 +609,7 @@ GeneratorTxnHook(TSCont contp, TSEvent event, void *edata) TSReleaseAssert(TSHttpTxnCacheLookupStatusGet(arg.txn, &status) == TS_SUCCESS); if (status != TS_CACHE_LOOKUP_HIT_FRESH) { // This transaction is going to be a cache miss, so intercept it. - VDEBUG("intercepting orgin server request for txn=%p", arg.txn); + VDEBUG("intercepting origin server request for txn=%p", arg.txn); TSHttpTxnServerIntercept(TSContCreate(GeneratorInterceptionHook, TSMutexCreate()), arg.txn); } diff --git a/plugins/header_rewrite/header_rewrite.cc b/plugins/header_rewrite/header_rewrite.cc index 60caecda9bb..3afdcc86425 100644 --- a/plugins/header_rewrite/header_rewrite.cc +++ b/plugins/header_rewrite/header_rewrite.cc @@ -330,7 +330,7 @@ TSPluginInit(int argc, const char *argv[]) // just appended to the configurations. TSDebug(PLUGIN_NAME, "Loading global configuration file %s", argv[i]); if (conf->parse_config(argv[i], TS_HTTP_READ_RESPONSE_HDR_HOOK)) { - TSDebug(PLUGIN_NAME, "Succesfully loaded global config file %s", argv[i]); + TSDebug(PLUGIN_NAME, "Successfully loaded global config file %s", argv[i]); got_config = true; } else { TSError("[header_rewrite] failed to parse configuration file %s", argv[i]); @@ -401,7 +401,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSE delete conf; return TS_ERROR; } else { - TSDebug(PLUGIN_NAME, "Succesfully loaded remap config file %s", argv[i]); + TSDebug(PLUGIN_NAME, "Successfully loaded remap config file %s", argv[i]); } } diff --git a/plugins/lua/ts_lua_transform.c b/plugins/lua/ts_lua_transform.c index 56c27fc51bd..57da47d727c 100644 --- a/plugins/lua/ts_lua_transform.c +++ b/plugins/lua/ts_lua_transform.c @@ -89,7 +89,7 @@ ts_lua_transform_handler(TSCont contp, ts_lua_http_transform_ctx *transform_ctx, empty_input = 0; if (!TSVIOBufferGet(input_vio)) { if (transform_ctx->output.vio) { - TSDebug(TS_LUA_DEBUG_TAG, "[%s] reenabling ouput VIO after input VIO does not exist", __FUNCTION__); + TSDebug(TS_LUA_DEBUG_TAG, "[%s] reenabling output VIO after input VIO does not exist", __FUNCTION__); TSVIONBytesSet(transform_ctx->output.vio, transform_ctx->total); TSVIOReenable(transform_ctx->output.vio); return 0; diff --git a/plugins/s3_auth/s3_auth.cc b/plugins/s3_auth/s3_auth.cc index de79dfe9a2e..51bd6148acd 100644 --- a/plugins/s3_auth/s3_auth.cc +++ b/plugins/s3_auth/s3_auth.cc @@ -909,7 +909,7 @@ event_handler(TSCont cont, TSEvent event, void *edata) } if (TS_HTTP_STATUS_OK == status) { - TSDebug(PLUGIN_NAME, "Succesfully signed the AWS S3 URL"); + TSDebug(PLUGIN_NAME, "Successfully signed the AWS S3 URL"); } else { TSDebug(PLUGIN_NAME, "Failed to sign the AWS S3 URL, status = %d", status); TSHttpTxnStatusSet(txnp, status); diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index 882e57eb27b..af463f993ab 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -494,7 +494,7 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) errPtr = "Parent hostname is too long"; goto MERROR; } else if (tmp - current == 0) { - errPtr = "Parent string is emtpy"; + errPtr = "Parent string is empty"; goto MERROR; } // Update the pRecords diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index c39f949124b..6cef39f65c2 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -212,7 +212,7 @@ ServerSessionPool::eventHandler(int event, void *data) if (connection_count_below_min) { Debug("http_ss", "[%" PRId64 "] [session_bucket] session received io notice [%s], " - "reseting timeout to maintain minimum number of connections", + "resetting timeout to maintain minimum number of connections", s->con_id, HttpDebugNames::get_event_name(event)); s->get_netvc()->set_inactivity_timeout(s->get_netvc()->get_inactivity_timeout()); s->get_netvc()->set_active_timeout(s->get_netvc()->get_active_timeout()); diff --git a/proxy/logging/LogBuffer.cc b/proxy/logging/LogBuffer.cc index e12cd21a87a..21b4c87955d 100644 --- a/proxy/logging/LogBuffer.cc +++ b/proxy/logging/LogBuffer.cc @@ -261,7 +261,7 @@ LogBuffer::checkout_write(size_t *write_offset, size_t write_size) } if (switch_state(old_s, new_s)) { - // we succeded in setting the new state + // we succeeded in setting the new state break; } } diff --git a/proxy/logging/LogObject.cc b/proxy/logging/LogObject.cc index 6ccb416f473..9f765dafcd9 100644 --- a/proxy/logging/LogObject.cc +++ b/proxy/logging/LogObject.cc @@ -409,7 +409,7 @@ LogObject::_checkout_write(size_t *write_offset, size_t bytes_needed) switch (result_code) { case LogBuffer::LB_OK: - // checkout succeded + // checkout succeeded retry = false; break; diff --git a/src/traffic_cache_tool/CacheTool.cc b/src/traffic_cache_tool/CacheTool.cc index e20a76f4783..13973a51a3e 100644 --- a/src/traffic_cache_tool/CacheTool.cc +++ b/src/traffic_cache_tool/CacheTool.cc @@ -870,7 +870,7 @@ Span::updateHeader() zret.push(0, errno, "Failed to update span - ", strerror(errno)); } } else { - std::cout << "Writing not enabled, no updates perfomed" << std::endl; + std::cout << "Writing not enabled, no updates performed" << std::endl; } return zret; } diff --git a/src/traffic_crashlog/traffic_crashlog.cc b/src/traffic_crashlog/traffic_crashlog.cc index c2848f9491e..42aff7aebf9 100644 --- a/src/traffic_crashlog/traffic_crashlog.cc +++ b/src/traffic_crashlog/traffic_crashlog.cc @@ -198,7 +198,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) mgmterr = TSInit(nullptr, (TSInitOptionT)(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS)); if (mgmterr != TS_ERR_OKAY) { char *msg = TSGetErrorMessage(mgmterr); - Warning("failed to intialize management API: %s", msg); + Warning("failed to initialize management API: %s", msg); TSfree(msg); } diff --git a/src/traffic_logstats/logstats.cc b/src/traffic_logstats/logstats.cc index 72885e1bc65..263292d5061 100644 --- a/src/traffic_logstats/logstats.cc +++ b/src/traffic_logstats/logstats.cc @@ -1828,7 +1828,7 @@ process_file(int in_fd, off_t offset, unsigned max_age) unsigned second_read_size = sizeof(LogBufferHeader) - first_read_size; nread = read(in_fd, &buffer[first_read_size], second_read_size); if (!nread || EOF == nread) { - Debug("logstats", "Second read of header failed (attemped %d bytes at offset %d, got nothing), errno=%d.", second_read_size, + Debug("logstats", "Second read of header failed (attempted %d bytes at offset %d, got nothing), errno=%d.", second_read_size, first_read_size, errno); return 1; } diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc index a0168969e9b..f7aaad20af1 100644 --- a/src/traffic_manager/traffic_manager.cc +++ b/src/traffic_manager/traffic_manager.cc @@ -143,7 +143,7 @@ rotateLogs() if (kill(tspid, SIGUSR2) != 0) { mgmt_log("Could not send SIGUSR2 to TS: %s", strerror(errno)); } else { - mgmt_log("Succesfully sent SIGUSR2 to TS!"); + mgmt_log("Successfully sent SIGUSR2 to TS!"); } } } diff --git a/src/traffic_server/CoreUtils.h b/src/traffic_server/CoreUtils.h index e3e77505585..f3760d33722 100644 --- a/src/traffic_server/CoreUtils.h +++ b/src/traffic_server/CoreUtils.h @@ -42,8 +42,8 @@ #define SP_REGNUM 15 /* Contains address of top of stack USP */ #define PC_REGNUM 12 /* Contains program counter EIP */ #define FP_REGNUM 5 /* Virtual frame pointer EBP */ -#define NO_OF_ARGS \ - 10 /* The argument depth upto which we would be looking into \ +#define NO_OF_ARGS \ + 10 /* The argument depth up to which we would be looking into \ the stack */ // contains local and in registers, frame pointer, and stack base @@ -61,8 +61,8 @@ struct core_stack_state { #include #include -#define NO_OF_ARGS \ - 10 /* The argument depth upto which we would be looking into \ +#define NO_OF_ARGS \ + 10 /* The argument depth up to which we would be looking into \ the stack */ // contains local and in registers, frame pointer, and stack base diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 7e17baf0271..9f7710fc1bb 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -4225,7 +4225,7 @@ REGRESSION_TEST(SDK_API_TSHttpHdr)(RegressionTest *test, int /* atype ATS_UNUSED SDK_RPRINT(test, "TSHttpHdrUrlSet&Get", "TestCase1", TC_FAIL, "TSHttpHdrUrlSet returns TS_ERROR"); } else { if (TSHttpHdrUrlGet(bufp1, hdr_loc1, &url_loc_Get) != TS_SUCCESS) { - SDK_RPRINT(test, "TSHttpHdrUrlSet&Get", "TestCase1", TC_FAIL, "TSHttpHdrUrlGet retuns TS_ERROR"); + SDK_RPRINT(test, "TSHttpHdrUrlSet&Get", "TestCase1", TC_FAIL, "TSHttpHdrUrlGet returns TS_ERROR"); } else { if (url_loc == url_loc_Get) { SDK_RPRINT(test, "TSHttpHdrUrlSet&Get", "TestCase1", TC_PASS, "ok"); diff --git a/src/tscore/ArgParser.cc b/src/tscore/ArgParser.cc index 47e8c2ebfc0..75f1f9fc217 100644 --- a/src/tscore/ArgParser.cc +++ b/src/tscore/ArgParser.cc @@ -172,7 +172,7 @@ ArgParser::parse(const char **argv) }; // if there is anything left, then output usage if (!args.empty()) { - std::string msg = "Unkown command, option or args:"; + std::string msg = "Unknown command, option or args:"; for (const auto &it : args) { msg = msg + " '" + it + "'"; } diff --git a/src/tscore/HostLookup.cc b/src/tscore/HostLookup.cc index d6a7439b5e4..0a47f38a44c 100644 --- a/src/tscore/HostLookup.cc +++ b/src/tscore/HostLookup.cc @@ -307,7 +307,7 @@ CharIndex::Insert(string_view match_data, HostBranch *toInsert) // Check to see if are at the level we supposed be at if (match_data.size() == 1) { - // The slot should always be emtpy, no duplicate keys are allowed + // The slot should always be empty, no duplicate keys are allowed ink_assert(cur->array[index].branch == nullptr); cur->array[index].branch = toInsert; break; From 3216eae41333e7a7526ef06dafec79725a1daef4 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 7 Jan 2019 08:02:04 -0700 Subject: [PATCH 113/526] Updated STATUS with all known releases --- STATUS | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/STATUS b/STATUS index d145713bc35..8bc06306f48 100644 --- a/STATUS +++ b/STATUS @@ -6,7 +6,13 @@ The current version of this file can be found at: * https://github.com/apache/trafficserver/blob/master/STATUS Release history: - 7.1.4 : Released on , 2018 + 8.0.2 : Release on , 2019 + 8.0.1 : Release on Nov 29th, 2018 + 8.0.0 : Release on Sep 25th, 2018 + + 7.1.6 : Released on , 2018 + 7.1.5 : Released on Nov 24th, 2018 + 7.1.4 : Released on Aug 1st, 2018 7.1.3 : Released on Apr 16th, 2018 7.1.2 : Released on Jan 10th, 2018 7.1.1 : Released on Sep 7th, 2017 From 3f753bed168ad925639dfce349056e4bde20c635 Mon Sep 17 00:00:00 2001 From: Jean Baptiste Favre Date: Mon, 7 Jan 2019 10:38:19 +0100 Subject: [PATCH 114/526] Add missing manpages to build list --- doc/manpages.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manpages.py b/doc/manpages.py index 8a4850025d9..97f90e345ea 100644 --- a/doc/manpages.py +++ b/doc/manpages.py @@ -33,6 +33,9 @@ ('appendices/command-line/traffic_top.en', 'traffic_top', u'Display Traffic Server statistics', None, '1'), ('appendices/command-line/tsxs.en', 'tsxs', u'Traffic Server plugin tool', None, '1'), ('appendices/command-line/traffic_via.en', 'traffic_via', u'Traffic Server Via header decoder', None, '1'), + ('appendices/command-line/traffic_layout.en', 'traffic_layout', u'Traffic Server sandbox management tool', None, '1'), + ('appendices/command-line/traffic_cache_tool.en', 'traffic_cache_tool', u'Traffic Server cache management tool', None, '1'), + ('appendices/command-line/traffic_wccp.en', 'traffic_wccp', u'Traffic Server WCCP client', None, '1'), ('admin-guide/files/cache.config.en', 'cache.config', u'Traffic Server cache configuration file', None, '5'), ('admin-guide/files/hosting.config.en', 'hosting.config', u'Traffic Server domain hosting configuration file', None, '5'), From 6f94430109104fe8b43afc9d039481a97aacac25 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 12 Dec 2018 13:12:12 -0800 Subject: [PATCH 115/526] Eliminate remaining hack around copy CTORs This finishes what #2081 started. --- plugins/experimental/fastcgi/src/Profiler.h | 10 -- plugins/experimental/inliner/util.h | 9 -- plugins/header_rewrite/condition.h | 6 +- plugins/header_rewrite/conditions.h | 132 +++++++++++++------- plugins/header_rewrite/lulu.h | 5 - plugins/header_rewrite/matcher.h | 7 +- plugins/header_rewrite/operator.h | 23 ++-- plugins/header_rewrite/operators.h | 127 +++++++++++++------ plugins/header_rewrite/parser.h | 12 +- plugins/header_rewrite/resources.h | 6 +- plugins/header_rewrite/ruleset.h | 6 +- plugins/header_rewrite/statement.h | 6 +- plugins/header_rewrite/value.h | 6 +- 13 files changed, 225 insertions(+), 130 deletions(-) diff --git a/plugins/experimental/fastcgi/src/Profiler.h b/plugins/experimental/fastcgi/src/Profiler.h index 9a882189140..620161e1d1d 100644 --- a/plugins/experimental/fastcgi/src/Profiler.h +++ b/plugins/experimental/fastcgi/src/Profiler.h @@ -28,12 +28,6 @@ namespace ats_plugin { -// A macro to disallow the copy constructor and operator= functions -// This should be used in the private: declarations for a class -#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(TypeName &) = delete; \ - void operator=(TypeName) = delete; - // A single profile, stores data of a taken profile class Profile { @@ -195,8 +189,6 @@ class Profiler // The mutex for safe access mutable std::mutex profiles_mutex_; - - // DISALLOW_COPY_AND_ASSIGN(Profiler); }; // Takes a profile during its life time @@ -233,7 +225,5 @@ class ProfileTaker // The owner profiler Profiler *owner_; - - // DISALLOW_COPY_AND_ASSIGN(ProfileTaker); }; } // namespace ats_plugin diff --git a/plugins/experimental/inliner/util.h b/plugins/experimental/inliner/util.h index 68caff0d23f..854da526c29 100644 --- a/plugins/experimental/inliner/util.h +++ b/plugins/experimental/inliner/util.h @@ -25,15 +25,6 @@ #include -#define DISALLOW_COPY_AND_ASSIGN(T) \ - T(const T &) = delete; \ - void operator=(const T &) = delete - -#define DISALLOW_IMPLICIT_CONSTRUCTORS(T) \ -private: \ - T(void); \ - DISALLOW_COPY_AND_ASSIGN(T) - namespace util { typedef std::vector Buffer; diff --git a/plugins/header_rewrite/condition.h b/plugins/header_rewrite/condition.h index 33bd19f4b4f..081d04f62ce 100644 --- a/plugins/header_rewrite/condition.h +++ b/plugins/header_rewrite/condition.h @@ -55,6 +55,10 @@ class Condition : public Statement delete _matcher; } + // noncopyable + Condition(const Condition &) = delete; + void operator=(const Condition &) = delete; + // Inline this, it's critical for speed (and only used twice) bool do_eval(const Resources &res) @@ -128,7 +132,5 @@ class Condition : public Statement Matcher *_matcher = nullptr; private: - DISALLOW_COPY_AND_ASSIGN(Condition); - CondModifiers _mods = COND_NONE; }; diff --git a/plugins/header_rewrite/conditions.h b/plugins/header_rewrite/conditions.h index 6c22c7f5a04..77ef5952b4a 100644 --- a/plugins/header_rewrite/conditions.h +++ b/plugins/header_rewrite/conditions.h @@ -42,6 +42,11 @@ class ConditionTrue : public Condition { public: ConditionTrue() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionTrue"); } + + // noncopyable + ConditionTrue(const ConditionTrue &) = delete; + void operator=(const ConditionTrue &) = delete; + void append_value(std::string &s, const Resources & /* res ATS_UNUSED */) override { @@ -55,9 +60,6 @@ class ConditionTrue : public Condition TSDebug(PLUGIN_NAME, "Evaluating TRUE()"); return true; } - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionTrue); }; // Always false @@ -65,6 +67,11 @@ class ConditionFalse : public Condition { public: ConditionFalse() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionFalse"); } + + // noncopyable + ConditionFalse(const ConditionFalse &) = delete; + void operator=(const ConditionFalse &) = delete; + void append_value(std::string &s, const Resources & /* res ATS_UNUSED */) override { @@ -78,9 +85,6 @@ class ConditionFalse : public Condition TSDebug(PLUGIN_NAME, "Evaluating FALSE()"); return false; } - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionFalse); }; // Check the HTTP return status @@ -90,15 +94,17 @@ class ConditionStatus : public Condition public: ConditionStatus() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionStatus"); } + + // noncopyable + ConditionStatus(const ConditionStatus &) = delete; + void operator=(const ConditionStatus &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; void initialize_hooks() override; // Return status only valid in certain hooks - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionStatus); }; // Check the HTTP method @@ -108,14 +114,16 @@ class ConditionMethod : public Condition public: ConditionMethod() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionMethod"); } + + // noncopyable + ConditionMethod(const ConditionMethod &) = delete; + void operator=(const ConditionMethod &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionMethod); }; // Random 0 to (N-1) @@ -125,6 +133,11 @@ class ConditionRandom : public Condition public: ConditionRandom() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionRandom"); } + + // noncopyable + ConditionRandom(const ConditionRandom &) = delete; + void operator=(const ConditionRandom &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -132,8 +145,6 @@ class ConditionRandom : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionRandom); - unsigned int _seed = 0; unsigned int _max = 0; }; @@ -143,6 +154,11 @@ class ConditionAccess : public Condition { public: ConditionAccess() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionAccess"); } + + // noncopyable + ConditionAccess(const ConditionAccess &) = delete; + void operator=(const ConditionAccess &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -150,8 +166,6 @@ class ConditionAccess : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionAccess); - time_t _next = 0; bool _last = false; }; @@ -163,6 +177,11 @@ class ConditionCookie : public Condition public: ConditionCookie() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionCookie"); } + + // noncopyable + ConditionCookie(const ConditionCookie &) = delete; + void operator=(const ConditionCookie &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -170,8 +189,6 @@ class ConditionCookie : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionCookie); - // Nginx-style cookie parsing: // nginx/src/http/ngx_http_parse.c:ngx_http_parse_multi_header_lines() inline int @@ -235,6 +252,10 @@ class ConditionHeader : public Condition TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionHeader, client %d", client); } + // noncopyable + ConditionHeader(const ConditionHeader &) = delete; + void operator=(const ConditionHeader &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -242,8 +263,6 @@ class ConditionHeader : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionHeader); - bool _client; }; @@ -254,14 +273,16 @@ class ConditionPath : public Condition public: explicit ConditionPath() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionPath"); } + + // noncopyable + ConditionPath(const ConditionPath &) = delete; + void operator=(const ConditionPath &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionPath); }; // query @@ -271,14 +292,16 @@ class ConditionQuery : public Condition public: explicit ConditionQuery() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionQuery"); } + + // noncopyable + ConditionQuery(const ConditionQuery &) = delete; + void operator=(const ConditionQuery &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionQuery); }; // url @@ -291,6 +314,10 @@ class ConditionUrl : public Condition explicit ConditionUrl(const UrlType type) : _type(type) { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionUrl"); } + // noncopyable + ConditionUrl(const ConditionUrl &) = delete; + void operator=(const ConditionUrl &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -299,8 +326,6 @@ class ConditionUrl : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionUrl); - UrlQualifiers _url_qual = URL_QUAL_NONE; UrlType _type; }; @@ -327,6 +352,10 @@ class ConditionDBM : public Condition // } } + // noncopyable + ConditionDBM(const ConditionDBM &) = delete; + void operator=(const ConditionDBM &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -334,7 +363,6 @@ class ConditionDBM : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionDBM); // MDBM* _dbm; std::string _file; Value _key; @@ -361,6 +389,11 @@ class ConditionIp : public Condition public: explicit ConditionIp() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionIp"); }; + + // noncopyable + ConditionIp(const ConditionIp &) = delete; + void operator=(const ConditionIp &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -369,7 +402,6 @@ class ConditionIp : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionIp); IpQualifiers _ip_qual = IP_QUAL_CLIENT; }; @@ -379,14 +411,16 @@ class ConditionIncomingPort : public Condition public: ConditionIncomingPort() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionIncomingPort"); } + + // noncopyable + ConditionIncomingPort(const ConditionIncomingPort &) = delete; + void operator=(const ConditionIncomingPort &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionIncomingPort); }; // Transact Count @@ -396,14 +430,16 @@ class ConditionTransactCount : public Condition public: ConditionTransactCount() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionTransactCount"); } + + // noncopyable + ConditionTransactCount(const ConditionTransactCount &) = delete; + void operator=(const ConditionTransactCount &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionTransactCount); }; // now: Keeping track of current time / day / hour etc. @@ -413,6 +449,11 @@ class ConditionNow : public Condition public: explicit ConditionNow() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionNow"); } + + // noncopyable + ConditionNow(const ConditionNow &) = delete; + void operator=(const ConditionNow &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -421,8 +462,6 @@ class ConditionNow : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionNow); - int64_t get_now_qualified(NowQualifiers qual) const; NowQualifiers _now_qual = NOW_QUAL_EPOCH; }; @@ -433,6 +472,10 @@ class ConditionGeo : public Condition public: explicit ConditionGeo() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionGeo"); } + // noncopyable + ConditionGeo(const ConditionGeo &) = delete; + void operator=(const ConditionGeo &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -454,8 +497,6 @@ class ConditionGeo : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionGeo); - int64_t get_geo_int(const sockaddr *addr) const; const char *get_geo_string(const sockaddr *addr) const; GeoQualifiers _geo_qual = GEO_QUAL_COUNTRY; @@ -467,6 +508,11 @@ class ConditionId : public Condition { public: explicit ConditionId() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionId"); }; + + // noncopyable + ConditionId(const ConditionId &) = delete; + void operator=(const ConditionId &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -475,7 +521,6 @@ class ConditionId : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionId); IdQualifiers _id_qual = ID_QUAL_UNIQUE; }; @@ -543,6 +588,10 @@ class ConditionStringLiteral : public Condition public: explicit ConditionStringLiteral(const std::string &v); + // noncopyable + ConditionStringLiteral(const ConditionStringLiteral &) = delete; + void operator=(const ConditionStringLiteral &) = delete; + void append_value(std::string &s, const Resources & /* res ATS_UNUSED */) override; protected: @@ -550,5 +599,4 @@ class ConditionStringLiteral : public Condition private: std::string _literal; - DISALLOW_COPY_AND_ASSIGN(ConditionStringLiteral); }; diff --git a/plugins/header_rewrite/lulu.h b/plugins/header_rewrite/lulu.h index c0c20f06380..cadc9023742 100644 --- a/plugins/header_rewrite/lulu.h +++ b/plugins/header_rewrite/lulu.h @@ -35,8 +35,3 @@ uint16_t getPort(sockaddr const *s_sockaddr); extern const char PLUGIN_NAME[]; extern const char PLUGIN_NAME_DBG[]; - -// From google styleguide: http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml -#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName &) = delete; \ - void operator=(const TypeName &) = delete diff --git a/plugins/header_rewrite/matcher.h b/plugins/header_rewrite/matcher.h index 3d8e71eece9..20e99b62cbf 100644 --- a/plugins/header_rewrite/matcher.h +++ b/plugins/header_rewrite/matcher.h @@ -46,11 +46,12 @@ class Matcher explicit Matcher(const MatcherOps op) : _op(op) { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for Matcher"); } virtual ~Matcher() { TSDebug(PLUGIN_NAME_DBG, "Calling DTOR for Matcher"); } + // noncopyable + Matcher(const Matcher &) = delete; + void operator=(const Matcher &) = delete; + protected: const MatcherOps _op; - -private: - DISALLOW_COPY_AND_ASSIGN(Matcher); }; // Template class to match on various types of data diff --git a/plugins/header_rewrite/operator.h b/plugins/header_rewrite/operator.h index b11a1de4e08..04821fcd330 100644 --- a/plugins/header_rewrite/operator.h +++ b/plugins/header_rewrite/operator.h @@ -44,6 +44,11 @@ class Operator : public Statement { public: Operator() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for Operator"); } + + // noncopyable + Operator(const Operator &) = delete; + void operator=(const Operator &) = delete; + OperModifiers get_oper_modifiers() const; void initialize(Parser &p) override; @@ -60,8 +65,6 @@ class Operator : public Statement virtual void exec(const Resources &res) const = 0; private: - DISALLOW_COPY_AND_ASSIGN(Operator); - OperModifiers _mods = OPER_NONE; }; @@ -73,13 +76,15 @@ class OperatorHeaders : public Operator { public: OperatorHeaders() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorHeaders"); } + + // noncopyable + OperatorHeaders(const OperatorHeaders &) = delete; + void operator=(const OperatorHeaders &) = delete; + void initialize(Parser &p) override; protected: std::string _header; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorHeaders); }; /////////////////////////////////////////////////////////////////////////////// @@ -90,11 +95,13 @@ class OperatorCookies : public Operator { public: OperatorCookies() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorCookies"); } + + // noncopyable + OperatorCookies(const OperatorCookies &) = delete; + void operator=(const OperatorCookies &) = delete; + void initialize(Parser &p) override; protected: std::string _cookie; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorCookies); }; diff --git a/plugins/header_rewrite/operators.h b/plugins/header_rewrite/operators.h index 2ff0d847977..f1d24fab342 100644 --- a/plugins/header_rewrite/operators.h +++ b/plugins/header_rewrite/operators.h @@ -36,14 +36,17 @@ class OperatorSetConfig : public Operator { public: OperatorSetConfig() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetConfig"); } + + // noncopyable + OperatorSetConfig(const OperatorSetConfig &) = delete; + void operator=(const OperatorSetConfig &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetConfig); - TSOverridableConfigKey _key = TS_CONFIG_NULL; TSRecordDataType _type = TS_RECORDDATATYPE_NULL; @@ -55,6 +58,11 @@ class OperatorSetStatus : public Operator { public: OperatorSetStatus() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetStatus"); } + + // noncopyable + OperatorSetStatus(const OperatorSetStatus &) = delete; + void operator=(const OperatorSetStatus &) = delete; + void initialize(Parser &p) override; protected: @@ -62,8 +70,6 @@ class OperatorSetStatus : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetStatus); - Value _status; const char *_reason = nullptr; int _reason_len = 0; @@ -73,6 +79,11 @@ class OperatorSetStatusReason : public Operator { public: OperatorSetStatusReason() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetStatusReason"); } + + // noncopyable + OperatorSetStatusReason(const OperatorSetStatusReason &) = delete; + void operator=(const OperatorSetStatusReason &) = delete; + void initialize(Parser &p) override; protected: @@ -80,8 +91,6 @@ class OperatorSetStatusReason : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetStatusReason); - Value _reason; }; @@ -89,14 +98,17 @@ class OperatorSetDestination : public Operator { public: OperatorSetDestination() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetDestination"); } + + // noncopyable + OperatorSetDestination(const OperatorSetDestination &) = delete; + void operator=(const OperatorSetDestination &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetDestination); - UrlQualifiers _url_qual = URL_QUAL_NONE; Value _value; }; @@ -105,6 +117,11 @@ class OperatorSetRedirect : public Operator { public: OperatorSetRedirect() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetRedirect"); } + + // noncopyable + OperatorSetRedirect(const OperatorSetRedirect &) = delete; + void operator=(const OperatorSetRedirect &) = delete; + void initialize(Parser &p) override; TSHttpStatus @@ -123,8 +140,6 @@ class OperatorSetRedirect : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetRedirect); - Value _status; Value _location; }; @@ -134,25 +149,29 @@ class OperatorNoOp : public Operator public: OperatorNoOp() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorNoOp"); } + // noncopyable + OperatorNoOp(const OperatorNoOp &) = delete; + void operator=(const OperatorNoOp &) = delete; + protected: void exec(const Resources & /* res ATS_UNUSED */) const override{}; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorNoOp); }; class OperatorSetTimeoutOut : public Operator { public: OperatorSetTimeoutOut() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetTimeoutOut"); } + + // noncopyable + OperatorSetTimeoutOut(const OperatorSetTimeoutOut &) = delete; + void operator=(const OperatorSetTimeoutOut &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetTimeoutOut); - enum TimeoutOutType { TO_OUT_UNDEFINED, TO_OUT_ACTIVE, @@ -169,14 +188,17 @@ class OperatorSkipRemap : public Operator { public: OperatorSkipRemap() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSkipRemap"); } + + // noncopyable + OperatorSkipRemap(const OperatorSkipRemap &) = delete; + void operator=(const OperatorSkipRemap &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSkipRemap); - bool _skip_remap = false; }; @@ -186,25 +208,29 @@ class OperatorRMHeader : public OperatorHeaders public: OperatorRMHeader() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorRMHeader"); } + // noncopyable + OperatorRMHeader(const OperatorRMHeader &) = delete; + void operator=(const OperatorRMHeader &) = delete; + protected: void exec(const Resources &res) const override; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorRMHeader); }; class OperatorAddHeader : public OperatorHeaders { public: OperatorAddHeader() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorAddHeader"); } + + // noncopyable + OperatorAddHeader(const OperatorAddHeader &) = delete; + void operator=(const OperatorAddHeader &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorAddHeader); - Value _value; }; @@ -212,14 +238,17 @@ class OperatorSetHeader : public OperatorHeaders { public: OperatorSetHeader() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetHeader"); } + + // noncopyable + OperatorSetHeader(const OperatorSetHeader &) = delete; + void operator=(const OperatorSetHeader &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetHeader); - Value _value; }; @@ -227,14 +256,17 @@ class OperatorCounter : public Operator { public: OperatorCounter() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorCounter"); } + + // noncopyable + OperatorCounter(const OperatorCounter &) = delete; + void operator=(const OperatorCounter &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorCounter); - std::string _counter_name; int _counter = TS_ERROR; }; @@ -244,25 +276,29 @@ class OperatorRMCookie : public OperatorCookies public: OperatorRMCookie() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorRMCookie"); } + // noncopyable + OperatorRMCookie(const OperatorRMCookie &) = delete; + void operator=(const OperatorRMCookie &) = delete; + protected: void exec(const Resources &res) const override; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorRMCookie); }; class OperatorAddCookie : public OperatorCookies { public: OperatorAddCookie() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorAddCookie"); } + + // noncopyable + OperatorAddCookie(const OperatorAddCookie &) = delete; + void operator=(const OperatorAddCookie &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorAddCookie); - Value _value; }; @@ -270,14 +306,17 @@ class OperatorSetCookie : public OperatorCookies { public: OperatorSetCookie() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetCookie"); } + + // noncopyable + OperatorSetCookie(const OperatorSetCookie &) = delete; + void operator=(const OperatorSetCookie &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetCookie); - Value _value; }; @@ -297,6 +336,11 @@ class OperatorSetConnDSCP : public Operator { public: OperatorSetConnDSCP() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetConnDSCP"); } + + // noncopyable + OperatorSetConnDSCP(const OperatorSetConnDSCP &) = delete; + void operator=(const OperatorSetConnDSCP &) = delete; + void initialize(Parser &p) override; protected: @@ -304,8 +348,6 @@ class OperatorSetConnDSCP : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetConnDSCP); - Value _ds_value; }; @@ -313,6 +355,11 @@ class OperatorSetConnMark : public Operator { public: OperatorSetConnMark() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetConnMark"); } + + // noncopyable + OperatorSetConnMark(const OperatorSetConnMark &) = delete; + void operator=(const OperatorSetConnMark &) = delete; + void initialize(Parser &p) override; protected: @@ -320,8 +367,6 @@ class OperatorSetConnMark : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetConnMark); - Value _ds_value; }; @@ -329,12 +374,14 @@ class OperatorSetDebug : public Operator { public: OperatorSetDebug() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetDebug"); } + + // noncopyable + OperatorSetDebug(const OperatorSetDebug &) = delete; + void operator=(const OperatorSetDebug &) = delete; + void initialize(Parser &p) override; protected: void initialize_hooks() override; void exec(const Resources &res) const override; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetDebug); }; diff --git a/plugins/header_rewrite/parser.h b/plugins/header_rewrite/parser.h index d6f4db09d06..385c824b833 100644 --- a/plugins/header_rewrite/parser.h +++ b/plugins/header_rewrite/parser.h @@ -35,6 +35,10 @@ class Parser public: explicit Parser(const std::string &line); + // noncopyable + Parser(const Parser &) = delete; + void operator=(const Parser &) = delete; + bool empty() const { @@ -76,7 +80,6 @@ class Parser private: void preprocess(std::vector tokens); - DISALLOW_COPY_AND_ASSIGN(Parser); bool _cond; bool _empty; @@ -94,10 +97,11 @@ class SimpleTokenizer public: explicit SimpleTokenizer(const std::string &line); - const std::vector &get_tokens() const; + // noncopyable + SimpleTokenizer(const SimpleTokenizer &) = delete; + void operator=(const SimpleTokenizer &) = delete; -private: - DISALLOW_COPY_AND_ASSIGN(SimpleTokenizer); + const std::vector &get_tokens() const; protected: std::vector _tokens; diff --git a/plugins/header_rewrite/resources.h b/plugins/header_rewrite/resources.h index d9d735aaa5e..ee5e0f0e6ee 100644 --- a/plugins/header_rewrite/resources.h +++ b/plugins/header_rewrite/resources.h @@ -59,6 +59,11 @@ class Resources } ~Resources() { destroy(); } + + // noncopyable + Resources(const Resources &) = delete; + void operator=(const Resources &) = delete; + void gather(const ResourceIDs ids, TSHttpHookID hook); bool ready() const @@ -78,7 +83,6 @@ class Resources private: void destroy(); - DISALLOW_COPY_AND_ASSIGN(Resources); bool _ready = false; }; diff --git a/plugins/header_rewrite/ruleset.h b/plugins/header_rewrite/ruleset.h index c963495b458..b7d2bd2b96c 100644 --- a/plugins/header_rewrite/ruleset.h +++ b/plugins/header_rewrite/ruleset.h @@ -45,6 +45,10 @@ class RuleSet delete _oper; } + // noncopyable + RuleSet(const RuleSet &) = delete; + void operator=(const RuleSet &) = delete; + // No reason to inline these void append(RuleSet *rule); bool add_condition(Parser &p, const char *filename, int lineno); @@ -107,8 +111,6 @@ class RuleSet RuleSet *next = nullptr; // Linked list private: - DISALLOW_COPY_AND_ASSIGN(RuleSet); - Condition *_cond = nullptr; // First pre-condition (linked list) Operator *_oper = nullptr; // First operator (linked list) TSHttpHookID _hook = TS_HTTP_READ_RESPONSE_HDR_HOOK; // Which hook is this rule for diff --git a/plugins/header_rewrite/statement.h b/plugins/header_rewrite/statement.h index e5aa291f5cf..7a006ba9455 100644 --- a/plugins/header_rewrite/statement.h +++ b/plugins/header_rewrite/statement.h @@ -104,6 +104,10 @@ class Statement delete _next; } + // noncopyable + Statement(const Statement &) = delete; + void operator=(const Statement &) = delete; + // Which hook are we adding this statement to? bool set_hook(TSHttpHookID hook); TSHttpHookID @@ -152,8 +156,6 @@ class Statement Statement *_next = nullptr; // Linked list private: - DISALLOW_COPY_AND_ASSIGN(Statement); - ResourceIDs _rsrc = RSRC_NONE; bool _initialized = false; TSHttpHookID _hook = TS_HTTP_READ_RESPONSE_HDR_HOOK; diff --git a/plugins/header_rewrite/value.h b/plugins/header_rewrite/value.h index e15e67a8ec2..13821cd6d3b 100644 --- a/plugins/header_rewrite/value.h +++ b/plugins/header_rewrite/value.h @@ -43,6 +43,10 @@ class Value : Statement virtual ~Value(); + // noncopyable + Value(const Value &) = delete; + void operator=(const Value &) = delete; + void set_value(const std::string &val); void @@ -88,8 +92,6 @@ class Value : Statement } private: - DISALLOW_COPY_AND_ASSIGN(Value); - int _int_value = 0; double _float_value = 0.0; std::string _value; From 00408ff6112567c63ba9fb90aa0c430d39559bc8 Mon Sep 17 00:00:00 2001 From: dyrock Date: Fri, 4 Jan 2019 22:40:07 +0000 Subject: [PATCH 116/526] Added null value init for VConn user_args. --- iocore/eventsystem/I_VConnection.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iocore/eventsystem/I_VConnection.h b/iocore/eventsystem/I_VConnection.h index 5c5b083f18d..62f604155f9 100644 --- a/iocore/eventsystem/I_VConnection.h +++ b/iocore/eventsystem/I_VConnection.h @@ -406,7 +406,7 @@ class AnnotatedVConnection : public VConnection }; protected: - std::array user_args; + std::array user_args{nullptr}; }; struct DummyVConnection : public AnnotatedVConnection { From bbe91a0fe80b23da8369b114ba947c40b0cb8c42 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Fri, 4 Jan 2019 15:05:20 +0000 Subject: [PATCH 117/526] Add addreess checks to cert selection test. --- .../tls/tls_check_cert_selection.test.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/gold_tests/tls/tls_check_cert_selection.test.py b/tests/gold_tests/tls/tls_check_cert_selection.test.py index 01684d69ffc..b029af5cde0 100644 --- a/tests/gold_tests/tls/tls_check_cert_selection.test.py +++ b/tests/gold_tests/tls/tls_check_cert_selection.test.py @@ -51,7 +51,7 @@ 'map / https://foo.com:{1}'.format(ts.Variables.ssl_port, server.Variables.Port)) ts.Disk.ssl_multicert_config.AddLines([ - 'ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key', + 'dest_ip=127.0.0.1 ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key', 'ssl_cert_name=signed2-bar.pem ssl_key_name=signed-bar.key', 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' ]) @@ -104,3 +104,29 @@ tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") tr2.TimeOut = 5 +# Should receive random.server.com +tr2 = Test.AddTestRun("random.server.com cert") +tr2.Processes.Default.Command = "curl -v -k --resolve 'random.server.com:{0}:127.0.0.1' https://random.server.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=random.server.com", "Cert should contain random.server.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=foo.com", "Cert should not contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") + +# No SNI match should match specific IP address, foo.com +# SNI name and returned cert name will not match, so must use -k to avoid cert verification +tr2 = Test.AddTestRun("Bad SNI") +tr2.Processes.Default.Command = "curl -v -k --cacert signer.pem --resolve 'bad.sni.com:{0}:127.0.0.1' https://bad.sni.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=foo.com", "Cert should contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") + From c9efb666c062cd4674c2d09804596b2a391168bf Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 31 Dec 2018 11:17:30 -0700 Subject: [PATCH 118/526] Tries to enforce .git installation of pre-commit --- Makefile.am | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Makefile.am b/Makefile.am index 2705e0c24bd..188b65357e9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -42,6 +42,9 @@ EXTRA_DIST=CHANGES INSTALL STATUS NOTICE LAYOUT LICENSE example contrib README-E # Default value when creating release candidates RC = 0 +# Make sure the git pre-commit hook gets installed on build time +all-local: $(abs_top_srcdir)/.git/hooks/pre-commit + # igalic can't remember if this is make check or make test and neither should you. test: check @@ -91,6 +94,10 @@ examples: all install-examples: examples @cd example && $(MAKE) $(AM_MAKEFLAGS) install pkglibdir=$(pkglibexecdir) +$(abs_top_srcdir)/.git/hooks/pre-commit: $(abs_top_srcdir)/tools/git/pre-commit + [ -d $(abs_top_srcdir)/.git/hooks ] && \ + cp $(abs_top_srcdir)/tools/git/pre-commit $(abs_top_srcdir)/.git/hooks/pre-commit + install-data-hook: if BUILD_DOCS @cd doc && $(MAKE) $(AM_MAKEFLAGS) install-man From ce6caf95ccf4abcff1d67c21488e55fee30f83e3 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Mon, 17 Dec 2018 20:36:17 +0000 Subject: [PATCH 119/526] don't start a new self scheduling config handler on each config reload --- plugins/regex_revalidate/regex_revalidate.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/regex_revalidate/regex_revalidate.c b/plugins/regex_revalidate/regex_revalidate.c index 4b009df2b8b..0a697d73588 100644 --- a/plugins/regex_revalidate/regex_revalidate.c +++ b/plugins/regex_revalidate/regex_revalidate.c @@ -19,8 +19,7 @@ limitations under the License. */ -#include "tscore/ink_defs.h" -#include "tscore/ink_platform.h" +#include #include #include @@ -32,7 +31,6 @@ #include #include #include -#include #ifdef HAVE_PCRE_PCRE_H #include @@ -331,7 +329,7 @@ list_config(plugin_state_t *pstate, invalidate_t *i) } static int -free_handler(TSCont cont, TSEvent event ATS_UNUSED, void *edata ATS_UNUSED) +free_handler(TSCont cont, TSEvent event, void *edata) { invalidate_t *iptr; @@ -343,7 +341,7 @@ free_handler(TSCont cont, TSEvent event ATS_UNUSED, void *edata ATS_UNUSED) } static int -config_handler(TSCont cont, TSEvent event ATS_UNUSED, void *edata ATS_UNUSED) +config_handler(TSCont cont, TSEvent event, void *edata) { plugin_state_t *pstate; invalidate_t *i, *iptr; @@ -379,7 +377,10 @@ config_handler(TSCont cont, TSEvent event ATS_UNUSED, void *edata ATS_UNUSED) TSMutexUnlock(mutex); - TSContSchedule(cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); + // Don't reschedule for TS_EVENT_MGMT_UPDATE + if (event == TS_EVENT_TIMEOUT) { + TSContSchedule(cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); + } return 0; } From 8bbfc5927f960af412ee4e11ba1deea3ae4d4c9a Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Fri, 21 Dec 2018 15:22:25 -0600 Subject: [PATCH 120/526] re-ordering event numbers --- doc/developer-guide/api/types/TSEvent.en.rst | 5 +- include/ts/apidefs.h.in | 54 ++++++++++++-------- src/traffic_server/InkAPITest.cc | 2 +- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/doc/developer-guide/api/types/TSEvent.en.rst b/doc/developer-guide/api/types/TSEvent.en.rst index 96e8b735e8c..3104c1392d0 100644 --- a/doc/developer-guide/api/types/TSEvent.en.rst +++ b/doc/developer-guide/api/types/TSEvent.en.rst @@ -171,12 +171,12 @@ Enumeration Members .. c:macro:: TS_EVENT_LIFECYCLE_CACHE_READY -.. c:macro:: TS_EVENT_LIFECYCLE_MSG - .. c:macro:: TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED .. c:macro:: TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED +.. c:macro:: TS_EVENT_LIFECYCLE_MSG + .. c:macro:: TS_EVENT_VCONN_START .. c:macro:: TS_EVENT_VCONN_CLOSE @@ -213,4 +213,3 @@ These are the event types used to drive continuations in the event system. .. cpp:var:: EventType EVENT_IMMEDIATE See :c:macro:`EVENT_IMMEDIATE`. - diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 76189d1eb6b..bb3851f580d 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -334,12 +334,6 @@ typedef enum { Event: TS_EVENT_LIFECYCLE_CACHE_READY - TS_LIFECYCLE_MSG_HOOK - - Called in response to an external agent. The data is a pointer to an instance of TSPluginMsg. - - Event: TS_EVENT_LIFECYCLE_MSG - TS_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED_HOOK called every time after a server SSL_CTX has finished the initialization. @@ -354,6 +348,12 @@ typedef enum { Event: TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED + TS_LIFECYCLE_MSG_HOOK + + Called in response to an external agent. The data is a pointer to an instance of TSPluginMsg. + + Event: TS_EVENT_LIFECYCLE_MSG + TS_LIFECYCLE_TASK_THREADS_READY_HOOK called once, after the task threads have been started. @@ -398,6 +398,7 @@ typedef enum { TS_EVENT_TIMEOUT = 2, TS_EVENT_ERROR = 3, TS_EVENT_CONTINUE = 4, + TS_EVENT_VCONN_READ_READY = 100, TS_EVENT_VCONN_WRITE_READY = 101, TS_EVENT_VCONN_READ_COMPLETE = 102, @@ -405,10 +406,17 @@ typedef enum { TS_EVENT_VCONN_EOS = 104, TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, + TS_EVENT_VCONN_START = 107, + TS_EVENT_VCONN_CLOSE = 108, + TS_EVENT_VCONN_OUTBOUND_START = 109, + TS_EVENT_VCONN_OUTBOUND_CLOSE = 110, + TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible + TS_EVENT_NET_CONNECT = 200, TS_EVENT_NET_CONNECT_FAILED = 201, TS_EVENT_NET_ACCEPT = 202, TS_EVENT_NET_ACCEPT_FAILED = 204, + TS_EVENT_INTERNAL_206 = 206, TS_EVENT_INTERNAL_207 = 207, TS_EVENT_INTERNAL_208 = 208, @@ -416,7 +424,9 @@ typedef enum { TS_EVENT_INTERNAL_210 = 210, TS_EVENT_INTERNAL_211 = 211, TS_EVENT_INTERNAL_212 = 212, + TS_EVENT_HOST_LOOKUP = 500, + TS_EVENT_CACHE_OPEN_READ = 1102, TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, TS_EVENT_CACHE_OPEN_WRITE = 1108, @@ -439,11 +449,15 @@ typedef enum { TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, TS_EVENT_CACHE_READ_READY = 1134, TS_EVENT_CACHE_READ_COMPLETE = 1135, + TS_EVENT_INTERNAL_1200 = 1200, + TS_EVENT_SSL_SESSION_GET = 2000, TS_EVENT_SSL_SESSION_NEW = 2001, TS_EVENT_SSL_SESSION_REMOVE = 2002, + TS_EVENT_AIO_DONE = 3900, + TS_EVENT_HTTP_CONTINUE = 60000, TS_EVENT_HTTP_ERROR = 60001, TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, @@ -462,19 +476,17 @@ typedef enum { TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, TS_EVENT_HTTP_PRE_REMAP = 60016, TS_EVENT_HTTP_POST_REMAP = 60017, - TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60018, - TS_EVENT_LIFECYCLE_PORTS_READY = 60019, - TS_EVENT_LIFECYCLE_CACHE_READY = 60020, - TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60021, - TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60022, - TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60027, - TS_EVENT_LIFECYCLE_SHUTDOWN = 60028, - TS_EVENT_VCONN_START = 60023, - TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible - TS_EVENT_VCONN_CLOSE = 60026, - TS_EVENT_LIFECYCLE_MSG = 60024, - TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60025, - TS_EVENT_MGMT_UPDATE = 60100, + TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60018, + + TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60100, + TS_EVENT_LIFECYCLE_PORTS_READY = 60101, + TS_EVENT_LIFECYCLE_CACHE_READY = 60102, + TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60103, + TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60104, + TS_EVENT_LIFECYCLE_MSG = 60105, + TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60106, + TS_EVENT_LIFECYCLE_SHUTDOWN = 60107, + TS_EVENT_INTERNAL_60200 = 60200, TS_EVENT_INTERNAL_60201 = 60201, TS_EVENT_INTERNAL_60202 = 60202, @@ -482,8 +494,8 @@ typedef enum { TS_EVENT_SSL_SERVERNAME = 60204, TS_EVENT_SSL_VERIFY_SERVER = 60205, TS_EVENT_SSL_VERIFY_CLIENT = 60206, - TS_EVENT_VCONN_OUTBOUND_START = 60207, - TS_EVENT_VCONN_OUTBOUND_CLOSE = 60208 + + TS_EVENT_MGMT_UPDATE = 60300 } TSEvent; #define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */ diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 9f7710fc1bb..6d38500766f 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -6682,7 +6682,7 @@ typedef enum { ORIG_TS_EVENT_HTTP_SSN_CLOSE = 60014, ORIG_TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, - ORIG_TS_EVENT_MGMT_UPDATE = 60100 + ORIG_TS_EVENT_MGMT_UPDATE = 60300 } ORIG_TSEvent; typedef enum { From 97b1db7fd997937c884560a70d04c5cfecc1a8a8 Mon Sep 17 00:00:00 2001 From: scw00 Date: Mon, 7 Jan 2019 19:15:23 +0800 Subject: [PATCH 121/526] Fixes Clang-Analyzer issue of H2 Dependence Tree --- proxy/http2/Http2DependencyTree.h | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/proxy/http2/Http2DependencyTree.h b/proxy/http2/Http2DependencyTree.h index 7b71303b113..e7820d50341 100644 --- a/proxy/http2/Http2DependencyTree.h +++ b/proxy/http2/Http2DependencyTree.h @@ -119,10 +119,10 @@ template class Tree Node *find(uint32_t id, bool *is_max_leaf = nullptr); Node *find_shadow(uint32_t id, bool *is_max_leaf = nullptr); Node *add(uint32_t parent_id, uint32_t id, uint32_t weight, bool exclusive, T t, bool shadow = false); - void remove(Node *node); - void reprioritize(uint32_t new_parent_id, uint32_t id, bool exclusive); - void reprioritize(Node *node, uint32_t id, bool exclusive); + Node *reprioritize(uint32_t new_parent_id, uint32_t id, bool exclusive); + Node *reprioritize(Node *node, uint32_t id, bool exclusive); Node *top(); + void remove(Node *node); void activate(Node *node); void deactivate(Node *node, uint32_t sent); void update(Node *node, uint32_t sent); @@ -269,7 +269,7 @@ Tree::add(uint32_t parent_id, uint32_t id, uint32_t weight, bool exclusive, T node->weight = weight; node->shadow = false; // Move the shadow node into the proper position in the tree - reprioritize(node, parent_id, exclusive); + node = reprioritize(node, parent_id, exclusive); return node; } @@ -389,36 +389,36 @@ Tree::remove(Node *node) } template -void +Node * Tree::reprioritize(uint32_t id, uint32_t new_parent_id, bool exclusive) { Node *node = find(id); if (node == nullptr) { - return; + return node; } - reprioritize(node, new_parent_id, exclusive); + return reprioritize(node, new_parent_id, exclusive); } template -void +Node * Tree::reprioritize(Node *node, uint32_t new_parent_id, bool exclusive) { if (node == nullptr) { - return; + return node; } Node *old_parent = node->parent; if (old_parent->id == new_parent_id) { // Do nothing - return; + return node; } // should not change the root node ink_assert(node->parent); Node *new_parent = find(new_parent_id); if (new_parent == nullptr) { - return; + return node; } // If node is dependent on the new parent, must move the new parent first if (new_parent_id != 0 && in_parent_chain(node, new_parent)) { @@ -429,7 +429,10 @@ Tree::reprioritize(Node *node, uint32_t new_parent_id, bool exclusive) // delete the shadow node if (node->is_shadow() && node->children.empty() && node->queue->empty()) { remove(node); + return nullptr; } + + return node; } template From cf1426b612a656135388804f22dfeb5fe8864d81 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Fri, 21 Dec 2018 12:33:28 +0900 Subject: [PATCH 122/526] Make H2 stream counters atomic --- proxy/http2/Http2ConnectionState.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index ef94a933a5d..c2eea39025f 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -23,6 +23,7 @@ #pragma once +#include #include "HTTP2.h" #include "HPACK.h" #include "Http2Stream.h" @@ -304,16 +305,16 @@ class Http2ConnectionState : public Continuation Queue stream_list; Http2StreamId latest_streamid_in = 0; Http2StreamId latest_streamid_out = 0; - int stream_requests = 0; + std::atomic stream_requests = 0; // Counter for current active streams which is started by client - uint32_t client_streams_in_count = 0; + std::atomic client_streams_in_count = 0; // Counter for current acive streams which is started by server - uint32_t client_streams_out_count = 0; + std::atomic client_streams_out_count = 0; // Counter for current active streams and streams in the process of shutting down - uint32_t total_client_streams_count = 0; + std::atomic total_client_streams_count = 0; // NOTE: Id of stream which MUST receive CONTINUATION frame. // - [RFC 7540] 6.2 HEADERS From f516ff88f955f28af2dc296d670e4e77fbe27d5b Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Fri, 21 Dec 2018 12:06:25 +0900 Subject: [PATCH 123/526] Acquire a lock before checking H2ConnectionState::ua_session --- proxy/http2/Http2ConnectionState.cc | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index a7cf6e738da..4df6e856a4e 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -1242,13 +1242,9 @@ Http2ConnectionState::release_stream(Http2Stream *stream) --total_client_streams_count; } - if (ua_session) { - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - if (!ua_session) { - // Workaround fix for GitHub #4504. The `ua_session` could be freed while waiting for acquiring the above lock. - return; - } - + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + if (this->ua_session) { + ink_assert(this->mutex == ua_session->mutex); // If the number of clients is 0 and ua_session is active, then mark the connection as inactive if (total_client_streams_count == 0 && ua_session->is_active()) { ua_session->clear_session_active(); From fb513abe68dfe95aaf3d972d7b4d20f40ef27382 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 6 Dec 2018 23:18:47 +0000 Subject: [PATCH 124/526] Allow for override of ssl.client files --- doc/admin-guide/files/records.config.en.rst | 5 + .../functions/TSHttpOverridableConfig.en.rst | 7 +- include/ts/apidefs.h.in | 3 + iocore/net/I_NetVConnection.h | 31 ++-- iocore/net/P_SSLConfig.h | 8 +- iocore/net/P_SSLSNI.h | 2 +- iocore/net/P_UnixNetVConnection.h | 8 +- iocore/net/SSLClientUtils.cc | 37 ----- iocore/net/SSLConfig.cc | 132 +++++++++++----- iocore/net/SSLNetVConnection.cc | 20 ++- iocore/net/SSLSNIConfig.cc | 5 +- iocore/net/SSLUtils.cc | 3 +- mgmt/RecordsConfig.cc | 4 +- plugins/lua/ts_lua_http_config.c | 4 + proxy/http/HttpConfig.h | 15 +- proxy/http/HttpSM.cc | 3 + src/traffic_server/InkAPI.cc | 45 +++--- src/traffic_server/InkAPITest.cc | 4 +- .../tls/tls_client_cert_override.test.py | 145 ++++++++++++++++++ .../tls/tls_verify_ca_override.test.py | 137 +++++++++++++++++ 20 files changed, 489 insertions(+), 129 deletions(-) create mode 100644 tests/gold_tests/tls/tls_client_cert_override.test.py create mode 100644 tests/gold_tests/tls/tls_verify_ca_override.test.py diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 34d9e3a7635..e2443846b44 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3508,6 +3508,7 @@ Client-Related Configuration .. ts:cv:: CONFIG proxy.config.ssl.client.cert.filename STRING NULL :reloadable: + :overridable: The filename of SSL client certificate installed on |TS|. @@ -3519,6 +3520,7 @@ Client-Related Configuration .. ts:cv:: CONFIG proxy.config.ssl.client.private_key.filename STRING NULL :reloadable: + :overridable: The filename of the |TS| private key. Change this variable only if the private key is not located in the |TS| SSL @@ -3532,11 +3534,14 @@ Client-Related Configuration file. .. ts:cv:: CONFIG proxy.config.ssl.client.CA.cert.filename STRING NULL + :reloadable: + :overridable: The filename of the certificate authority against which the origin server will be verified. .. ts:cv:: CONFIG proxy.config.ssl.client.CA.cert.path STRING NULL + :reloadable: Specifies the location of the certificate authority file against which the origin server will be verified. diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index 91eaaff7d8a..a38fd566786 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -152,7 +152,7 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED` :ts:cv:`proxy.config.http.post.check.content_length.enabled` :c:macro:`TS_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT` :ts:cv:`proxy.config.http.post_connect_attempts_timeout` :c:macro:`TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY` :ts:cv:`proxy.config.http.redirect_use_orig_cache_key` -TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED proxy.config.http.request_buffer_enabled +:c:macro:`TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED` :ts:cv:`proxy.config.http.request_buffer_enabled` :c:macro:`TS_CONFIG_HTTP_REQUEST_HEADER_MAX_SIZE` :ts:cv:`proxy.config.http.request_header_max_size` :c:macro:`TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE` :ts:cv:`proxy.config.http.response_header_max_size` :c:macro:`TS_CONFIG_HTTP_RESPONSE_SERVER_ENABLED` :ts:cv:`proxy.config.http.response_server_enabled` @@ -184,6 +184,11 @@ TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED proxy.config :c:macro:`TS_CONFIG_URL_REMAP_PRISTINE_HOST_HDR` :ts:cv:`proxy.config.url_remap.pristine_host_hdr` :c:macro:`TS_CONFIG_WEBSOCKET_ACTIVE_TIMEOUT` :ts:cv:`proxy.config.websocket.active_timeout` :c:macro:`TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT` :ts:cv:`proxy.config.websocket.no_activity_timeout` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY` :ts:cv:`proxy.config.ssl.client.verify.server.policy` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES` :ts:cv:`proxy.config.ssl.client.verify.server.properties` +:c:macro:`TS_CONFIG_SSL_CLIENT_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` +:c:macro:`TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME` :ts:cv:`proxy.config.ssl.client.private_key.filename` +:c:macro:`TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.CA.cert.filename` ================================================================== ==================================================================== Examples diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index bb3851f580d..5e7bf7c3cac 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -789,6 +789,7 @@ typedef enum { TS_CONFIG_SRV_ENABLED, TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD, TS_CONFIG_SSL_CERT_FILENAME, + TS_CONFIG_SSL_CLIENT_CERT_FILENAME = TS_CONFIG_SSL_CERT_FILENAME, TS_CONFIG_SSL_CERT_FILEPATH, TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB, TS_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER, @@ -814,6 +815,8 @@ typedef enum { TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, TS_CONFIG_SSL_CLIENT_SNI_POLICY, + TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME, + TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME, TS_CONFIG_LAST_ENTRY } TSOverridableConfigKey; diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index e9b0427a429..1a8d6013126 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -186,7 +186,20 @@ struct NetVCOptions { /** * Client certificate to use in response to OS's certificate request */ - ats_scoped_str clientCertificate; + const char *ssl_client_cert_name = nullptr; + /* + * File containing private key matching certificate + */ + const char *ssl_client_private_key_name = nullptr; + /* + * File containing CA certs for verifying origin's cert + */ + const char *ssl_client_ca_cert_name = nullptr; + /* + * Directory containing CA certs for verifying origin's cert + */ + const char *ssl_client_ca_cert_path = nullptr; + /// Reset all values to defaults. /** @@ -233,13 +246,6 @@ struct NetVCOptions { } return *this; } - self & - set_client_certname(const char *name) - { - clientCertificate = ats_strdup(name); - // clientCertificate = name; - return *this; - } self & operator=(self const &that) @@ -255,9 +261,8 @@ struct NetVCOptions { * memcpy removes the extra reference to that's copy of the string * Removing the release will eventualy cause a double free crash */ - sni_servername = nullptr; // release any current name. - ssl_servername = nullptr; - clientCertificate = nullptr; + sni_servername = nullptr; // release any current name. + ssl_servername = nullptr; memcpy(static_cast(this), &that, sizeof(self)); if (that.sni_servername) { sni_servername.release(); // otherwise we'll free the source string. @@ -267,10 +272,6 @@ struct NetVCOptions { ssl_servername.release(); // otherwise we'll free the source string. this->ssl_servername = ats_strdup(that.ssl_servername); } - if (that.clientCertificate) { - clientCertificate.release(); // otherwise we'll free the source string. - this->clientCertificate = ats_strdup(that.clientCertificate); - } } return *this; } diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 3bccf9f1516..5c93abbbd8f 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -80,7 +80,9 @@ struct SSLConfigParams : public ConfigInfo { int ssl_session_cache_auto_clear; char *clientCertPath; + char *clientCertPathOnly; char *clientKeyPath; + char *clientKeyPathOnly; char *clientCACertFilename; char *clientCACertPath; YamlSNIConfig::Policy verifyServerPolicy; @@ -115,11 +117,15 @@ struct SSLConfigParams : public ConfigInfo { SSL_CTX *client_ctx; + // Making this mutable since this is a updatable + // cache on an otherwise immutable config object + // The ctx_map owns the client SSL_CTX objects and is responseible for cleaning them up mutable std::unordered_map ctx_map; mutable ink_mutex ctxMapLock; SSL_CTX *getClientSSL_CTX(void) const; - SSL_CTX *getNewCTX(const char *client_cert, const char *key_file) const; + SSL_CTX *getCTX(const char *client_cert, const char *key_file, const char *ca_bundle_file, const char *ca_bundle_path) const; + void cleanupCTXTable(); void initialize(); void cleanup(); diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index a1d75045417..220de51736b 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -46,7 +46,7 @@ struct NextHopProperty { YamlSNIConfig::Policy verifyServerPolicy = YamlSNIConfig::Policy::UNSET; // whether to verify the next hop YamlSNIConfig::Property verifyServerProperties = YamlSNIConfig::Property::UNSET; // what to verify on the next hop SSL_CTX *ctx = nullptr; // ctx generated off the certificate to present to this server - NextHopProperty(); + NextHopProperty() {} }; using actionVector = std::vector>; diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 1516ddda2f8..0a320319b1c 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -66,9 +66,11 @@ NetVCOptions::reset() etype = ET_NET; - sni_servername = nullptr; - ssl_servername = nullptr; - clientCertificate = nullptr; + sni_servername = nullptr; + ssl_servername = nullptr; + ssl_client_cert_name = nullptr; + ssl_client_private_key_name = nullptr; + ssl_client_ca_cert_name = nullptr; } TS_INLINE void diff --git a/iocore/net/SSLClientUtils.cc b/iocore/net/SSLClientUtils.cc index 7f0ceb2875c..89842f543fd 100644 --- a/iocore/net/SSLClientUtils.cc +++ b/iocore/net/SSLClientUtils.cc @@ -143,7 +143,6 @@ SSLInitClientContext(const SSLConfigParams *params) { ink_ssl_method_t meth = nullptr; SSL_CTX *client_ctx = nullptr; - char *clientKeyPtr = nullptr; // Note that we do not call RAND_seed() explicitly here, we depend on OpenSSL // to do the seeding of the PRNG for us. This is the case for all platforms that @@ -183,44 +182,8 @@ SSLInitClientContext(const SSLConfigParams *params) } #endif - // if no path is given for the client private key, - // assume it is contained in the client certificate file. - clientKeyPtr = params->clientKeyPath; - if (clientKeyPtr == nullptr) { - clientKeyPtr = params->clientCertPath; - } - - if (params->clientCertPath != nullptr && params->clientCertPath[0] != '\0') { - if (!SSL_CTX_use_certificate_chain_file(client_ctx, params->clientCertPath)) { - SSLError("failed to load client certificate from %s", params->clientCertPath); - goto fail; - } - - if (!SSL_CTX_use_PrivateKey_file(client_ctx, clientKeyPtr, SSL_FILETYPE_PEM)) { - SSLError("failed to load client private key file from %s", clientKeyPtr); - goto fail; - } - - if (!SSL_CTX_check_private_key(client_ctx)) { - SSLError("client private key (%s) does not match the certificate public key (%s)", clientKeyPtr, params->clientCertPath); - goto fail; - } - } - SSL_CTX_set_verify(client_ctx, SSL_VERIFY_PEER, verify_callback); SSL_CTX_set_verify_depth(client_ctx, params->client_verify_depth); - - if (params->clientCACertFilename != nullptr || params->clientCACertPath != nullptr) { - if (!SSL_CTX_load_verify_locations(client_ctx, params->clientCACertFilename, params->clientCACertPath)) { - SSLError("invalid client CA Certificate file (%s) or CA Certificate path (%s)", params->clientCACertFilename, - params->clientCACertPath); - goto fail; - } - } else if (!SSL_CTX_set_default_verify_paths(client_ctx)) { - SSLError("failed to set the default verify paths"); - goto fail; - } - if (SSLConfigParams::init_ssl_ctx_cb) { SSLConfigParams::init_ssl_ctx_cb(client_ctx, false); } diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index a6dbed7eebb..2cdeca0ad63 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -89,12 +89,12 @@ SSLConfigParams::reset() { serverCertPathOnly = serverCertChainFilename = configFilePath = serverCACertFilename = serverCACertPath = clientCertPath = clientKeyPath = clientCACertFilename = clientCACertPath = cipherSuite = client_cipherSuite = dhparamsFile = serverKeyPathOnly = - nullptr; - server_tls13_cipher_suites = nullptr; - client_tls13_cipher_suites = nullptr; - server_groups_list = nullptr; - client_groups_list = nullptr; - client_ctx = nullptr; + clientKeyPathOnly = clientCertPathOnly = nullptr; + server_tls13_cipher_suites = nullptr; + client_tls13_cipher_suites = nullptr; + server_groups_list = nullptr; + client_groups_list = nullptr; + client_ctx = nullptr; clientCertLevel = client_verify_depth = verify_depth = 0; verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; verifyServerProperties = YamlSNIConfig::Property::NONE; @@ -116,7 +116,9 @@ SSLConfigParams::cleanup() serverCACertFilename = (char *)ats_free_null(serverCACertFilename); serverCACertPath = (char *)ats_free_null(serverCACertPath); clientCertPath = (char *)ats_free_null(clientCertPath); + clientCertPathOnly = (char *)ats_free_null(clientCertPathOnly); clientKeyPath = (char *)ats_free_null(clientKeyPath); + clientKeyPathOnly = (char *)ats_free_null(clientKeyPathOnly); clientCACertFilename = (char *)ats_free_null(clientCACertFilename); clientCACertPath = (char *)ats_free_null(clientCACertPath); configFilePath = (char *)ats_free_null(configFilePath); @@ -131,7 +133,7 @@ SSLConfigParams::cleanup() server_groups_list = (char *)ats_free_null(server_groups_list); client_groups_list = (char *)ats_free_null(client_groups_list); - SSLReleaseContext(client_ctx); + cleanupCTXTable(); reset(); } @@ -410,14 +412,14 @@ SSLConfigParams::initialize() REC_ReadConfigStringAlloc(ssl_client_cert_filename, "proxy.config.ssl.client.cert.filename"); REC_ReadConfigStringAlloc(ssl_client_cert_path, "proxy.config.ssl.client.cert.path"); if (ssl_client_cert_filename && ssl_client_cert_path) { - set_paths_helper(ssl_client_cert_path, ssl_client_cert_filename, nullptr, &clientCertPath); + set_paths_helper(ssl_client_cert_path, ssl_client_cert_filename, &clientCertPathOnly, &clientCertPath); } ats_free_null(ssl_client_cert_filename); ats_free_null(ssl_client_cert_path); REC_ReadConfigStringAlloc(ssl_client_private_key_filename, "proxy.config.ssl.client.private_key.filename"); REC_ReadConfigStringAlloc(ssl_client_private_key_path, "proxy.config.ssl.client.private_key.path"); - set_paths_helper(ssl_client_private_key_path, ssl_client_private_key_filename, nullptr, &clientKeyPath); + set_paths_helper(ssl_client_private_key_path, ssl_client_private_key_filename, &clientKeyPathOnly, &clientKeyPath); ats_free_null(ssl_client_private_key_filename); ats_free_null(ssl_client_private_key_path); @@ -434,41 +436,12 @@ SSLConfigParams::initialize() // Enable client regardless of config file settings as remap file // can cause HTTP layer to connect using SSL. But only if SSL // initialization hasn't failed already. - client_ctx = SSLInitClientContext(this); + client_ctx = this->getCTX(this->clientCertPath, this->clientKeyPath, this->clientCACertFilename, this->clientCACertPath); if (!client_ctx) { SSLError("Can't initialize the SSL client, HTTPS in remap rules will not function"); } } -// creates a new context attaching the provided certificate -SSL_CTX * -SSLConfigParams::getNewCTX(const char *client_cert, const char *client_key) const -{ - SSL_CTX *nclient_ctx = nullptr; - nclient_ctx = SSLInitClientContext(this); - if (!nclient_ctx) { - SSLError("Can't initialize the SSL client, HTTPS in remap rules will not function"); - return nullptr; - } - if (client_cert != nullptr && client_cert[0] != '\0') { - if (!SSL_CTX_use_certificate_chain_file(nclient_ctx, (const char *)client_cert)) { - SSLError("failed to load client certificate from %s", this->clientCertPath); - SSLReleaseContext(nclient_ctx); - return nullptr; - } - } - // If there is not private key specified, perhaps it is in the file with the cert - if (client_key == nullptr || client_key[0] == '\0') { - client_key = client_cert; - } - // Try loading the private key - if (client_key != nullptr && client_key[0] != '\0') { - // If it failed, then we are just going to use the previously set private key from records.config - SSL_CTX_use_PrivateKey_file(nclient_ctx, client_key, SSL_FILETYPE_PEM); - } - return nclient_ctx; -} - SSL_CTX * SSLConfigParams::getClientSSL_CTX() const { @@ -682,3 +655,84 @@ SSLTicketParams::cleanup() ticket_block_free(default_global_keyblock); ticket_key_filename = (char *)ats_free_null(ticket_key_filename); } + +SSL_CTX * +SSLConfigParams::getCTX(const char *client_cert, const char *key_file, const char *ca_bundle_file, const char *ca_bundle_path) const +{ + SSL_CTX *client_ctx = nullptr; + std::string key; + ts::bwprint(key, "{}:{}:{}:{}", client_cert, key_file, ca_bundle_file, ca_bundle_path); + + ink_mutex_acquire(&ctxMapLock); + auto iter = ctx_map.find(key); + if (iter != ctx_map.end()) { + client_ctx = iter->second; + ink_mutex_release(&ctxMapLock); + return client_ctx; + } + ink_mutex_release(&ctxMapLock); + + // Not yet in the table. Make the cert and add it to the table + client_ctx = SSLInitClientContext(this); + + if (client_cert) { + // Set public and private keys + if (!SSL_CTX_use_certificate_chain_file(client_ctx, client_cert)) { + SSLError("failed to load client certificate from %s", client_cert); + goto fail; + } + if (!key_file || key_file[0] == '\0') { + key_file = client_cert; + } + if (!SSL_CTX_use_PrivateKey_file(client_ctx, key_file, SSL_FILETYPE_PEM)) { + SSLError("failed to load client private key file from %s", key_file); + goto fail; + } + + if (!SSL_CTX_check_private_key(client_ctx)) { + SSLError("client private key (%s) does not match the certificate public key (%s)", key_file, client_cert); + goto fail; + } + } + + // Set CA information for verifying peer cert + if (ca_bundle_file != nullptr || ca_bundle_path != nullptr) { + if (!SSL_CTX_load_verify_locations(client_ctx, ca_bundle_file, ca_bundle_path)) { + SSLError("invalid client CA Certificate file (%s) or CA Certificate path (%s)", ca_bundle_file, ca_bundle_path); + goto fail; + } + } else if (!SSL_CTX_set_default_verify_paths(client_ctx)) { + SSLError("failed to set the default verify paths"); + goto fail; + } + + ink_mutex_acquire(&ctxMapLock); + iter = ctx_map.find(key); + if (iter != ctx_map.end()) { + SSL_CTX_free(client_ctx); + client_ctx = iter->second; + } else { + ctx_map.insert(std::make_pair(key, client_ctx)); + } + ink_mutex_release(&ctxMapLock); + return client_ctx; + +fail: + if (client_ctx) { + SSL_CTX_free(client_ctx); + } + return nullptr; +} + +void +SSLConfigParams::cleanupCTXTable() +{ + ink_mutex_acquire(&ctxMapLock); + auto iter = ctx_map.begin(); + while (iter != ctx_map.end()) { + SSL_CTX_free(iter->second); + ++iter; + } + ctx_map.clear(); + ink_mutex_release(&ctxMapLock); +} diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 96221d49696..1e5dccf8cc2 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -22,6 +22,7 @@ */ #include "tscore/ink_config.h" #include "tscore/EventNotify.h" +#include "tscore/I_Layout.h" #include "records/I_RecHttp.h" #include "P_Net.h" #include "P_SSLNextProtocolSet.h" @@ -970,7 +971,24 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) auto nps = sniParam->getPropertyConfig(serverKey); SSL_CTX *clientCTX = nullptr; - if (nps) { + // First Look to see if there are override parameters + if (options.ssl_client_cert_name) { + std::string certFilePath = Layout::get()->relative_to(params->clientCertPathOnly, options.ssl_client_cert_name); + std::string keyFilePath; + if (options.ssl_client_private_key_name) { + keyFilePath = Layout::get()->relative_to(params->clientKeyPathOnly, options.ssl_client_private_key_name); + } + std::string caCertFilePath; + if (options.ssl_client_ca_cert_name) { + caCertFilePath = Layout::get()->relative_to(params->clientCACertPath, options.ssl_client_ca_cert_name); + } + clientCTX = + params->getCTX(certFilePath.c_str(), keyFilePath.empty() ? nullptr : keyFilePath.c_str(), + caCertFilePath.empty() ? params->clientCACertFilename : caCertFilePath.c_str(), params->clientCACertPath); + } else if (options.ssl_client_ca_cert_name) { + std::string caCertFilePath = Layout::get()->relative_to(params->clientCACertPath, options.ssl_client_ca_cert_name); + clientCTX = params->getCTX(params->clientCertPath, params->clientKeyPath, caCertFilePath.c_str(), params->clientCACertPath); + } else if (nps) { clientCTX = nps->ctx; } else { // Just stay with the values passed down from the SM for verify clientCTX = params->client_ctx; diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 524d2ab9228..470a54419ce 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -41,7 +41,6 @@ static ConfigUpdateHandler *sniConfigUpdate; struct NetAccept; std::unordered_map snpsMap; -NextHopProperty::NextHopProperty() {} const NextHopProperty * SNIConfigParams::getPropertyConfig(const std::string &servername) const @@ -83,8 +82,8 @@ SNIConfigParams::loadSNIConfig() auto clientCTX = params->getClientSSL_CTX(); const char *certFile = item.client_cert.data(); const char *keyFile = item.client_key.data(); - if (certFile) { - clientCTX = params->getNewCTX(certFile, keyFile); + if (certFile && certFile[0] != '\0') { + clientCTX = params->getCTX(certFile, keyFile, params->clientCACertFilename, params->clientCACertPath); } auto nps = next_hop_list.emplace(next_hop_list.end()); diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 5e53b0bdd28..d0ae28386c5 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -2289,10 +2289,11 @@ ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, } #endif /* HAVE_OPENSSL_SESSION_TICKETS */ +// Release SSL_CTX and the associated data. This works for both +// client and server contexts and gracefully accepts nullptr. void SSLReleaseContext(SSL_CTX *ctx) { - // SSL_CTX_free() does nothing if ctx in nullptr, so there's no need to check. SSL_CTX_free(ctx); } diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index e9cb40fda08..7c329f1fab0 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1150,9 +1150,9 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.client.private_key.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.filename", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.filename", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.client.sni_policy", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c index 74bc20169f3..c7f46cbd4ac 100644 --- a/plugins/lua/ts_lua_http_config.c +++ b/plugins/lua/ts_lua_http_config.c @@ -138,6 +138,8 @@ typedef enum { TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, TS_LUA_CONFIG_SSL_CLIENT_SNI_POLICY = TS_CONFIG_SSL_CLIENT_SNI_POLICY, + TS_LUA_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME = TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME, + TS_LUA_CONFIG_SSL_CLIENT_CA_CERT_FILENAME = TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME, TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY, } TSLuaOverridableConfigKey; @@ -266,6 +268,8 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_SNI_POLICY), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY), diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index f274168cc7f..59f0faf7bfa 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -552,8 +552,9 @@ struct OverridableHttpConfigParams { global_user_agent_header_size(0), cache_heuristic_lm_factor(0.10), background_fill_threshold(0.5), - client_cert_filename(nullptr), - client_cert_filepath(nullptr), + ssl_client_cert_filename(nullptr), + ssl_client_private_key_filename(nullptr), + ssl_client_ca_cert_filename(nullptr), cache_vary_default_text(nullptr), cache_vary_default_images(nullptr), cache_vary_default_other(nullptr) @@ -798,8 +799,9 @@ struct OverridableHttpConfigParams { MgmtFloat background_fill_threshold; // Various strings, good place for them here ... - char *client_cert_filename; - char *client_cert_filepath; + char *ssl_client_cert_filename; + char *ssl_client_private_key_filename; + char *ssl_client_ca_cert_filename; char *cache_vary_default_text; char *cache_vary_default_images; @@ -968,8 +970,9 @@ inline HttpConfigParams::~HttpConfigParams() ats_free(oride.body_factory_template_base); ats_free(oride.proxy_response_server_string); ats_free(oride.global_user_agent_header); - ats_free(oride.client_cert_filename); - ats_free(oride.client_cert_filepath); + ats_free(oride.ssl_client_cert_filename); + ats_free(oride.ssl_client_private_key_filename); + ats_free(oride.ssl_client_ca_cert_filename); ats_free(oride.cache_vary_default_text); ats_free(oride.cache_vary_default_images); ats_free(oride.cache_vary_default_other); diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 75095d9d698..18dad8e8ea9 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -5044,6 +5044,9 @@ HttpSM::do_http_server_open(bool raw) if (scheme_to_use == URL_WKSIDX_HTTPS || HttpTransactHeaders::is_method_idempotent(t_state.method)) { opt.f_tcp_fastopen = (t_state.txn_conf->sock_option_flag_out & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN); } + opt.ssl_client_cert_name = t_state.txn_conf->ssl_client_cert_filename; + opt.ssl_client_private_key_name = t_state.txn_conf->ssl_client_private_key_filename; + opt.ssl_client_ca_cert_name = t_state.txn_conf->ssl_client_ca_cert_filename; if (scheme_to_use == URL_WKSIDX_HTTPS) { SMDebug("http", "calling sslNetProcessor.connect_re"); diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index c537e166292..61de6059a50 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -8198,18 +8198,16 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr case TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD: ret = _memberp_to_generic(&overridableHttpConfig->forward_connect_method, conv); break; - case TS_CONFIG_SSL_CERT_FILENAME: - ret = _memberp_to_generic(&overridableHttpConfig->client_cert_filename, conv); - break; - case TS_CONFIG_SSL_CERT_FILEPATH: - ret = _memberp_to_generic(&overridableHttpConfig->client_cert_filepath, conv); - break; case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER: ret = _memberp_to_generic(&overridableHttpConfig->ssl_client_verify_server, conv); break; case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY: case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES: case TS_CONFIG_SSL_CLIENT_SNI_POLICY: + case TS_CONFIG_SSL_CLIENT_CERT_FILENAME: + case TS_CONFIG_SSL_CERT_FILEPATH: + case TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME: + case TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME: // String, must be handled elsewhere break; case TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB: @@ -8395,16 +8393,6 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char s->t_state.txn_conf->body_factory_template_base_len = 0; } break; - case TS_CONFIG_SSL_CERT_FILENAME: - if (value && length > 0) { - s->t_state.txn_conf->client_cert_filename = const_cast(value); - } - break; - case TS_CONFIG_SSL_CERT_FILEPATH: - if (value && length > 0) { - s->t_state.txn_conf->client_cert_filepath = const_cast(value); - } - break; case TS_CONFIG_HTTP_INSERT_FORWARDED: if (value && length > 0) { ts::LocalBufferWriter<1024> error; @@ -8431,6 +8419,24 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char s->t_state.txn_conf->ssl_client_sni_policy = const_cast(value); } break; + case TS_CONFIG_SSL_CLIENT_CERT_FILENAME: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_cert_filename = const_cast(value); + } + break; + case TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_private_key_filename = const_cast(value); + } + break; + case TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_ca_cert_filename = const_cast(value); + } + break; + case TS_CONFIG_SSL_CERT_FILEPATH: + /* noop */ + break; default: { MgmtConverter const *conv; void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); @@ -8505,7 +8511,6 @@ static const std::unordered_map SDK_Overridable_Configs = { "proxy.config.ssl.client.verify.server", "proxy.config.ssl.client.verify.server.policy", "proxy.config.ssl.client.verify.server.properties", - "proxy.config.ssl.client.sni_policy"}}; + "proxy.config.ssl.client.sni_policy", + "proxy.config.ssl.client.private_key.filename", + "proxy.config.ssl.client.CA.cert.filename"}}; REGRESSION_TEST(SDK_API_OVERRIDABLE_CONFIGS)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus) { diff --git a/tests/gold_tests/tls/tls_client_cert_override.test.py b/tests/gold_tests/tls/tls_client_cert_override.test.py new file mode 100644 index 00000000000..e5d21b8f442 --- /dev/null +++ b/tests/gold_tests/tls/tls_client_cert_override.test.py @@ -0,0 +1,145 @@ +''' +Test offering client cert to origin +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test conf_remp to specify different client certificates to offer to the origin +''' + +Test.SkipUnless(Condition.HasProgram("grep", "grep needs to be installed on system for this test to work")) + +ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=False) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": "true"}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": "true"}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server.Setup.Copy("ssl/signer.pem") +server.Setup.Copy("ssl/signer2.pem") +server.Setup.Copy("ssl/signed-foo.pem") +server.Setup.Copy("ssl/signed-foo.key") +server.Setup.Copy("ssl/signed2-foo.pem") +server.Setup.Copy("ssl/signed2-bar.pem") +server.Setup.Copy("ssl/signed-bar.key") +server2.Setup.Copy("ssl/signer.pem") +server2.Setup.Copy("ssl/signer2.pem") +server2.Setup.Copy("ssl/signed-foo.pem") +server2.Setup.Copy("ssl/signed-foo.key") +server2.Setup.Copy("ssl/signed2-foo.pem") +server2.Setup.Copy("ssl/signed2-bar.pem") +server2.Setup.Copy("ssl/signed-bar.key") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed2-foo.pem") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed2-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.cert.filename': 'signed-foo.pem', + 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.private_key.filename': 'signed-foo.key', + 'proxy.config.url_remap.pristine_host_hdr' : 1, +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.Port, "signed-foo.pem", "signed-foo.key") +) +ts.Disk.remap_config.AddLine( + 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.Port, "signed2-foo.pem", "signed-foo.key") +) +ts.Disk.remap_config.AddLine( + 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.Port, "signed2-foo.pem", "signed-foo.key") +) +ts.Disk.remap_config.AddLine( + 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.Port, "signed-foo.pem", "signed-foo.key") +) + +# Should succeed +tr = Test.AddTestRun("Connect with correct client cert to first server") +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(server2) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:example.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") +tr.TimeOut = 5 + +#Should fail +trfail = Test.AddTestRun("Connect with bad client cert to first server") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/badcase1'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") +trfail.TimeOut = 5 + +# Should succeed +trbar = Test.AddTestRun("Connect with correct client cert to second server") +trbar.StillRunningAfter = ts +trbar.StillRunningAfter = server +trbar.StillRunningAfter = server2 +trbar.Processes.Default.Command = "curl -H host:bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) +trbar.Processes.Default.ReturnCode = 0 +trbar.Processes.Default.TimeOut = 5 +trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") +trbar.TimeOut = 5 + +#Should fail +trbarfail = Test.AddTestRun("Connect with bad client cert to second server") +trbarfail.StillRunningAfter = ts +trbarfail.StillRunningAfter = server +trbarfail.StillRunningAfter = server2 +trbarfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/badcase2'.format(ts.Variables.port) +trbarfail.Processes.Default.ReturnCode = 0 +trbarfail.Processes.Default.TimeOut = 5 +trbarfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") +trbarfail.TimeOut = 5 + + diff --git a/tests/gold_tests/tls/tls_verify_ca_override.test.py b/tests/gold_tests/tls/tls_verify_ca_override.test.py new file mode 100644 index 00000000000..016ae9935be --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_ca_override.test.py @@ -0,0 +1,137 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options. Exercise conf_remap for ca bundle +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server1 = Test.MakeOriginServer("server1", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server2 = Test.MakeOriginServer("server2", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed2-foo.pem".format(Test.RunDirectory)}) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server1.addResponse("sessionlog.json", request_foo_header, response_header) +server1.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server2.addResponse("sessionlog.json", request_bar_header, response_header) +server2.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed2-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") +ts.addSSLfile("ssl/signer2.pem") +ts.addSSLfile("ssl/signer2.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.Port, ts.Variables.SSLDir, "signer.pem") +) +ts.Disk.remap_config.AddLine( + 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.Port, ts.Variables.SSLDir, "signer2.pem") +) +ts.Disk.remap_config.AddLine( + 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.Port, ts.Variables.SSLDir, "signer2.pem") +) +ts.Disk.remap_config.AddLine( + 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.Port, ts.Variables.SSLDir, "signer.pem") +) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'ENFORCED', + 'proxy.config.ssl.client.verify.server.properties': 'SIGNATURE', + 'proxy.config.ssl.client.CA.cert.path': '/tmp', + 'proxy.config.ssl.client.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# Should succeed +tr = Test.AddTestRun("Use corrcect ca bundle for server 1") +tr.Processes.Default.Command = 'curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/case1'.format(ts.Variables.port) +tr.ReturnCode = 0 +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed2-foo.pem") +tr.Processes.Default.StartBefore(server1) +tr.Processes.Default.StartBefore(server2) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.StillRunningAfter = server1 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +# Should succed. No message +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.TimeOut = 5 + +tr2 = Test.AddTestRun("Use incorrect ca bundle for server 1") +tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/badcase1".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server1 +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about name mismatch +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.TimeOut = 5 + +tr2 = Test.AddTestRun("Use currect ca bundle for server 2") +tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/case2".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server2 +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about signature +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.TimeOut = 5 + +tr3 = Test.AddTestRun("User incorrect ca bundle for server 2") +tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/badcase2".format(ts.Variables.port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server2 +tr3.StillRunningAfter = ts +# Should succeed. No error messages +tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") +tr3.Processes.Default.TimeOut = 5 + + From 70b42d1c429380dd4cd9fd1c0effd1f0cbd71143 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 7 Jan 2019 20:42:35 -0700 Subject: [PATCH 125/526] Fixes the hook install conditional, Makefile.am is finicky... --- Makefile.am | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 188b65357e9..133c31debe1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -95,8 +95,10 @@ install-examples: examples @cd example && $(MAKE) $(AM_MAKEFLAGS) install pkglibdir=$(pkglibexecdir) $(abs_top_srcdir)/.git/hooks/pre-commit: $(abs_top_srcdir)/tools/git/pre-commit - [ -d $(abs_top_srcdir)/.git/hooks ] && \ - cp $(abs_top_srcdir)/tools/git/pre-commit $(abs_top_srcdir)/.git/hooks/pre-commit + @if [ -d $(abs_top_srcdir)/.git/hooks ]; then \ + echo; echo "\tNote: Installing ATS .git/hooks/pre-commit"; echo; \ + cp $(abs_top_srcdir)/tools/git/pre-commit $(abs_top_srcdir)/.git/hooks/pre-commit; \ + fi install-data-hook: if BUILD_DOCS From 8046477d6c871be61a9d2ec6b41f2524a3fde699 Mon Sep 17 00:00:00 2001 From: fengshuaitao Date: Wed, 17 Jan 2018 15:08:29 +0800 Subject: [PATCH 126/526] Fix an failed assertion in HttpSM::parse_range_and_compare Signed-off-by: fengshuaitao --- proxy/http/HttpSM.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 18dad8e8ea9..08482eb1a6d 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -4231,6 +4231,8 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) const char *s, *e, *tmp; RangeRecord *ranges = nullptr; int64_t start, end; + int64_t cutoff = INT64_MAX / 10; + int64_t cutlim = INT64_MAX % 10; ink_assert(field != nullptr && t_state.range_setup == HttpTransact::RANGE_NONE && t_state.ranges == nullptr); @@ -4287,6 +4289,12 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) start = -1; } else { for (start = 0; s < e && *s >= '0' && *s <= '9'; ++s) { + // check the int64 overflow in case of high gcc with O3 option + // thinking the start is always positive + if (start >= cutoff && (start > cutoff || *s - '0' > cutlim)) { + t_state.range_setup = HttpTransact::RANGE_NONE; + goto Lfaild; + } start = start * 10 + (*s - '0'); } // skip last white spaces @@ -4319,6 +4327,12 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) end = content_length - 1; } else { for (end = 0; s < e && *s >= '0' && *s <= '9'; ++s) { + // check the int64 overflow in case of high gcc with O3 option + // thinking the start is always positive + if (end >= cutoff && (end > cutoff || *s - '0' > cutlim)) { + t_state.range_setup = HttpTransact::RANGE_NONE; + goto Lfaild; + } end = end * 10 + (*s - '0'); } // skip last white spaces From 6e84a42f6870a100bc9ea71e968ecb0c611a0375 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Tue, 8 Jan 2019 14:13:18 -0800 Subject: [PATCH 127/526] Fixed clang 5.0.0 issue with brace initialization --- iocore/eventsystem/I_VConnection.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iocore/eventsystem/I_VConnection.h b/iocore/eventsystem/I_VConnection.h index 62f604155f9..bc72ba5fe9b 100644 --- a/iocore/eventsystem/I_VConnection.h +++ b/iocore/eventsystem/I_VConnection.h @@ -406,7 +406,7 @@ class AnnotatedVConnection : public VConnection }; protected: - std::array user_args{nullptr}; + std::array user_args{{nullptr}}; }; struct DummyVConnection : public AnnotatedVConnection { From ee16b64cfe56ee4ac30128a6dc465d9238a5f42f Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Tue, 13 Nov 2018 21:52:24 +0000 Subject: [PATCH 128/526] Repurpose sub claim and add cdniuc support --- plugins/experimental/uri_signing/jwt.c | 14 +++++--------- plugins/experimental/uri_signing/jwt.h | 3 ++- plugins/experimental/uri_signing/parse.c | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index 0942c84dd0a..9324ca866ec 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -63,6 +63,7 @@ parse_jwt(json_t *raw) jwt->cdniv = parse_integer_default(json_object_get(raw, "cdniv"), 1); jwt->cdnicrit = json_string_value(json_object_get(raw, "cdnicrit")); jwt->cdniip = json_string_value(json_object_get(raw, "cdniip")); + jwt->cdniuc = json_string_value(json_object_get(raw, "cdniuc")); jwt->cdniets = json_integer_value(json_object_get(raw, "cdniets")); jwt->cdnistt = json_integer_value(json_object_get(raw, "cdnistt")); jwt->cdnistd = parse_integer_default(json_object_get(raw, "cdnistd"), 0); @@ -114,13 +115,8 @@ jwt_validate(struct jwt *jwt) return false; } - if (!jwt->sub) { /* Mandatory claim. Will be validated after key verification. */ - PluginDebug("Initial JWT Failure: missing sub"); - return false; - } - if (!unsupported_string_claim(jwt->aud)) { - PluginDebug("Initial JWT Failure: missing sub"); + PluginDebug("Initial JWT Failure: aud unsupported"); return false; } @@ -163,17 +159,17 @@ jwt_validate(struct jwt *jwt) } bool -jwt_check_uri(const char *sub, const char *uri) +jwt_check_uri(const char *cdniuc, const char *uri) { static const char CONT_URI_STR[] = "uri"; static const char CONT_URI_PATTERN_STR[] = "uri-pattern"; static const char CONT_URI_REGEX_STR[] = "uri-regex"; - if (!sub || !*sub || !uri) { + if (!cdniuc || !*cdniuc || !uri) { return false; } - const char *kind = sub, *container = sub; + const char *kind = cdniuc, *container = cdniuc; while (*container && *container != ':') { ++container; } diff --git a/plugins/experimental/uri_signing/jwt.h b/plugins/experimental/uri_signing/jwt.h index f0bd67aa2fa..5e09f028e7c 100644 --- a/plugins/experimental/uri_signing/jwt.h +++ b/plugins/experimental/uri_signing/jwt.h @@ -31,6 +31,7 @@ struct jwt { int cdniv; const char *cdnicrit; const char *cdniip; + const char *cdniuc; int cdniets; int cdnistt; int cdnistd; @@ -38,7 +39,7 @@ struct jwt { struct jwt *parse_jwt(json_t *raw); void jwt_delete(struct jwt *jwt); bool jwt_validate(struct jwt *jwt); -bool jwt_check_uri(const char *sub, const char *uri); +bool jwt_check_uri(const char *cdniuc, const char *uri); struct _cjose_jwk_int; char *renew(struct jwt *jwt, const char *iss, struct _cjose_jwk_int *jwk, const char *alg, const char *package); diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index 6f7d421319d..a53c60fcf85 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -184,7 +184,7 @@ validate_jws(cjose_jws_t *jws, struct config *cfg, const char *uri, size_t uri_c } } - if (!jwt_check_uri(jwt->sub, uri)) { + if (!jwt_check_uri(jwt->cdniuc, uri)) { PluginDebug("Valid key for %16p that does not match uri.", jws); goto jwt_fail; } From 732dd2095547387fb0b51d86460405c668acac78 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Mon, 7 Jan 2019 11:48:26 +0800 Subject: [PATCH 129/526] The mutex of NetAccept::action_->continuation is optional when the EVENT_ERROR event is called back. In general, `NetAccept::action_->continuation` is a type of `ProtocolProbeSessionAccept` object. The mutex of `ProtocolProbeSessionAccept` is NULL to allow parallel accepts. Resolve issue #4726. --- iocore/net/UnixNetAccept.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index 21f39c4ac85..9c3e40fb72d 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -322,7 +322,7 @@ NetAccept::do_blocking_accept(EThread *t) return 0; } if (!action_->cancelled) { - SCOPED_MUTEX_LOCK(lock, action_->mutex, t); + SCOPED_MUTEX_LOCK(lock, action_->mutex ? action_->mutex : t->mutex, t); action_->continuation->handleEvent(EVENT_ERROR, (void *)(intptr_t)res); Warning("accept thread received fatal error: errno = %d", errno); } From 1ad12700c2750774cf98bc3c83ff5aea6f83a79c Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 8 Jan 2019 09:07:43 +0900 Subject: [PATCH 130/526] Cleanup: Make _next_round_robin uint64_t --- iocore/eventsystem/I_EventProcessor.h | 6 +++--- iocore/eventsystem/P_UnixEventProcessor.h | 7 +------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/iocore/eventsystem/I_EventProcessor.h b/iocore/eventsystem/I_EventProcessor.h index 3730b4915cf..2bf90103109 100644 --- a/iocore/eventsystem/I_EventProcessor.h +++ b/iocore/eventsystem/I_EventProcessor.h @@ -303,9 +303,9 @@ class EventProcessor : public Processor /// The thread group ID is the index into an array of these and so is not stored explicitly. struct ThreadGroupDescriptor { std::string _name; ///< Name for the thread group. - int _count = 0; ///< # of threads of this type. - std::atomic _started = 0; ///< # of started threads of this type. - int _next_round_robin = 0; ///< Index of thread to use for events assigned to this group. + int _count = 0; ///< # of threads of this type. + std::atomic _started = 0; ///< # of started threads of this type. + uint64_t _next_round_robin = 0; ///< Index of thread to use for events assigned to this group. Que(Event, link) _spawnQueue; ///< Events to dispatch when thread is spawned. EThread *_thread[MAX_THREADS_IN_EACH_TYPE] = {}; ///< The actual threads in this group. std::function _afterStartCallback = nullptr; diff --git a/iocore/eventsystem/P_UnixEventProcessor.h b/iocore/eventsystem/P_UnixEventProcessor.h index ba58de9ee7c..ed9772995f1 100644 --- a/iocore/eventsystem/P_UnixEventProcessor.h +++ b/iocore/eventsystem/P_UnixEventProcessor.h @@ -54,12 +54,7 @@ EventProcessor::assign_thread(EventType etype) ink_assert(etype < MAX_EVENT_TYPES); if (tg->_count > 1) { - // When "_next_round_robin" grows big enough, it becomes a negative number, - // meaning "next" is also negative. And since "next" is used as an index - // into array "_thread", the result is returning NULL when assigning threads. - // So we need to cast "_next_round_robin" to unsigned int so the result stays - // positive. - next = static_cast(++tg->_next_round_robin) % tg->_count; + next = ++tg->_next_round_robin % tg->_count; } else { next = 0; } From 571d11ecf49c410470e2efbc1eca469618629084 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 9 Jan 2019 09:46:41 -0700 Subject: [PATCH 131/526] This improves on #3008, making the code clearer --- proxy/http/HttpSM.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 08482eb1a6d..875dba887c3 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -4231,8 +4231,6 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) const char *s, *e, *tmp; RangeRecord *ranges = nullptr; int64_t start, end; - int64_t cutoff = INT64_MAX / 10; - int64_t cutlim = INT64_MAX % 10; ink_assert(field != nullptr && t_state.range_setup == HttpTransact::RANGE_NONE && t_state.ranges == nullptr); @@ -4291,11 +4289,13 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) for (start = 0; s < e && *s >= '0' && *s <= '9'; ++s) { // check the int64 overflow in case of high gcc with O3 option // thinking the start is always positive - if (start >= cutoff && (start > cutoff || *s - '0' > cutlim)) { + int64_t new_start = start * 10 + (*s - '0'); + + if (new_start < start) { // Overflow t_state.range_setup = HttpTransact::RANGE_NONE; goto Lfaild; } - start = start * 10 + (*s - '0'); + start = new_start; } // skip last white spaces for (; s < e && ParseRules::is_ws(*s); ++s) { @@ -4329,11 +4329,13 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) for (end = 0; s < e && *s >= '0' && *s <= '9'; ++s) { // check the int64 overflow in case of high gcc with O3 option // thinking the start is always positive - if (end >= cutoff && (end > cutoff || *s - '0' > cutlim)) { + int64_t new_end = end * 10 + (*s - '0'); + + if (new_end < end) { // Overflow t_state.range_setup = HttpTransact::RANGE_NONE; goto Lfaild; } - end = end * 10 + (*s - '0'); + end = new_end; } // skip last white spaces for (; s < e && ParseRules::is_ws(*s); ++s) { From 792a02222b0cebca2807e98ad75caa79b677132c Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Mon, 24 Dec 2018 10:44:09 -0600 Subject: [PATCH 132/526] fix NXDOMAIN problems --- iocore/hostdb/HostDB.cc | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 6e50c443d3d..de09e9187ef 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -441,6 +441,7 @@ HostDBContinuation::init(HostDBHash const &the_hash, Options const &opt) action = opt.cont; } else { // ink_assert(!"this sucks"); + ink_zero(action); action.mutex = mutex; } } @@ -1591,10 +1592,10 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) EThread *t = e ? e->ethread : this_ethread(); MUTEX_TRY_LOCK(lock, action.mutex, t); - // Go ahead and grab the continuation mutex or just grab the action mutex again of there is no continuation mutex - MUTEX_TRY_LOCK(lock2, (action.continuation && action.continuation->mutex) ? action.continuation->mutex : action.mutex, t); - // Don't continue unless we have both mutexes - if (!lock.is_locked() || !lock2.is_locked()) { + + // Separating lock checks here to make sure things don't break + // when we check if the action is cancelled. + if (!lock.is_locked()) { mutex->thread_holding->schedule_in(this, HOST_DB_RETRY_PERIOD); return EVENT_CONT; } @@ -1604,6 +1605,14 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) return EVENT_DONE; } + // Go ahead and grab the continuation mutex or just grab the action mutex again of there is no continuation mutex + MUTEX_TRY_LOCK(lock2, (action.continuation && action.continuation->mutex) ? action.continuation->mutex : action.mutex, t); + // Don't continue unless we have both mutexes + if (!lock2.is_locked()) { + mutex->thread_holding->schedule_in(this, HOST_DB_RETRY_PERIOD); + return EVENT_CONT; + } + if (!hostdb_enable || (!*hash.host_name && !hash.ip.isValid())) { if (action.continuation) { action.continuation->handleEvent(EVENT_HOST_DB_LOOKUP, nullptr); From 4f5baf097467ff6d3f76d566e57509800f2099d3 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Fri, 2 Nov 2018 18:05:12 +0000 Subject: [PATCH 133/526] Add support for JWS to be passed as a URI path parameter in URI signing plugin --- plugins/experimental/uri_signing/parse.c | 71 ++++++++++++------- plugins/experimental/uri_signing/parse.h | 2 +- .../experimental/uri_signing/uri_signing.c | 2 +- 3 files changed, 48 insertions(+), 27 deletions(-) diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index a53c60fcf85..603c4ec9f53 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -29,37 +29,64 @@ #include cjose_jws_t * -get_jws_from_query(const char *uri, size_t uri_ct, const char *paramName) +get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName) { - PluginDebug("Parsing JWS from query string: %.*s", (int)uri_ct, uri); - const char *query = uri; - const char *end = uri + uri_ct; - while (query != end && *query != '?') { - ++query; - } - if (query == end) { + /* Reserved characters as defined by the URI Generic Syntax RFC: https://tools.ietf.org/html/rfc3986#section-2.2 */ + const char *reserved_string = ":/?#[]@!$&\'()*+,;="; + + /* If param name ends in reserved character this will be treated as the termination symbol when parsing for package. Default is + * '='. */ + char termination_symbol; + size_t termination_ct; + size_t param_ct = strlen(paramName); + + if (param_ct <= 0) { + PluginDebug("URI signing package name cannot be empty"); return NULL; } - ++query; + if (strchr(reserved_string, paramName[param_ct - 1])) { + termination_symbol = paramName[param_ct - 1]; + termination_ct = param_ct - 1; + } else { + termination_symbol = '='; + termination_ct = param_ct; + } + + PluginDebug("Parsing JWS from query string: %.*s", (int)uri_ct, uri); + const char *param = uri; + const char *end = uri + uri_ct; + const char *key, *key_end; + const char *value, *value_end; - const char *key = query, *key_end; - const char *value = query, *value_end; for (;;) { - while (value != end && *value != '=') { - ++value; + /* Search the URI for a reserved character. */ + while (param != end && strchr(reserved_string, *param) == NULL) { + ++param; } + if (param == end) { + break; + } + + ++param; + /* Parse the parameter for a key value pair separated by the termination symbol. */ + key = param; + value = param; + while (value != end && *value != termination_symbol) { + ++value; + } if (value == end) { break; } - key_end = value; - value_end = ++value; - while (value_end != end && *value_end != '&') { - ++value_end; - } + key_end = value; - if (!strncmp(paramName, key, (size_t)(key_end - key))) { + /* If the Parameter key is our target parameter name, attempt to import a JWS from the value. */ + if ((size_t)(key_end - key) == termination_ct && !strncmp(paramName, key, (size_t)(key_end - key))) { + value_end = ++value; + while (value_end != end && strchr(reserved_string, *value_end) == NULL) { + ++value_end; + } PluginDebug("Decoding JWS: %.*s", (int)(key_end - key), key); cjose_err err = {0}; cjose_jws_t *jws = cjose_jws_import(value, (size_t)(value_end - value), &err); @@ -70,12 +97,6 @@ get_jws_from_query(const char *uri, size_t uri_ct, const char *paramName) } return jws; } - - if (value_end == end) { - break; - } - - key = value = value_end + 1; } PluginDebug("Unable to locate signing key in uri: %.*s", (int)uri_ct, uri); return NULL; diff --git a/plugins/experimental/uri_signing/parse.h b/plugins/experimental/uri_signing/parse.h index 8002f8781fe..8d82c63ea7f 100644 --- a/plugins/experimental/uri_signing/parse.h +++ b/plugins/experimental/uri_signing/parse.h @@ -19,7 +19,7 @@ #include struct _cjose_jws_int; -struct _cjose_jws_int *get_jws_from_query(const char *uri, size_t uri_ct, const char *paramName); +struct _cjose_jws_int *get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName); struct _cjose_jws_int *get_jws_from_cookie(const char **cookie, size_t *cookie_ct, const char *paramName); struct config; diff --git a/plugins/experimental/uri_signing/uri_signing.c b/plugins/experimental/uri_signing/uri_signing.c index 55ba1173045..e9a2a81579e 100644 --- a/plugins/experimental/uri_signing/uri_signing.c +++ b/plugins/experimental/uri_signing/uri_signing.c @@ -175,7 +175,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } - cjose_jws_t *jws = get_jws_from_query(url, url_ct, package); + cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package); if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } From 8b264377b13b99c7c7662cdafa53f28f6840a752 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 10 Jan 2019 09:37:25 -0600 Subject: [PATCH 134/526] PR-4779: Fix TLS Bridge plugin to compile regular expressions from file configuration correctly. --- plugins/experimental/tls_bridge/tls_bridge.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/experimental/tls_bridge/tls_bridge.cc b/plugins/experimental/tls_bridge/tls_bridge.cc index 368a808cfac..e8b78b6ef91 100644 --- a/plugins/experimental/tls_bridge/tls_bridge.cc +++ b/plugins/experimental/tls_bridge/tls_bridge.cc @@ -109,7 +109,9 @@ void BridgeConfig::load_pair(std::string_view rxp, std::string_view service, ts::file::path const &src, int ln) { Regex r; - if (r.compile(rxp.data(), Regex::ANCHORED)) { + // Unfortunately PCRE can only compile null terminated strings... + std::string pattern{rxp}; + if (r.compile(pattern.c_str(), Regex::ANCHORED)) { _items.emplace_back(rxp, std::move(r), service); } else { char buff[std::numeric_limits::digits10 + 2]; From adf3299dd4b41d934c0d8322ed7623be935aff9a Mon Sep 17 00:00:00 2001 From: Steven Feltner Date: Thu, 12 Jul 2018 09:21:33 -0700 Subject: [PATCH 135/526] PROXY Protocol transformed to Forwarded HTTP header. --- doc/admin-guide/configuration/index.en.rst | 1 + .../configuration/proxy-protocol.en.rst | 100 ++++++++++ doc/admin-guide/files/records.config.en.rst | 33 +++- doc/static/images/admin/proxy-protocol.png | 0 include/tscore/IpMapConf.h | 2 + iocore/net/Connection.cc | 4 + iocore/net/I_NetProcessor.h | 4 + iocore/net/I_NetVConnection.h | 132 ++++++++++++- iocore/net/Makefile.am | 2 + iocore/net/P_NetVConnection.h | 39 +++- iocore/net/P_SSLConfig.h | 4 + iocore/net/P_UnixNetVConnection.h | 38 ++-- iocore/net/ProxyProtocol.cc | 179 ++++++++++++++++++ iocore/net/ProxyProtocol.h | 55 ++++++ iocore/net/SSLConfig.cc | 8 + iocore/net/SSLNetVConnection.cc | 45 +++++ iocore/net/UnixNetAccept.cc | 3 + iocore/net/UnixNetProcessor.cc | 5 + lib/records/I_RecHttp.h | 9 + lib/records/RecHttp.cc | 33 ++++ mgmt/LocalManager.cc | 4 + mgmt/RecordsConfig.cc | 2 + proxy/PluginVC.cc | 6 + proxy/PluginVC.h | 1 + proxy/ProtocolProbeSessionAccept.cc | 40 ++++ proxy/ProtocolProbeSessionAccept.h | 2 + proxy/http/HttpConfig.cc | 3 + proxy/http/HttpConfig.h | 2 + proxy/http/HttpProxyServerMain.cc | 3 +- proxy/http/HttpTransact.cc | 7 +- proxy/http/HttpTransact.h | 3 + 31 files changed, 742 insertions(+), 27 deletions(-) create mode 100644 doc/admin-guide/configuration/proxy-protocol.en.rst create mode 100644 doc/static/images/admin/proxy-protocol.png create mode 100644 iocore/net/ProxyProtocol.cc create mode 100644 iocore/net/ProxyProtocol.h diff --git a/doc/admin-guide/configuration/index.en.rst b/doc/admin-guide/configuration/index.en.rst index cc2e3f603e6..1aecdbe29fe 100644 --- a/doc/admin-guide/configuration/index.en.rst +++ b/doc/admin-guide/configuration/index.en.rst @@ -32,3 +32,4 @@ Proxy Cache Configuration transparent-proxy.en transparent-forward-proxying.en hierachical-caching.en + proxy-protocol.en diff --git a/doc/admin-guide/configuration/proxy-protocol.en.rst b/doc/admin-guide/configuration/proxy-protocol.en.rst new file mode 100644 index 00000000000..2d7673b2914 --- /dev/null +++ b/doc/admin-guide/configuration/proxy-protocol.en.rst @@ -0,0 +1,100 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _proxy-protocol: + +Proxy Protocol +**************** + +The `PROXY protocol `_ +provides a means of passing connection information between layers of the proxy +infrastructure. Without the PROXY protocol, |TS| would only have connection +information from the previous hop connecting to |TS| and not the actual +originating client connection information. This can be done over either HTTP or +TLS connections. + +.. note:: + + The current version only supports transforming client IP from PROXY Version 1 + header to the Forwarded: header. + +In the current implementation, the client IP address in the PROXY protocol header +is passed to the origin server via an HTTP `Forwarded: +`_ header. + +The Proxy Protocol must be enabled on each port. See +:ts:cv:`proxy.config.http.server_ports` for information on how to enable the +Proxy Protocol on a port. Once enabled, all incoming requests must be prefaced +with the PROXY v1 header. Any request not preface by this header will be +dropped. + +As a security measure, an optional whitelist of trusted IP addresses may be +configured with :ts:cv:`proxy.config.http.proxy_protocol_whitelist`. + + .. important:: + + If the whitelist is configured, requests will only be accepted from these + IP addressses and must be prefaced with the PROXY v1 header. + +See :ts:cv:`proxy.config.http.insert_forwarded` for configuration information. +Detection of the PROXY protocol header is automatic. If the PROXY header +precludes the request, it will automatically be parse and made available to the +Forwarded: request header sent to the origin server. + +Example +------- + +As an example, consider the following topology: + +.. figure:: ../../static/images/admin/proxy-protocol.png + :align: center + :alt: PROXY protocol transformed into a Forwarded: header + + PROXY protocol header flow + +Without the PROXY protocol header, the client IP would only be reported +accurately to the Load Balancer. |TS| would only see the connection from the +Load Balancer. Similarly, the Web Server would only see the connection from +|TS|. In the example above, if the client initiated a TLS connection, the Web +Server would see the connection originating from |TS| at ``10.0.0.2``: + +.. code-block:: lua + + Forwarded: for=10.0.0.2;by=10.0.0.1;proto=https;host=test000001.com + +If the Load Balancer has the Proxy Protocol enabled, requests sent through the +Load Balancer will be preceded with the PROXY header. |TS| will detect the +PROXY header and transform that into the Forwarded: HTTP header if configured to +insert the Forwarded: header with the ``for`` paramter. In the example above, +if the client initiated a TLS connection, the Web Server can use the Forwarded: +header to determine the TLS connection originated from the client at ``192.168.1.100``: + +.. code-block:: lua + + Forwarded: for=192.168.2.100;by=10.0.0.2;proto=https;host=test000001.com + + +References +========== + +- `The PROXY protocol Versions 1 & 2 + `_ + +- `Forwarded HTTP Extension + `_ diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index e2443846b44..c4c02a805ea 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -610,6 +610,7 @@ HTTP Engine ip-out Value Local outbound IP address. ip-resolve Value IP address resolution style. proto Value List of supported session protocols. + pp Enable Proxy Protocol. ssl SSL terminated. tr-full Fully transparent (inbound and outbound) tr-in Inbound transparent. @@ -646,6 +647,12 @@ proto all available protocols. For non-TLS proxy ports the default is HTTP only. +pp + Enables Proxy Protocol on the port. If Proxy Protocol is enabled on the + port, all incoming requests must be prefaced with the PROXY header. See + :ref:`Proxy Protocol ` for more details on how to configure + this option properly. + tr-full Fully transparent. This is a convenience option and is identical to specifying both ``tr-in`` and ``tr-out``. @@ -1730,6 +1737,30 @@ Proxy User Variables is prohibited by RFC 7239. Currently, for the ``host`` parameter to provide the original host from the incoming client request, `proxy.config.url_remap.pristine_host_hdr`_ must be enabled. +.. ts:cv:: CONFIG proxy.config.http.proxy_protocol_whitelist STRING `````` + + This defines a whitelist of server IPs that are trusted to provide + connections with Proxy Protocol information. This is a comma delimited list + of IP addresses. Addressed may be listed individually, in a range separated + by a dash or by using CIDR notation. + + ======================= =========================================================== + Example Effect + ======================= =========================================================== + ``10.0.2.123`` A single IP Address. + ``10.0.3.1-10.0.3.254`` A range of IP address. + ``10.0.4.0/24`` A range of IP address specified by CIDR notation. + ======================= ============================================================ + + .. important:: + + If Proxy Protocol is enabled on the port, but this directive is not + defined any server may initiate a connection with Proxy Protocol + information. + See :ts:cv:`proxy.config.http.server_ports` for information on how to enable Proxy Protocol on a port. + + See :ref:`proxy-protocol` for more discussion on how |TS| tranforms the `Forwarded: header. + .. ts:cv:: CONFIG proxy.config.http.normalize_ae INT 1 :reloadable: :overridable: @@ -2631,7 +2662,7 @@ HostDB Set the file path for an external host file. If this is set (non-empty) then the file is presumed to be a hosts file in - the standard `host file format `_. + the standard . It is read and the entries there added to the HostDB. The file is periodically checked for a more recent modification date in which case it is reloaded. The interval is set with :ts:cv:`proxy.config.hostdb.host_file.interval`. diff --git a/doc/static/images/admin/proxy-protocol.png b/doc/static/images/admin/proxy-protocol.png new file mode 100644 index 00000000000..e69de29bb2d diff --git a/include/tscore/IpMapConf.h b/include/tscore/IpMapConf.h index 3e96bba6ba7..7d229adf15d 100644 --- a/include/tscore/IpMapConf.h +++ b/include/tscore/IpMapConf.h @@ -27,6 +27,8 @@ class IpMap; // declare in name only. +// Returns 0 if successful, error string otherwise +int read_addr(char *line, int n, int *i, sockaddr *addr, char *err); // Returns 0 if successful, error string otherwise char *Load_IpMap_From_File(IpMap *map, int fd, char const *key_str); // Returns 0 if successful, error string otherwise diff --git a/iocore/net/Connection.cc b/iocore/net/Connection.cc index 2384033a776..1595bb9791a 100644 --- a/iocore/net/Connection.cc +++ b/iocore/net/Connection.cc @@ -241,6 +241,10 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #endif } + if (opt.f_proxy_protocol) { + Debug("proxyprotocol", "Proxy Protocol enabled."); + } + #if defined(TCP_MAXSEG) if (NetProcessor::accept_mss > 0) { if ((res = safe_setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, (char *)&NetProcessor::accept_mss, sizeof(int))) < 0) { diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index acaa1a6515c..b8dee4e9dbc 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -24,6 +24,7 @@ #pragma once +#include "tscore/IpMap.h" #include "I_EventSystem.h" #include "I_Socks.h" struct socks_conf_struct; @@ -96,6 +97,9 @@ class NetProcessor : public Processor */ bool f_inbound_transparent; + /// Proxy Protocol enabled + bool f_proxy_protocol; + /// Default constructor. /// Instance is constructed with default values. AcceptOptions() { this->reset(); } diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index 1a8d6013126..786c7afe708 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -34,6 +34,8 @@ #include "ts/apidefs.h" #include #include "YamlSNIConfig.h" +#include "tscpp/util/TextView.h" +#include "tscore/IpMap.h" #define CONNECT_SUCCESS 1 #define CONNECT_FAILURE 0 @@ -130,10 +132,12 @@ struct NetVCOptions { @see ip_family */ IpAddr local_ip; + /** Local port for connection. Set to 0 for "don't care" (default). - */ + */ uint16_t local_port; + /// How to bind the local address. /// @note Default is @c ANY_ADDR. addr_bind_style addr_binding; @@ -642,6 +646,9 @@ class NetVConnection : public AnnotatedVConnection /** Set remote sock addr struct. */ virtual void set_remote_addr() = 0; + /** Set remote sock addr struct. */ + virtual void set_remote_addr(const sockaddr *) = 0; + // for InkAPI bool get_is_internal_request() const @@ -668,6 +675,19 @@ class NetVConnection : public AnnotatedVConnection is_transparent = state; } + /// Get the proxy protocol enabled flag + bool + get_is_proxy_protocol() const + { + return is_proxy_protocol; + } + /// Set the proxy protocol enabled flag on the port + void + set_is_proxy_protocol(bool state = true) + { + is_proxy_protocol = state; + } + virtual int populate_protocol(std::string_view *results, int n) const { @@ -684,6 +704,113 @@ class NetVConnection : public AnnotatedVConnection NetVConnection(const NetVConnection &) = delete; NetVConnection &operator=(const NetVConnection &) = delete; + enum class ProxyProtocolVersion { + UNDEFINED, + V1, + V2, + }; + + enum class ProxyProtocolData { + UNDEFINED, + SRC, + DST, + }; + + int + set_proxy_protocol_addr(const ProxyProtocolData src_or_dst, ts::TextView &ip_addr_str) + { + int ret = -1; + + if (src_or_dst == ProxyProtocolData::SRC) { + ret = ats_ip_pton(ip_addr_str, &pp_info.src_addr); + } else { + ret = ats_ip_pton(ip_addr_str, &pp_info.dst_addr); + } + return ret; + } + + int + set_proxy_protocol_src_addr(ts::TextView src) + { + return set_proxy_protocol_addr(ProxyProtocolData::SRC, src); + } + + int + set_proxy_protocol_dst_addr(ts::TextView src) + { + return set_proxy_protocol_addr(ProxyProtocolData::DST, src); + } + + int + set_proxy_protocol_port(const ProxyProtocolData src_or_dst, in_port_t port) + { + if (src_or_dst == ProxyProtocolData::SRC) { + pp_info.src_addr.port() = htons(port); + } else { + pp_info.dst_addr.port() = htons(port); + } + return port; + } + + int + set_proxy_protocol_src_port(in_port_t port) + { + return set_proxy_protocol_port(ProxyProtocolData::SRC, port); + } + + int + set_proxy_protocol_dst_port(in_port_t port) + { + return set_proxy_protocol_port(ProxyProtocolData::DST, port); + } + + void + set_proxy_protocol_version(const ProxyProtocolVersion ver) + { + pp_info.proxy_protocol_version = ver; + } + + ProxyProtocolVersion + get_proxy_protocol_version() + { + return pp_info.proxy_protocol_version; + } + + sockaddr const *get_proxy_protocol_addr(const ProxyProtocolData); + + sockaddr const * + get_proxy_protocol_src_addr() + { + return get_proxy_protocol_addr(ProxyProtocolData::SRC); + } + + uint16_t + get_proxy_protocol_src_port() + { + return ats_ip_port_host_order(this->get_proxy_protocol_addr(ProxyProtocolData::SRC)); + } + + sockaddr const * + get_proxy_protocol_dst_addr() + { + return get_proxy_protocol_addr(ProxyProtocolData::DST); + } + + uint16_t + get_proxy_protocol_dst_port() + { + return ats_ip_port_host_order(this->get_proxy_protocol_addr(ProxyProtocolData::DST)); + }; + + struct ProxyProtocol { + ProxyProtocolVersion proxy_protocol_version = ProxyProtocolVersion::UNDEFINED; + uint16_t ip_family; + IpEndpoint src_addr; + IpEndpoint dst_addr; + }; + + ProxyProtocol pp_info; + protected: IpEndpoint local_addr; IpEndpoint remote_addr; @@ -694,6 +821,8 @@ class NetVConnection : public AnnotatedVConnection bool is_internal_request; /// Set if this connection is transparent. bool is_transparent; + /// Set if proxy protocol is enabled + bool is_proxy_protocol; /// Set if the next write IO that empties the write buffer should generate an event. int write_buffer_empty_event; /// NetVConnection Context. @@ -708,6 +837,7 @@ inline NetVConnection::NetVConnection() got_remote_addr(false), is_internal_request(false), is_transparent(false), + is_proxy_protocol(false), write_buffer_empty_event(0), netvc_context(NET_VCONNECTION_UNSET) { diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index e9efb3c3778..9eb93963b71 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -130,6 +130,8 @@ libinknet_a_SOURCES = \ P_UnixNetVConnection.h \ P_UnixPollDescriptor.h \ P_UnixUDPConnection.h \ + ProxyProtocol.h \ + ProxyProtocol.cc \ Socks.cc \ SSLCertLookup.cc \ SSLSessionCache.cc \ diff --git a/iocore/net/P_NetVConnection.h b/iocore/net/P_NetVConnection.h index a81563304ca..3d85e7a1d35 100644 --- a/iocore/net/P_NetVConnection.h +++ b/iocore/net/P_NetVConnection.h @@ -23,24 +23,28 @@ #include "I_NetVConnection.h" -TS_INLINE sockaddr const * +inline sockaddr const * NetVConnection::get_remote_addr() { if (!got_remote_addr) { - set_remote_addr(); + if (pp_info.proxy_protocol_version != ProxyProtocolVersion::UNDEFINED) { + set_remote_addr(get_proxy_protocol_src_addr()); + } else { + set_remote_addr(); + } got_remote_addr = true; } return &remote_addr.sa; } -TS_INLINE IpEndpoint const & +inline IpEndpoint const & NetVConnection::get_remote_endpoint() { get_remote_addr(); // Make sure the vallue is filled in return remote_addr; } -TS_INLINE in_addr_t +inline in_addr_t NetVConnection::get_remote_ip() { sockaddr const *addr = this->get_remote_addr(); @@ -48,13 +52,13 @@ NetVConnection::get_remote_ip() } /// @return The remote port in host order. -TS_INLINE uint16_t +inline uint16_t NetVConnection::get_remote_port() { return ats_ip_port_host_order(this->get_remote_addr()); } -TS_INLINE sockaddr const * +inline sockaddr const * NetVConnection::get_local_addr() { if (!got_local_addr) { @@ -68,7 +72,7 @@ NetVConnection::get_local_addr() return &local_addr.sa; } -TS_INLINE in_addr_t +inline in_addr_t NetVConnection::get_local_ip() { sockaddr const *addr = this->get_local_addr(); @@ -76,8 +80,27 @@ NetVConnection::get_local_ip() } /// @return The local port in host order. -TS_INLINE uint16_t +inline uint16_t NetVConnection::get_local_port() { return ats_ip_port_host_order(this->get_local_addr()); } + +inline sockaddr const * +NetVConnection::get_proxy_protocol_addr(const ProxyProtocolData src_or_dst) +{ + if (src_or_dst == ProxyProtocolData::SRC) { + if ((pp_info.src_addr.isValid() && pp_info.src_addr.port() != 0) || + (ats_is_ip4(&pp_info.src_addr) && INADDR_ANY != ats_ip4_addr_cast(&pp_info.src_addr)) // IPv4 + || (ats_is_ip6(&pp_info.src_addr) && !IN6_IS_ADDR_UNSPECIFIED(&pp_info.src_addr.sin6.sin6_addr))) { + return &pp_info.src_addr.sa; + } + } else { + if ((pp_info.dst_addr.isValid() && pp_info.dst_addr.port() != 0) || + (ats_is_ip4(&pp_info.dst_addr) && INADDR_ANY != ats_ip4_addr_cast(&pp_info.dst_addr)) // IPv4 + || (ats_is_ip6(&pp_info.dst_addr) && !IN6_IS_ADDR_UNSPECIFIED(&pp_info.dst_addr.sin6.sin6_addr))) { + return &pp_info.dst_addr.sa; + } + } + return nullptr; +} diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 5c93abbbd8f..df178660511 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -36,6 +36,7 @@ #include #include "P_SSLCertLookup.h" #include "YamlSNIConfig.h" +#include struct SSLCertLookup; struct ssl_ticket_key_block; @@ -109,6 +110,8 @@ struct SSLConfigParams : public ConfigInfo { static size_t session_cache_max_bucket_size; static bool session_cache_skip_on_lock_contention; + static IpMap *proxy_protocol_ipmap; + static init_ssl_ctx_func init_ssl_ctx_cb; static load_ssl_file_func load_ssl_file_cb; @@ -130,6 +133,7 @@ struct SSLConfigParams : public ConfigInfo { void initialize(); void cleanup(); void reset(); + void SSLConfigInit(IpMap *global); }; ///////////////////////////////////////////////////////////// diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 0a320319b1c..94c141801b9 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -41,7 +41,7 @@ class UnixNetVConnection; class NetHandler; struct PollDescriptor; -TS_INLINE void +inline void NetVCOptions::reset() { ip_proto = USE_TCP; @@ -73,7 +73,7 @@ NetVCOptions::reset() ssl_client_ca_cert_name = nullptr; } -TS_INLINE void +inline void NetVCOptions::set_sock_param(int _recv_bufsize, int _send_bufsize, unsigned long _opt_flags, unsigned long _packet_mark, unsigned long _packet_tos) { @@ -293,6 +293,7 @@ class UnixNetVConnection : public NetVConnection void set_local_addr() override; void set_remote_addr() override; + void set_remote_addr(const sockaddr *) override; int set_tcp_init_cwnd(int init_cwnd) override; int set_tcp_congestion_control(int side) override; void apply_options() override; @@ -322,14 +323,21 @@ extern ClassAllocator netVCAllocator; typedef int (UnixNetVConnection::*NetVConnHandler)(int, void *); -TS_INLINE void +inline void UnixNetVConnection::set_remote_addr() { ats_ip_copy(&remote_addr, &con.addr); this->control_flags.set_flag(ContFlags::DEBUG_OVERRIDE, diags->test_override_ip(remote_addr)); } -TS_INLINE void +inline void +UnixNetVConnection::set_remote_addr(const sockaddr *new_sa) +{ + ats_ip_copy(&remote_addr, new_sa); + this->control_flags.set_flag(ContFlags::DEBUG_OVERRIDE, diags->test_override_ip(remote_addr)); +} + +inline void UnixNetVConnection::set_local_addr() { int local_sa_size = sizeof(local_addr); @@ -339,19 +347,19 @@ UnixNetVConnection::set_local_addr() ATS_UNUSED_RETURN(safe_getsockname(con.fd, &local_addr.sa, &local_sa_size)); } -TS_INLINE ink_hrtime +inline ink_hrtime UnixNetVConnection::get_active_timeout() { return active_timeout_in; } -TS_INLINE ink_hrtime +inline ink_hrtime UnixNetVConnection::get_inactivity_timeout() { return inactivity_timeout_in; } -TS_INLINE void +inline void UnixNetVConnection::set_active_timeout(ink_hrtime timeout_in) { Debug("socket", "Set active timeout=%" PRId64 ", NetVC=%p", timeout_in, this); @@ -359,7 +367,7 @@ UnixNetVConnection::set_active_timeout(ink_hrtime timeout_in) next_activity_timeout_at = Thread::get_hrtime() + timeout_in; } -TS_INLINE void +inline void UnixNetVConnection::cancel_inactivity_timeout() { Debug("socket", "Cancel inactive timeout for NetVC=%p", this); @@ -367,7 +375,7 @@ UnixNetVConnection::cancel_inactivity_timeout() set_inactivity_timeout(0); } -TS_INLINE void +inline void UnixNetVConnection::cancel_active_timeout() { Debug("socket", "Cancel active timeout for NetVC=%p", this); @@ -375,7 +383,7 @@ UnixNetVConnection::cancel_active_timeout() next_activity_timeout_at = 0; } -TS_INLINE int +inline int UnixNetVConnection::set_tcp_init_cwnd(int init_cwnd) { #ifdef TCP_INIT_CWND @@ -390,7 +398,7 @@ UnixNetVConnection::set_tcp_init_cwnd(int init_cwnd) #endif } -TS_INLINE int +inline int UnixNetVConnection::set_tcp_congestion_control(int side) { #ifdef TCP_CONGESTION @@ -425,21 +433,21 @@ UnixNetVConnection::set_tcp_congestion_control(int side) #endif } -TS_INLINE UnixNetVConnection::~UnixNetVConnection() {} +inline UnixNetVConnection::~UnixNetVConnection() {} -TS_INLINE SOCKET +inline SOCKET UnixNetVConnection::get_socket() { return con.fd; } -TS_INLINE void +inline void UnixNetVConnection::set_action(Continuation *c) { action_ = c; } -TS_INLINE const Action * +inline const Action * UnixNetVConnection::get_action() const { return &action_; diff --git a/iocore/net/ProxyProtocol.cc b/iocore/net/ProxyProtocol.cc new file mode 100644 index 00000000000..83c3bcf9dfe --- /dev/null +++ b/iocore/net/ProxyProtocol.cc @@ -0,0 +1,179 @@ +/** @file + * + * PROXY protocol definitions and parsers. + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tscore/ink_assert.h" +#include "tscpp/util/TextView.h" +#include "ProxyProtocol.h" +#include "I_NetVConnection.h" + +bool +ssl_has_proxy_v1(NetVConnection *sslvc, char *buffer, int64_t *bytes_r) +{ + ts::TextView tv; + + tv.assign(buffer, *bytes_r); + + // Client must send at least 15 bytes to get a reasonable match. + if (tv.size() < PROXY_V1_CONNECTION_HEADER_LEN_MIN) { + Debug("proxyprotocol_v1", "ssl_has_proxy_v1: not enough recv'd"); + return false; + } + + // if we don't have the PROXY preface, we don't have a ProxyV1 header + if (0 != memcmp(PROXY_V1_CONNECTION_PREFACE, buffer, PROXY_V1_CONNECTION_PREFACE_LEN)) { + Debug("proxyprotocol_v1", "ssl_has_proxy_v1: failed the memcmp(%s, %s, %lu)", PROXY_V1_CONNECTION_PREFACE, buffer, + PROXY_V1_CONNECTION_PREFACE_LEN); + return false; + } + + // Find the terminating newline + ts::TextView::size_type pos = tv.find('\n'); + if (pos == tv.npos) { + Debug("proxyprotocol_v1", "ssl_has_proxy_v1: newline not found"); + return false; + } + + // Parse the TextView before moving the bytes in the buffer + if (!proxy_protov1_parse(sslvc, tv)) { + *bytes_r = -EAGAIN; + return false; + } + *bytes_r -= pos + 1; + if (*bytes_r <= 0) { + *bytes_r = -EAGAIN; + } else { + Debug("ssl", "Moving %" PRId64 " characters remaining in the buffer from %p to %p", *bytes_r, buffer + pos + 1, buffer); + memmove(buffer, buffer + pos + 1, *bytes_r); + } + return true; +} + +bool +http_has_proxy_v1(IOBufferReader *reader, NetVConnection *netvc) +{ + char buf[PROXY_V1_CONNECTION_HEADER_LEN_MAX + 1]; + ts::TextView tv; + + tv.assign(buf, reader->memcpy(buf, sizeof(buf), 0)); + + // Client must send at least 15 bytes to get a reasonable match. + if (tv.size() < PROXY_V1_CONNECTION_HEADER_LEN_MIN) { + return false; + } + + if (0 != memcmp(PROXY_V1_CONNECTION_PREFACE, buf, PROXY_V1_CONNECTION_PREFACE_LEN)) { + return false; + } + + // Find the terminating LF, which should already be in the buffer. + ts::TextView::size_type pos = tv.find('\n'); + if (pos == tv.npos) { // not found, it's not a proxy protocol header. + return false; + } + reader->consume(pos + 1); // clear out the header. + + // Now that we know we have a valid PROXY V1 preface, let's parse the + // remainder of the header + + return proxy_protov1_parse(netvc, tv); +} + +bool +proxy_protov1_parse(NetVConnection *netvc, ts::TextView hdr) +{ + static const std::string_view PREFACE{PROXY_V1_CONNECTION_PREFACE, PROXY_V1_CONNECTION_PREFACE_LEN}; + ts::TextView token; + in_port_t port; + + // All the cases are special and sequence, might as well unroll them. + + // The header should begin with the PROXY preface + token = hdr.split_prefix_at(' '); + if (0 == token.size() || token != PREFACE) { + Debug("proxyprotocol_v1", "proxy_protov1_parse: header [%.*s] does not start with preface [%.*s]", static_cast(hdr.size()), + hdr.data(), static_cast(PREFACE.size()), PREFACE.data()); + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = PREFACE", static_cast(token.size()), token.data()); + + // The INET protocol family - TCP4, TCP6 or UNKNOWN + token = hdr.split_prefix_at(' '); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = INET Family", static_cast(token.size()), token.data()); + + // Next up is the layer 3 source address + // - 255.255.255.255 or ffff:f...f:ffff ffff:f...f:fff + token = hdr.split_prefix_at(' '); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Source Address", static_cast(token.size()), token.data()); + if (0 != netvc->set_proxy_protocol_src_addr(token)) { + return false; + } + + // Next is the layer3 destination address + // - 255.255.255.255 or ffff:f...f:ffff ffff:f...f:fff + token = hdr.split_prefix_at(' '); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Destination Address", static_cast(token.size()), token.data()); + if (0 != netvc->set_proxy_protocol_dst_addr(token)) { + return false; + } + + // Next is the TCP source port represented as a decimal number in the range of [0..65535] inclusive. + token = hdr.split_prefix_at(' '); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Source Port", static_cast(token.size()), token.data()); + + if (0 == (port = ts::svtoi(token))) { + Debug("proxyprotocol_v1", "proxy_protov1_parse: src port [%d] token [%.*s] failed to parse", port, + static_cast(token.size()), token.data()); + return false; + } + netvc->set_proxy_protocol_src_port(port); + + // Next is the TCP destination port represented as a decimal number in the range of [0..65535] inclusive. + // Final trailer is CR LF so split at CR. + token = hdr.split_prefix_at('\r'); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Destination Port", static_cast(token.size()), token.data()); + if (0 == (port = ts::svtoi(token))) { + Debug("proxyprotocol_v1", "proxy_protov1_parse: dst port [%d] token [%.*s] failed to parse", port, + static_cast(token.size()), token.data()); + return false; + } + netvc->set_proxy_protocol_dst_port(port); + + netvc->set_proxy_protocol_version(NetVConnection::ProxyProtocolVersion::V1); + + return true; +} diff --git a/iocore/net/ProxyProtocol.h b/iocore/net/ProxyProtocol.h new file mode 100644 index 00000000000..b561205f05c --- /dev/null +++ b/iocore/net/ProxyProtocol.h @@ -0,0 +1,55 @@ +/** @file + + PROXY Protocol + + See: https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#ifndef ProxyProtocol_H_ +#define ProxyProtocol_H_ + +#include "tscore/ink_defs.h" +#include "tscore/ink_memory.h" +#include +#include +#include "I_VConnection.h" +#include "I_NetVConnection.h" +#include "I_IOBuffer.h" + +// http://www.haproxy.org/download/1.8/doc/proxy-protocol.txt + +extern bool proxy_protov1_parse(NetVConnection *, ts::TextView hdr); +extern bool ssl_has_proxy_v1(NetVConnection *, char *, int64_t *); +extern bool http_has_proxy_v1(IOBufferReader *, NetVConnection *); + +const char *const PROXY_V1_CONNECTION_PREFACE = "\x50\x52\x4F\x58\x59"; +const char *const PROXY_V2_CONNECTION_PREFACE = "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A\x02"; + +const size_t PROXY_V1_CONNECTION_PREFACE_LEN = strlen(PROXY_V1_CONNECTION_PREFACE); // 5 +const size_t PROXY_V2_CONNECTION_PREFACE_LEN = 13; + +const size_t PROXY_V1_CONNECTION_HEADER_LEN_MIN = 15; +const size_t PROXY_V2_CONNECTION_HEADER_LEN_MIN = 16; + +const size_t PROXY_V1_CONNECTION_HEADER_LEN_MAX = 108; +const size_t PROXY_V2_CONNECTION_HEADER_LEN_MAX = 16; + +#endif /* ProxyProtocol_H_ */ diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 2cdeca0ad63..b46ff5f90fb 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -41,6 +41,7 @@ #include "P_SSLCertLookup.h" #include "SSLSessionCache.h" #include +#include int SSLConfig::configid = 0; int SSLCertificateConfig::configid = 0; @@ -57,6 +58,7 @@ bool SSLConfigParams::session_cache_skip_on_lock_contention = false; size_t SSLConfigParams::session_cache_max_bucket_size = 100; init_ssl_ctx_func SSLConfigParams::init_ssl_ctx_cb = nullptr; load_ssl_file_func SSLConfigParams::load_ssl_file_cb = nullptr; +IpMap *SSLConfigParams::proxy_protocol_ipmap = nullptr; int SSLConfigParams::async_handshake_enabled = 0; char *SSLConfigParams::engine_conf_file = nullptr; @@ -84,6 +86,12 @@ SSLConfigParams::~SSLConfigParams() ink_mutex_destroy(&ctxMapLock); } +void +SSLConfigInit(IpMap *global) +{ + SSLConfigParams::proxy_protocol_ipmap = global; +} + void SSLConfigParams::reset() { diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 1e5dccf8cc2..86cc813a1a1 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -34,6 +34,8 @@ #include "P_SSLClientUtils.h" #include "P_SSLSNI.h" #include "HttpTunnel.h" +#include "ProxyProtocol.h" +#include #include #include @@ -366,6 +368,49 @@ SSLNetVConnection::read_raw_data() } NET_SUM_DYN_STAT(net_read_bytes_stat, r); + IpMap *pp_ipmap; + pp_ipmap = SSLConfigParams::proxy_protocol_ipmap; + + if (this->get_is_proxy_protocol()) { + Debug("proxyprotocol", "[SSLNetVConnection::read_raw_data] proxy protocol is enabled on this port"); + if (pp_ipmap->count() > 0) { + Debug("proxyprotocol", + "[SSLNetVConnection::read_raw_data] proxy protocol has a configured whitelist of trusted IPs - checking"); + + // At this point, using get_remote_addr() will return the ip of the + // proxy source IP, not the Proxy Protocol client ip. Since we are + // checking the ip of the actual source of this connection, this is + // what we want now. + void *payload = nullptr; + if (!pp_ipmap->contains(get_remote_addr(), &payload)) { + Debug("proxyprotocol", + "[SSLNetVConnection::read_raw_data] proxy protocol src IP is NOT in the configured whitelist of trusted IPs - " + "closing connection"); + r = -ENOTCONN; // Need a quick close/exit here to refuse the connection!!!!!!!!! + goto proxy_protocol_bypass; + } else { + char new_host[INET6_ADDRSTRLEN]; + Debug("proxyprotocol", "[SSLNetVConnection::read_raw_data] Source IP [%s] is in the trusted whitelist for proxy protocol", + ats_ip_ntop(this->get_remote_addr(), new_host, sizeof(new_host))); + } + } else { + Debug("proxyprotocol", + "[SSLNetVConnection::read_raw_data] proxy protocol DOES NOT have a configured whitelist of trusted IPs but " + "proxy protocol is ernabled on this port - processing all connections"); + } + + if (ssl_has_proxy_v1(this, buffer, &r)) { + Debug("proxyprotocol", "[SSLNetVConnection::read_raw_data] ssl has proxy_v1 header"); + set_remote_addr(get_proxy_protocol_src_addr()); + } else { + Debug("proxyprotocol", + "[SSLNetVConnection::read_raw_data] proxy protocol was enabled, but required header was not present in the " + "transaction - closing connection"); + } + } // end of Proxy Protocol processing + +proxy_protocol_bypass: + if (r > 0) { this->handShakeBuffer->fill(r); diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index 9c3e40fb72d..b730c86ca77 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -115,6 +115,7 @@ net_accept(NetAccept *na, void *ep, bool blockable) vc->submit_time = Thread::get_hrtime(); vc->action_ = *na->action_; vc->set_is_transparent(na->opt.f_inbound_transparent); + vc->set_is_proxy_protocol(na->opt.f_proxy_protocol); vc->set_context(NET_VCONNECTION_IN); #ifdef USE_EDGE_TRIGGER // Set the vc as triggered and place it in the read ready queue later in case there is already data on the socket. @@ -347,6 +348,7 @@ NetAccept::do_blocking_accept(EThread *t) vc->submit_time = Thread::get_hrtime(); vc->action_ = *action_; vc->set_is_transparent(opt.f_inbound_transparent); + vc->set_is_proxy_protocol(opt.f_proxy_protocol); vc->options.packet_mark = opt.packet_mark; vc->options.packet_tos = opt.packet_tos; vc->options.ip_family = opt.ip_family; @@ -495,6 +497,7 @@ NetAccept::acceptFastEvent(int event, void *ep) vc->submit_time = Thread::get_hrtime(); vc->action_ = *action_; vc->set_is_transparent(opt.f_inbound_transparent); + vc->set_is_proxy_protocol(opt.f_proxy_protocol); vc->options.packet_mark = opt.packet_mark; vc->options.packet_tos = opt.packet_tos; vc->options.ip_family = opt.ip_family; diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index 3d4771c401d..436e6faac37 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -51,6 +51,7 @@ NetProcessor::AcceptOptions::reset() packet_tos = 0; tfo_queue_length = 0; f_inbound_transparent = false; + f_proxy_protocol = false; return *this; } @@ -122,6 +123,10 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons Debug("http_tproxy", "Marked accept server %p on port %d as inbound transparent", na, opt.local_port); } + if (opt.f_proxy_protocol) { + Debug("http_tproxy", "Marked accept server %p on port %d for proxy protocol", na, opt.local_port); + } + int should_filter_int = 0; na->server.http_accept_filter = false; REC_ReadConfigInteger(should_filter_int, "proxy.config.net.defer_accept"); diff --git a/lib/records/I_RecHttp.h b/lib/records/I_RecHttp.h index 7e831fc590f..1c95ef72100 100644 --- a/lib/records/I_RecHttp.h +++ b/lib/records/I_RecHttp.h @@ -28,6 +28,7 @@ #include "ts/apidefs.h" #include "ts/apidefs.h" #include "tscore/ink_assert.h" +#include "tscore/IpMap.h" #include #include @@ -37,6 +38,11 @@ void RecHttpLoadIp(const char *name, ///< Name of value in configuration file. IpAddr &ip6 ///< [out] Ipv6 address. ); +/// Load up an IpMap with IP addresses from the configuration file. +void RecHttpLoadIpMap(const char *name, ///< Name of value in configuration file. + IpMap &ipmap ///< [out] IpMap. +); + /** A set of session protocols. This depends on using @c SessionProtocolNameRegistry to get the indices. */ @@ -239,6 +245,8 @@ struct HttpProxyPort { TransportType m_type; ///< Type of connection. in_port_t m_port; ///< Port on which to listen. uint8_t m_family; ///< IP address family. + /// True if proxy protocol is required on incoming requests. + bool m_proxy_protocol; /// True if inbound connects (from client) are transparent. bool m_inbound_transparent_p; /// True if outbound connections (to origin servers) are transparent. @@ -392,6 +400,7 @@ struct HttpProxyPort { static const char *const OPT_TRANSPARENT_FULL; ///< Full transparency. static const char *const OPT_TRANSPARENT_PASSTHROUGH; ///< Pass-through non-HTTP. static const char *const OPT_SSL; ///< SSL (experimental) + static const char *const OPT_PROXY_PROTO; ///< Proxy Protocol static const char *const OPT_PLUGIN; ///< Protocol Plugin handle (experimental) static const char *const OPT_BLIND_TUNNEL; ///< Blind tunnel. static const char *const OPT_COMPRESSED; ///< Compressed. diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index c7acb5e0f03..64193b8043f 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -30,6 +30,7 @@ #include "tscore/ink_inet.h" #include #include +#include SessionProtocolNameRegistry globalSessionProtocolNameRegistry; @@ -123,6 +124,30 @@ RecHttpLoadIp(const char *value_name, IpAddr &ip4, IpAddr &ip6) } } +void +RecHttpLoadIpMap(const char *value_name, IpMap &ipmap) +{ + char value[1024]; + IpAddr laddr; + IpAddr raddr; + void *payload = nullptr; + + if (REC_ERR_OKAY == RecGetRecordString(value_name, value, sizeof(value))) { + Debug("config", "RecHttpLoadIpMap: parsing the name [%s] and value [%s] to an IpMap", value_name, value); + Tokenizer tokens(", "); + int n_addrs = tokens.Initialize(value); + for (int i = 0; i < n_addrs; ++i) { + const char *val = tokens[i]; + + Debug("config", "RecHttpLoadIpMap: marking the value [%s] to an IpMap entry", val); + if (0 == ats_ip_range_parse(val, laddr, raddr)) { + ipmap.fill(laddr, raddr, payload); + } + } + } + Debug("config", "RecHttpLoadIpMap: parsed %zu IpMap entries", ipmap.count()); +} + const char *const HttpProxyPort::DEFAULT_VALUE = "8080"; const char *const HttpProxyPort::PORTS_CONFIG_NAME = "proxy.config.http.server_ports"; @@ -144,6 +169,7 @@ const char *const HttpProxyPort::OPT_TRANSPARENT_OUTBOUND = "tr-out"; const char *const HttpProxyPort::OPT_TRANSPARENT_FULL = "tr-full"; const char *const HttpProxyPort::OPT_TRANSPARENT_PASSTHROUGH = "tr-pass"; const char *const HttpProxyPort::OPT_SSL = "ssl"; +const char *const HttpProxyPort::OPT_PROXY_PROTO = "pp"; const char *const HttpProxyPort::OPT_PLUGIN = "plugin"; const char *const HttpProxyPort::OPT_BLIND_TUNNEL = "blind"; const char *const HttpProxyPort::OPT_COMPRESSED = "compressed"; @@ -176,6 +202,7 @@ HttpProxyPort::HttpProxyPort() m_type(TRANSPORT_DEFAULT), m_port(0), m_family(AF_INET), + m_proxy_protocol(false), m_inbound_transparent_p(false), m_outbound_transparent_p(false), m_transparent_passthrough(false), @@ -358,6 +385,8 @@ HttpProxyPort::processOptions(const char *opts) m_type = TRANSPORT_SSL; } else if (0 == strcasecmp(OPT_PLUGIN, item)) { m_type = TRANSPORT_PLUGIN; + } else if (0 == strcasecmp(OPT_PROXY_PROTO, item)) { + m_proxy_protocol = true; } else if (0 == strcasecmp(OPT_TRANSPARENT_INBOUND, item)) { #if TS_USE_TPROXY m_inbound_transparent_p = true; @@ -558,6 +587,10 @@ HttpProxyPort::print(char *out, size_t n) return n; } + if (m_proxy_protocol) { + zret += snprintf(out + zret, n - zret, ":%s", OPT_PROXY_PROTO); + } + if (m_outbound_transparent_p && m_inbound_transparent_p) { zret += snprintf(out + zret, n - zret, ":%s", OPT_TRANSPARENT_FULL); } else if (m_inbound_transparent_p) { diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index d23c913ae60..0d355242fc2 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -1057,6 +1057,10 @@ LocalManager::bindProxyPort(HttpProxyPort &port) mgmt_fatal(0, "[bindProxyPort] Unable to set socket options: %d : %s\n", port.m_port, strerror(errno)); } + if (port.m_proxy_protocol) { + Debug("lm", "[bindProxyPort] Proxy Protocol enabled"); + } + if (port.m_inbound_transparent_p) { #if TS_USE_TPROXY Debug("http_tproxy", "Listen port %d inbound transparency enabled.", port.m_port); diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 7c329f1fab0..d4a212262ad 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -534,6 +534,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http.insert_forwarded", RECD_STRING, "none", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , + {RECT_CONFIG, "proxy.config.http.proxy_protocol_whitelist", RECD_STRING, "none", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , {RECT_CONFIG, "proxy.config.http.insert_age_in_response", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , {RECT_CONFIG, "proxy.config.http.enable_http_stats", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc index 44d40765e7e..3e488ccc050 100644 --- a/proxy/PluginVC.cc +++ b/proxy/PluginVC.cc @@ -957,6 +957,12 @@ PluginVC::set_remote_addr() } } +void +PluginVC::set_remote_addr(const sockaddr * /* new_sa ATS_UNUSED */) +{ + return; +} + int PluginVC::set_tcp_init_cwnd(int /* init_cwnd ATS_UNUSED */) { diff --git a/proxy/PluginVC.h b/proxy/PluginVC.h index e3603829f60..1e879efcced 100644 --- a/proxy/PluginVC.h +++ b/proxy/PluginVC.h @@ -101,6 +101,7 @@ class PluginVC : public NetVConnection, public PluginIdentity SOCKET get_socket() override; void set_local_addr() override; void set_remote_addr() override; + void set_remote_addr(const sockaddr *) override; int set_tcp_init_cwnd(int init_cwnd) override; int set_tcp_congestion_control(int) override; diff --git a/proxy/ProtocolProbeSessionAccept.cc b/proxy/ProtocolProbeSessionAccept.cc index 8d332941d6d..31b6e52ad4d 100644 --- a/proxy/ProtocolProbeSessionAccept.cc +++ b/proxy/ProtocolProbeSessionAccept.cc @@ -25,6 +25,8 @@ #include "I_Machine.h" #include "ProtocolProbeSessionAccept.h" #include "http2/HTTP2.h" +#include "ProxyProtocol.h" +#include "I_NetVConnection.h" static bool proto_is_http2(IOBufferReader *reader) @@ -90,6 +92,44 @@ struct ProtocolProbeTrampoline : public Continuation, public ProtocolProbeSessio goto done; } + // if proxy_protocol is enabled via port descriptor AND the src IP is in + // the trusted whitelist for proxy protocol, then check to see if it is + // present + + IpMap *pp_ipmap; + pp_ipmap = probeParent->proxy_protocol_ipmap; + + if (netvc->get_is_proxy_protocol()) { + Debug("proxyprotocol", "ioCompletionEvent: proxy protocol is enabled on this port"); + if (pp_ipmap->count() > 0) { + Debug("proxyprotocol", "ioCompletionEvent: proxy protocol has a configured whitelist of trusted IPs - checking"); + void *payload = nullptr; + if (!pp_ipmap->contains(netvc->get_remote_addr(), &payload)) { + Debug("proxyprotocol", + "ioCompletionEvent: proxy protocol src IP is NOT in the configured whitelist of trusted IPs - closing connection"); + goto done; + } else { + char new_host[INET6_ADDRSTRLEN]; + Debug("proxyprotocol", "ioCompletionEvent: Source IP [%s] is trusted in the whitelist for proxy protocol", + ats_ip_ntop(netvc->get_remote_addr(), new_host, sizeof(new_host))); + } + } else { + Debug("proxyprotocol", + "ioCompletionEvent: proxy protocol DOES NOT have a configured whitelist of trusted IPs but proxy protocol is " + "ernabled on this port - processing all connections"); + } + + if (http_has_proxy_v1(reader, netvc)) { + Debug("proxyprotocol", "ioCompletionEvent: http has proxy_v1 header"); + netvc->set_remote_addr(netvc->get_proxy_protocol_src_addr()); + } else { + Debug("proxyprotocol", + "ioCompletionEvent: proxy protocol was enabled, but required header was not present in the transaction - " + "closing connection"); + goto done; + } + } // end of Proxy Protocol processing + if (proto_is_http2(reader)) { key = PROTO_HTTP2; } else { diff --git a/proxy/ProtocolProbeSessionAccept.h b/proxy/ProtocolProbeSessionAccept.h index 8ba84201760..c58dbe31bd2 100644 --- a/proxy/ProtocolProbeSessionAccept.h +++ b/proxy/ProtocolProbeSessionAccept.h @@ -53,6 +53,8 @@ class ProtocolProbeSessionAccept : public SessionAccept, public ProtocolProbeSes ProtocolProbeSessionAccept(const ProtocolProbeSessionAccept &) = delete; // disabled ProtocolProbeSessionAccept &operator=(const ProtocolProbeSessionAccept &) = delete; // disabled + IpMap *proxy_protocol_ipmap = nullptr; + private: int mainEvent(int event, void *netvc) override; diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index 887fe5e2b86..5521233bffc 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -961,6 +961,7 @@ load_negative_caching_var(RecRecord const *r, void *cookie) void HttpConfig::startup() { + extern void SSLConfigInit(IpMap * map); http_rsb = RecAllocateRawStatBlock((int)http_stat_count); register_stat_callbacks(); @@ -978,6 +979,8 @@ HttpConfig::startup() RecHttpLoadIp("proxy.local.incoming_ip_to_bind", c.inbound_ip4, c.inbound_ip6); RecHttpLoadIp("proxy.local.outgoing_ip_to_bind", c.outbound_ip4, c.outbound_ip6); + RecHttpLoadIpMap("proxy.config.http.proxy_protocol_whitelist", c.config_proxy_protocol_ipmap); + SSLConfigInit(&c.config_proxy_protocol_ipmap); HttpEstablishStaticConfigLongLong(c.server_max_connections, "proxy.config.http.server_max_connections"); HttpEstablishStaticConfigLongLong(c.max_websocket_connections, "proxy.config.http.websocket.max_number_of_connections"); diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index 59f0faf7bfa..61092e9b645 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -836,6 +836,8 @@ struct HttpConfigParams : public ConfigInfo { public: IpAddr inbound_ip4, inbound_ip6; IpAddr outbound_ip4, outbound_ip6; + IpAddr proxy_protocol_ip4, proxy_protocol_ip6; + IpMap config_proxy_protocol_ipmap; MgmtInt server_max_connections = 0; MgmtInt origin_min_keep_alive_connections = 0; // TODO: This one really ought to be overridable, but difficult right now. diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index a342e276f31..fc68a9a67bb 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -160,6 +160,7 @@ make_net_accept_options(const HttpProxyPort *port, unsigned nthreads) net.f_inbound_transparent = port->m_inbound_transparent_p; net.ip_family = port->m_family; net.local_port = port->m_port; + net.f_proxy_protocol = port->m_proxy_protocol; if (port->m_inbound_ip.isValid()) { net.local_ip = port->m_inbound_ip; @@ -169,7 +170,6 @@ make_net_accept_options(const HttpProxyPort *port, unsigned nthreads) net.local_ip = HttpConfig::m_master.inbound_ip4; } } - return net; } @@ -209,6 +209,7 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor, HttpProxyPort &port, unsigned ProtocolProbeSessionAccept *probe = new ProtocolProbeSessionAccept(); HttpSessionAccept *http = nullptr; // don't allocate this unless it will be used. probe->proxyPort = &port; + probe->proxy_protocol_ipmap = &HttpConfig::m_master.config_proxy_protocol_ipmap; if (port.m_session_protocol_preference.intersects(HTTP_PROTOCOL_SET)) { http = new HttpSessionAccept(accept_opt); diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 2c06b77c909..d97a516f5a4 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -5571,7 +5571,12 @@ HttpTransact::initialize_state_variables_from_request(State *s, HTTPHdr *obsolet ats_ip_copy(&s->request_data.src_ip, &s->client_info.src_addr); memset(&s->request_data.dest_ip, 0, sizeof(s->request_data.dest_ip)); if (vc) { - s->request_data.incoming_port = vc->get_local_port(); + s->request_data.incoming_port = vc->get_local_port(); + s->pp_info.proxy_protocol_version = vc->get_proxy_protocol_version(); + if (s->pp_info.proxy_protocol_version != NetVConnection::ProxyProtocolVersion::UNDEFINED) { + ats_ip_copy(s->pp_info.src_addr, vc->pp_info.src_addr); + ats_ip_copy(s->pp_info.dst_addr, vc->pp_info.dst_addr); + } } s->request_data.xact_start = s->client_request_time; s->request_data.api_info = &s->api_info; diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 031a0a0a2cb..082e9cdd054 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -921,6 +921,9 @@ class HttpTransact } internal_msg_buffer_size = 0; } + + NetVConnection::ProxyProtocol pp_info; + }; // End of State struct. static void HandleBlindTunnel(State *s); From eeca549e927da5937f38640d57cef67a486f000a Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Mon, 8 Oct 2018 23:00:12 +0000 Subject: [PATCH 136/526] slice plugin, intercept handler that uses cache_range_request --- .gitignore | 1 + doc/admin-guide/plugins/slice.en.rst | 170 +++++++++ plugins/Makefile.am | 1 + plugins/experimental/slice/Config.cc | 153 ++++++++ plugins/experimental/slice/Config.h | 40 ++ plugins/experimental/slice/ContentRange.cc | 55 +++ plugins/experimental/slice/ContentRange.h | 53 +++ plugins/experimental/slice/Data.cc | 65 ++++ plugins/experimental/slice/Data.h | 125 +++++++ plugins/experimental/slice/HttpHeader.cc | 340 +++++++++++++++++ plugins/experimental/slice/HttpHeader.h | 205 +++++++++++ plugins/experimental/slice/Makefile.inc | 63 ++++ plugins/experimental/slice/Makefile.tsxs | 64 ++++ plugins/experimental/slice/README.md | 84 +++++ plugins/experimental/slice/Range.cc | 187 ++++++++++ plugins/experimental/slice/Range.h | 74 ++++ plugins/experimental/slice/Stage.h | 149 ++++++++ plugins/experimental/slice/client.cc | 227 ++++++++++++ plugins/experimental/slice/client.h | 34 ++ plugins/experimental/slice/intercept.cc | 88 +++++ plugins/experimental/slice/intercept.h | 23 ++ plugins/experimental/slice/response.cc | 110 ++++++ plugins/experimental/slice/response.h | 28 ++ plugins/experimental/slice/server.cc | 342 ++++++++++++++++++ plugins/experimental/slice/server.h | 37 ++ plugins/experimental/slice/slice.cc | 205 +++++++++++ plugins/experimental/slice/slice.h | 54 +++ plugins/experimental/slice/slice_test.cc | 195 ++++++++++ plugins/experimental/slice/transfer.cc | 105 ++++++ plugins/experimental/slice/transfer.h | 34 ++ .../slice/unit-tests/slice_test.cc | 181 +++++++++ .../slice/unit-tests/test_config.cc | 70 ++++ .../slice/unit-tests/test_content_range.cc | 80 ++++ .../slice/unit-tests/test_range.cc | 99 +++++ 34 files changed, 3741 insertions(+) create mode 100644 doc/admin-guide/plugins/slice.en.rst create mode 100644 plugins/experimental/slice/Config.cc create mode 100644 plugins/experimental/slice/Config.h create mode 100644 plugins/experimental/slice/ContentRange.cc create mode 100644 plugins/experimental/slice/ContentRange.h create mode 100644 plugins/experimental/slice/Data.cc create mode 100644 plugins/experimental/slice/Data.h create mode 100644 plugins/experimental/slice/HttpHeader.cc create mode 100644 plugins/experimental/slice/HttpHeader.h create mode 100644 plugins/experimental/slice/Makefile.inc create mode 100644 plugins/experimental/slice/Makefile.tsxs create mode 100644 plugins/experimental/slice/README.md create mode 100644 plugins/experimental/slice/Range.cc create mode 100644 plugins/experimental/slice/Range.h create mode 100644 plugins/experimental/slice/Stage.h create mode 100644 plugins/experimental/slice/client.cc create mode 100644 plugins/experimental/slice/client.h create mode 100644 plugins/experimental/slice/intercept.cc create mode 100644 plugins/experimental/slice/intercept.h create mode 100644 plugins/experimental/slice/response.cc create mode 100644 plugins/experimental/slice/response.h create mode 100644 plugins/experimental/slice/server.cc create mode 100644 plugins/experimental/slice/server.h create mode 100644 plugins/experimental/slice/slice.cc create mode 100644 plugins/experimental/slice/slice.h create mode 100644 plugins/experimental/slice/slice_test.cc create mode 100644 plugins/experimental/slice/transfer.cc create mode 100644 plugins/experimental/slice/transfer.h create mode 100644 plugins/experimental/slice/unit-tests/slice_test.cc create mode 100644 plugins/experimental/slice/unit-tests/test_config.cc create mode 100644 plugins/experimental/slice/unit-tests/test_content_range.cc create mode 100644 plugins/experimental/slice/unit-tests/test_range.cc diff --git a/.gitignore b/.gitignore index acffd829f9a..f721723dc9a 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,7 @@ proxy/logging/test_LogUtils plugins/header_rewrite/header_rewrite_test plugins/experimental/esi/*_test +plugins/experimental/slice/test_* plugins/experimental/sslheaders/test_sslheaders plugins/s3_auth/test_s3auth diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst new file mode 100644 index 00000000000..34a219e6b4d --- /dev/null +++ b/doc/admin-guide/plugins/slice.en.rst @@ -0,0 +1,170 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _admin-plugins-slice: + +Slice Plugin +*************** + +This plugin takes client requests and breaks them up into +successive aligned block requests. This supports both +whole asset and single range requests. + +Purpose +======= + +This slice plugin, along with the `cache_range_requests` +plugin allows the following: + +- Fulfill arbitrary range requests by fetching a minimum + number of cacheable aligned blocks to fulfill the request. +- Breaks up very large assets into much smaller cache + blocks that can be spread across multiple storage + devices and within cache groups. + +Configuration +============ + +This plugin is intended for use as a remap plugin and is +configured in :file:`remap.config`. + +Or preferably per remap rule in :file:`remap.config`:: + + map http://ats/ http://parent/ @plugin=slice.so \ + @plugin=cache_range_requests.so + +In this case, the plugin will use the default behaviour: + +- Fulfill whole file or range requests by requesting cacheable + block aligned ranges from the parent and assemble them + into client responses, either 200 or 206 depending on the + client request. +- Default block size is 1mb (1048576 bytes). +- This plugin depends on the cache_range_requests plugin + to perform actual parent fetching and block caching + and If-* conditional header evaluations. + +Plugin Options +-------------- + +Slice block sizes can specified using the blockbytes parameter:: + + @plugin=slice.so @pparam=blockbytes:1000000 @cache_range_requests.so + +In adition to bytes, 'k', 'm' and 'g' suffixes may be used for +kilobytes, megabytes and gigabytes:: + + @plugin=slice.so @pparam=blockbytes:5m @cache_range_requests.so + @plugin=slice.so @pparam=blockbytes:512k @cache_range_requests.so + @plugin=slice.so @pparam=blockbytes:32m @cache_range_requests.so + +paramater ``blockbytes`` is checked to be between 32kb and 32mb +inclusive. + +For testing and extreme purposes the parameter ``bytesover`` may +be used instead which is unchecked:: + + @plugin=slice.so @pparam=bytesover:1G @cache_range_requests.so + @plugin=slice.so @pparam=bytesover:13 @cache_range_requests.so + +After modifying :file:`remap.config`, restart or reload traffic server +(sudo traffic_ctl config reload) or (sudo traffic_ctl server restart) +to activate the new configuration values. + +Implementation Notes +==================== + +This slice plugin is by no means a best solution for adding +blocking support to ATS. + +The slice plugin as is designed to provide a basic capability to block +requests for arbitrary range requests and for blocking large assets for +ease of caching. + +Slice *ONLY* handles slicing up requests into blocks, it delegates +actual caching and fetching to the cache_range_requests.so plugin. + +Plugin Function +--------------- + +Below is a quick functional outline of how a request is served +by a remap rule containing the Slice plugin with cache_range_requests: + +For each client request that comes in all remap plugins are run up +until the slice plugin is hit. If the slice plugin *can* be run (ie: +GET request) it will handle the request and STOP any further plugins +from executing. + +At this point the request is sliced into 1 or more blocks by +adding in range request headers ("Range: bytes="). A special +header X-Slicer-Info header is added and the pristine URL is +restored. + +For each of these blocks separate sequential TSHttpConnect(s) are made +back into the front end of ATS and all of the remap plugins are rerun. +Slice skips the remap due to presense of the X-Slicer-Info header and +allows cache_range_requests.so to serve the slice block back to Slice +either via cache OR parent request. + +Slice assembles a header based on the first slice block response and +sends it to the client. If necessary it then skips over bytes in +the first block and starts sending byte content, examining each +block header and sends its bytes to the client until the client +request is satisfied. + +Any extra bytes at the end of the last block are consumed by +the the Slice plugin to allow cache_range_requests to finish +the block fetch to ensure the block is cached. + +Important Notes +=============== + +This plugin also assumes that the content requested is cacheable. + +Any first block server response that is not a 206 is passed directly +down to the client. If that response is a '200' only the first +portion of the response is passed back and the transaction is closed. + +Only the first server response block is used to evaluate any "If-" +conditional headers. Subsequent server slice block requests +remove these headers. + +The only 416 response that this plugin handles itself is if the +requested range is inside the last slice block but past the end of +the asset contents. Other 416 responses are handled by the parents. + +If a client aborts mid transaction the current slice block continues to +be read from the server until it is complete to ensure that the block +is cached. + +Slice *always* makes ``blockbytes`` sized requests which are handled +by cache_range_requests. The parent will trim those requests to +account for the asset Content-Length so only the appropriate number +of bytes are actually transferred and cached. + +Current Limitations +=================== + +By restoring the prisine Url the plugin as it works today reuses the +same remap rule for each slice block. This is wasteful in that it reruns +the previous remap rules, and those remap rules must be smart enough to +check for the existence of any headers they may have created the +first time they have were visited. + +Since the Slice plugin is written as an intercept handler it loses the +ability to use state machine hooks and transaction states. + diff --git a/plugins/Makefile.am b/plugins/Makefile.am index e1d0dadc26d..09e814a8034 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -73,6 +73,7 @@ include experimental/mp4/Makefile.inc include experimental/multiplexer/Makefile.inc include experimental/remap_purge/Makefile.inc include experimental/server_push_preload/Makefile.inc +include experimental/slice/Makefile.inc include experimental/sslheaders/Makefile.inc include experimental/stale_while_revalidate/Makefile.inc include experimental/stream_editor/Makefile.inc diff --git a/plugins/experimental/slice/Config.cc b/plugins/experimental/slice/Config.cc new file mode 100644 index 00000000000..c8b51b0496e --- /dev/null +++ b/plugins/experimental/slice/Config.cc @@ -0,0 +1,153 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Config.h" + +#include +#include +#include +#include +#include + +int64_t +Config::bytesFrom(std::string const &valstr) +{ + char const *const nptr = valstr.c_str(); + char *endptr = nullptr; + int64_t blockbytes = strtoll(nptr, &endptr, 10); + + if (nullptr != endptr && nptr < endptr) { + size_t const dist = endptr - nptr; + if (dist < valstr.size() && 0 <= blockbytes) { + switch (tolower(*endptr)) { + case 'g': + blockbytes *= ((int64_t)1024 * (int64_t)1024 * (int64_t)1024); + break; + case 'm': + blockbytes *= ((int64_t)1024 * (int64_t)1024); + break; + case 'k': + blockbytes *= (int64_t)1024; + break; + default: + break; + } + } + } + + if (blockbytes < 0) { + blockbytes = 0; + } + + return blockbytes; +} + +bool +Config::fromArgs(int const argc, char const *const argv[], char *const errbuf, int const errbuf_size) +{ +#if !defined(SLICE_UNIT_TEST) + DEBUG_LOG("Number of arguments: %d", argc); + for (int index = 0; index < argc; ++index) { + DEBUG_LOG("args[%d] = %s", index, argv[index]); + } +#endif + + std::map keyvals; + + static std::string const bbstr(blockbytesstr); + static std::string const bostr(bytesoverstr); + + // collect all args + for (int index = 0; index < argc; ++index) { + std::string const argstr = argv[index]; + + std::size_t const spos = argstr.find_first_of(":"); + if (spos != std::string::npos) { + std::string key = argstr.substr(0, spos); + std::string val = argstr.substr(spos + 1); + + if (!key.empty()) { + std::for_each(key.begin(), key.end(), [](char &ch) { ch = tolower(ch); }); + + // blockbytes and bytesover collide + if (bbstr == key) { + keyvals.erase(bostr); + } else if (bostr == key) { + keyvals.erase(bbstr); + } + + keyvals[std::move(key)] = std::move(val); + } + } + } + + std::map::const_iterator itfind; + + // blockbytes checked range string + itfind = keyvals.find(bbstr); + if (keyvals.end() != itfind) { + std::string val = itfind->second; + if (!val.empty()) { + int64_t const blockbytes = bytesFrom(val); + + if (blockbytes < blockbytesmin || blockbytesmax < blockbytes) { +#if !defined(SLICE_UNIT_TEST) + DEBUG_LOG("Block Bytes %" PRId64 " outside checked limits %" PRId64 "-%" PRId64, blockbytes, blockbytesmin, blockbytesmax); + DEBUG_LOG("Block Bytes kept at %" PRId64, m_blockbytes); +#endif + } else { +#if !defined(SLICE_UNIT_TEST) + DEBUG_LOG("Block Bytes set to %" PRId64, blockbytes); +#endif + m_blockbytes = blockbytes; + } + } + + keyvals.erase(itfind); + } + + // bytesover unchecked range string + itfind = keyvals.find(bostr); + if (keyvals.end() != itfind) { + std::string val = itfind->second; + if (!val.empty()) { + int64_t const bytesover = bytesFrom(val); + + if (bytesover <= 0) { +#if !defined(SLICE_UNIT_TEST) + DEBUG_LOG("Bytes Over %" PRId64 " <= 0", bytesover); + DEBUG_LOG("Block Bytes kept at %" PRId64, m_blockbytes); +#endif + } else { +#if !defined(SLICE_UNIT_TEST) + DEBUG_LOG("Block Bytes set to %" PRId64, bytesover); +#endif + m_blockbytes = bytesover; + } + } + keyvals.erase(itfind); + } + + for (std::map::const_iterator itkv(keyvals.cbegin()); keyvals.cend() != itkv; ++itkv) { +#if !defined(SLICE_UNIT_TEST) + ERROR_LOG("Unhandled pparam %s", itkv->first.c_str()); +#endif + } + + return true; +} diff --git a/plugins/experimental/slice/Config.h b/plugins/experimental/slice/Config.h new file mode 100644 index 00000000000..e107f3f0167 --- /dev/null +++ b/plugins/experimental/slice/Config.h @@ -0,0 +1,40 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "slice.h" + +#include + +// Data Structures and Classes +struct Config { + static int64_t const blockbytesmin = 1024 * 256; // 256KB + static int64_t const blockbytesmax = 1024 * 1024 * 32; // 32MB + static int64_t const blockbytesdefault = 1024 * 1024; // 1MB + + static constexpr char const *const blockbytesstr = "blockbytes"; + static constexpr char const *const bytesoverstr = "bytesover"; + + int64_t m_blockbytes{blockbytesdefault}; + + // Last one wins + bool fromArgs(int const argc, char const *const argv[], char *const errbuf, int const errbuf_size); + + static int64_t bytesFrom(std::string const &valstr); +}; diff --git a/plugins/experimental/slice/ContentRange.cc b/plugins/experimental/slice/ContentRange.cc new file mode 100644 index 00000000000..c9d9edde3de --- /dev/null +++ b/plugins/experimental/slice/ContentRange.cc @@ -0,0 +1,55 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "ContentRange.h" + +#include +#include + +static char const *const format = "bytes %" PRId64 "-%" PRId64 "/%" PRId64; + +bool +ContentRange::fromStringClosed(char const *const valstr) +{ + int const fields = sscanf(valstr, format, &m_beg, &m_end, &m_length); + + if (3 == fields && m_beg <= m_end) { + m_end += 1; + } else { + m_beg = m_end = m_length = -1; + } + + return isValid(); +} + +bool +ContentRange::toStringClosed(char *const rangestr, int *const rangelen) const +{ + if (!isValid()) { + if (0 < *rangelen) { + rangestr[0] = '\0'; + } + *rangelen = 0; + return false; + } + + int const lenin = *rangelen; + *rangelen = snprintf(rangestr, lenin, format, m_beg, (m_end - 1), m_length); + + return (0 < *rangelen && *rangelen < lenin); +} diff --git a/plugins/experimental/slice/ContentRange.h b/plugins/experimental/slice/ContentRange.h new file mode 100644 index 00000000000..0ca05588dc9 --- /dev/null +++ b/plugins/experimental/slice/ContentRange.h @@ -0,0 +1,53 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +/** + represents value parsed from a blocked Content-Range reponse header field. + Range is converted from closed range into a half open range for. + */ +struct ContentRange { + int64_t m_beg; + int64_t m_end; // half open + int64_t m_length; // full content length + + ContentRange() : m_beg(-1), m_end(-1), m_length(-1) {} + explicit ContentRange(int64_t const begin, int64_t const end, int64_t const len) : m_beg(begin), m_end(end), m_length(len) {} + bool + isValid() const + { + return 0 <= m_beg && m_beg < m_end && m_end <= m_length; + } + + /** parsed from a Content-Range field + */ + bool fromStringClosed(char const *const valstr); + + /** usable for Content-Range field + */ + bool toStringClosed(char *const rangestr, int *const rangelen) const; + + int64_t + rangeSize() const + { + return m_end - m_beg; + } +}; diff --git a/plugins/experimental/slice/Data.cc b/plugins/experimental/slice/Data.cc new file mode 100644 index 00000000000..a901867808b --- /dev/null +++ b/plugins/experimental/slice/Data.cc @@ -0,0 +1,65 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Data.h" + +#include +#include +#include +#include +#include + +namespace +{ +std::mutex mutex; +int64_t inplay = 0; +std::unique_ptr thread; +} // namespace + +void +monitor() +{ + std::lock_guard guard(mutex); + // while (0 < inplay) + while (1) { + mutex.unlock(); + std::this_thread::sleep_for(std::chrono::seconds(10)); + std::cerr << "Inplay: " << inplay << std::endl; + mutex.lock(); + } + // thread.release(); +} + +void +incrData() +{ + std::lock_guard const guard(mutex); + if (!thread) { + thread.reset(new std::thread(monitor)); + } + + ++inplay; +} + +void +decrData() +{ + std::lock_guard const guard(mutex); + --inplay; + assert(0 <= inplay); +} diff --git a/plugins/experimental/slice/Data.h b/plugins/experimental/slice/Data.h new file mode 100644 index 00000000000..f7707196c72 --- /dev/null +++ b/plugins/experimental/slice/Data.h @@ -0,0 +1,125 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +#include "HttpHeader.h" +#include "Range.h" +#include "Stage.h" +#include "slice.h" + +#include + +void incrData(); + +void decrData(); + +struct Data { + Data(Data const &) = delete; + Data &operator=(Data const &) = delete; + + int64_t const m_blockbytes_config; // configured slice block size + sockaddr_storage m_client_ip; + + // for pristine url coming in + TSMBuffer m_urlbuffer{nullptr}; + TSMLoc m_urlloc{nullptr}; + + char m_hostname[8192]; + int m_hostlen; + char m_etag[8192]; + int m_etaglen; + char m_lastmodified[8192]; + int m_lastmodifiedlen; + + TSHttpStatus m_statustype; // 200 or 206 + + bool m_bail; // non 206/200 response + + Range m_req_range; // converted to half open interval + int64_t m_contentlen; + + int64_t m_blocknum; // block number to work on, -1 bad/stop + int64_t m_blockexpected; // body bytes expected + int64_t m_blockskip; // number of bytes to skip in this block + int64_t m_blockconsumed; // body bytes consumed + bool m_iseos; // server in EOS state + + int64_t m_bytestosend; // header + content bytes to send + int64_t m_bytessent; // number of bytes written to the client + + bool m_server_block_header_parsed; + bool m_server_first_header_parsed; + + Stage m_upstream; + Stage m_dnstream; + + HdrMgr m_req_hdrmgr; // manager for server request + HdrMgr m_resp_hdrmgr; // manager for client response + + TSHttpParser m_http_parser{nullptr}; //!< cached for reuse + + explicit Data(int64_t const blockbytes) + : m_blockbytes_config(blockbytes), + m_client_ip(), + m_urlbuffer(nullptr), + m_urlloc(nullptr), + m_hostlen(0), + m_etaglen(0), + m_lastmodifiedlen(0), + m_statustype(TS_HTTP_STATUS_NONE), + m_bail(false), + m_req_range(-1, -1), + m_contentlen(-1) + + , + m_blocknum(-1), + m_blockexpected(0), + m_blockskip(0), + m_blockconsumed(0), + m_iseos(false) + + , + m_bytestosend(0), + m_bytessent(0), + m_server_block_header_parsed(false), + m_server_first_header_parsed(false), + m_http_parser(nullptr) + { + // incrData(); + m_hostname[0] = '\0'; + m_lastmodified[0] = '\0'; + m_etag[0] = '\0'; + } + + ~Data() + { + // decrData(); + if (nullptr != m_urlbuffer) { + if (nullptr != m_urlloc) { + TSHandleMLocRelease(m_urlbuffer, TS_NULL_MLOC, m_urlloc); + } + TSMBufferDestroy(m_urlbuffer); + } + if (nullptr != m_http_parser) { + TSHttpParserDestroy(m_http_parser); + } + } +}; diff --git a/plugins/experimental/slice/HttpHeader.cc b/plugins/experimental/slice/HttpHeader.cc new file mode 100644 index 00000000000..0f1a78f001d --- /dev/null +++ b/plugins/experimental/slice/HttpHeader.cc @@ -0,0 +1,340 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "HttpHeader.h" + +#include "slice.h" + +#include +#include + +TSHttpType +HttpHeader::type() const +{ + if (isValid()) { + return TSHttpHdrTypeGet(m_buffer, m_lochdr); + } else { + return TS_HTTP_TYPE_UNKNOWN; + } +} + +TSHttpStatus +HttpHeader::status() const +{ + TSHttpStatus res = TS_HTTP_STATUS_NONE; + if (isValid()) { + res = TSHttpHdrStatusGet(m_buffer, m_lochdr); + } + return res; +} + +bool +HttpHeader::setStatus(TSHttpStatus const newstatus) +{ + if (!isValid()) { + return false; + } + + return TS_SUCCESS == TSHttpHdrStatusSet(m_buffer, m_lochdr, newstatus); +} + +bool +HttpHeader::setUrl(TSMBuffer const bufurl, TSMLoc const locurl) +{ + if (!isValid()) { + return false; + } + + TSMLoc locurlout; + TSReturnCode rcode = TSHttpHdrUrlGet(m_buffer, m_lochdr, &locurlout); + if (TS_SUCCESS != rcode) { + return false; + } + + // copy the url + rcode = TSUrlCopy(m_buffer, locurlout, bufurl, locurl); + + // set url active + if (TS_SUCCESS == rcode) { + rcode = TSHttpHdrUrlSet(m_buffer, m_lochdr, locurlout); + } + + TSHandleMLocRelease(m_buffer, TS_NULL_MLOC, locurlout); + + return TS_SUCCESS == rcode; +} + +bool +HttpHeader::setReason(char const *const valstr, int const vallen) +{ + if (isValid()) { + return TS_SUCCESS == TSHttpHdrReasonSet(m_buffer, m_lochdr, valstr, vallen); + } else { + return false; + } +} + +char const * +HttpHeader::getCharPtr(CharPtrGetFunc func, int *const len) const +{ + char const *res = nullptr; + if (isValid()) { + int reslen = 0; + res = func(m_buffer, m_lochdr, &reslen); + + if (nullptr != len) { + *len = reslen; + } + } + + return res; +} + +bool +HttpHeader::hasKey(char const *const key, int const keylen) const +{ + if (!isValid()) { + return false; + } + + TSMLoc const locfield(TSMimeHdrFieldFind(m_buffer, m_lochdr, key, keylen)); + if (nullptr != locfield) { + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + return true; + } + + return false; +} + +bool +HttpHeader::removeKey(char const *const keystr, int const keylen) +{ + if (!isValid()) { + return false; + } + + bool status = true; + + TSMLoc const locfield = TSMimeHdrFieldFind(m_buffer, m_lochdr, keystr, keylen); + if (nullptr != locfield) { + int const rcode = TSMimeHdrFieldRemove(m_buffer, m_lochdr, locfield); + status = (TS_SUCCESS == rcode); + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + } + + return status; +} + +bool +HttpHeader::valueForKey(char const *const keystr, int const keylen, char *const valstr, int *const vallen, int const index) const +{ + if (!isValid()) { + return false; + } + + bool status = false; + + TSMLoc const locfield = TSMimeHdrFieldFind(m_buffer, m_lochdr, keystr, keylen); + + if (nullptr != locfield) { + int getlen = 0; + char const *const getstr = TSMimeHdrFieldValueStringGet(m_buffer, m_lochdr, locfield, index, &getlen); + + int const valcap = *vallen; + if (nullptr != getstr && 0 < getlen && getlen < (valcap - 1)) { + char *const endp = stpncpy(valstr, getstr, getlen); + + *vallen = endp - valstr; + status = (*vallen < valcap); + + if (status) { + *endp = '\0'; + } + } + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + } else { + *vallen = 0; + } + + if (!status) { + } + + return status; +} + +bool +HttpHeader::setKeyVal(char const *const keystr, int const keylen, char const *const valstr, int const vallen, int const index) +{ + if (!isValid()) { + return false; + } + + bool status(false); + + TSMLoc locfield(TSMimeHdrFieldFind(m_buffer, m_lochdr, keystr, keylen)); + + if (nullptr != locfield) { + status = TS_SUCCESS == TSMimeHdrFieldValueStringSet(m_buffer, m_lochdr, locfield, index, valstr, vallen); + } else { + int rcode = TSMimeHdrFieldCreateNamed(m_buffer, m_lochdr, keystr, keylen, &locfield); + + if (TS_SUCCESS == rcode) { + rcode = TSMimeHdrFieldValueStringSet(m_buffer, m_lochdr, locfield, index, valstr, vallen); + if (TS_SUCCESS == rcode) { + rcode = TSMimeHdrFieldAppend(m_buffer, m_lochdr, locfield); + status = (TS_SUCCESS == rcode); + } + } + } + + if (nullptr != locfield) { + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + } + + return status; +} + +std::string +HttpHeader::toString() const +{ + std::string res; + + if (!isValid()) { + return ""; + } + + TSHttpType const htype(type()); + + switch (htype) { + case TS_HTTP_TYPE_REQUEST: { + res.append(method()); + + TSMLoc locurl = nullptr; + TSReturnCode const rcode = TSHttpHdrUrlGet(m_buffer, m_lochdr, &locurl); + if (TS_SUCCESS == rcode && nullptr != locurl) { + int urllen = 0; + char *const urlstr = TSUrlStringGet(m_buffer, locurl, &urllen); + res.append(" "); + res.append(urlstr, urllen); + TSfree(urlstr); + + TSHandleMLocRelease(m_buffer, m_lochdr, locurl); + } else { + res.append(" UnknownURL"); + } + + res.append(" HTTP/unparsed"); + } break; + + case TS_HTTP_TYPE_RESPONSE: { + char bufstr[1024]; + /* + int const version = TSHttpHdrVersionGet(m_buffer, m_lochdr); + snprintf(bufstr, 1023, "%d ", version); + res.append(bufstr); + */ + res.append("HTTP/unparsed"); + + int const status = TSHttpHdrStatusGet(m_buffer, m_lochdr); + snprintf(bufstr, 1023, " %d ", status); + res.append(bufstr); + + int reasonlen = 0; + char const *const hreason = reason(&reasonlen); + + res.append(hreason, reasonlen); + } break; + + default: + case TS_HTTP_TYPE_UNKNOWN: + res.append("UNKNOWN"); + break; + } + + res.append("\r\n"); + + int const numhdrs = TSMimeHdrFieldsCount(m_buffer, m_lochdr); + + for (int indexhdr = 0; indexhdr < numhdrs; ++indexhdr) { + TSMLoc const locfield = TSMimeHdrFieldGet(m_buffer, m_lochdr, indexhdr); + + int keylen = 0; + char const *const keystr = TSMimeHdrFieldNameGet(m_buffer, m_lochdr, locfield, &keylen); + + res.append(keystr, keylen); + res.append(": "); + int vallen = 0; + char const *const valstr = TSMimeHdrFieldValueStringGet(m_buffer, m_lochdr, locfield, -1, &vallen); + + res.append(valstr, vallen); + res.append("\r\n"); + + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + } + + res.append("\r\n"); + + return res; +} + +/////// HdrMgr + +TSParseResult +HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc) +{ + TSParseResult parse_res = TS_PARSE_CONT; + + if (nullptr == m_buffer) { + m_buffer = TSMBufferCreate(); + } + if (nullptr == m_lochdr) { + m_lochdr = TSHttpHdrCreate(m_buffer); + } + + int64_t read_avail = TSIOBufferReaderAvail(reader); + if (0 < read_avail) { + TSIOBufferBlock block = TSIOBufferReaderStart(reader); + int64_t consumed = 0; + + parse_res = TS_PARSE_CONT; + + while (nullptr != block && 0 < read_avail) { + int64_t blockbytes = 0; + char const *const bstart = TSIOBufferBlockReadStart(block, reader, &blockbytes); + + char const *ptr = bstart; + char const *endptr = ptr + blockbytes; + + parse_res = parsefunc(http_parser, m_buffer, m_lochdr, &ptr, endptr); + + int64_t const bytes_parsed(ptr - bstart); + + consumed += bytes_parsed; + read_avail -= bytes_parsed; + + if (TS_PARSE_CONT == parse_res) { + block = TSIOBufferBlockNext(block); + } else { + break; + } + } + TSIOBufferReaderConsume(reader, consumed); + } + + return parse_res; +} diff --git a/plugins/experimental/slice/HttpHeader.h b/plugins/experimental/slice/HttpHeader.h new file mode 100644 index 00000000000..35727612335 --- /dev/null +++ b/plugins/experimental/slice/HttpHeader.h @@ -0,0 +1,205 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +/** + An ATS Http header exists in a marshall buffer at a given location. + Unfortunately how that marshall buffer is created and how that + location is determined depends on where those buffers came from. + + A TSHttpTxn manages the buffer itself and creates a location which + has to be managed. + + A TSHttpParsed populates a created buffer that has had TSHttpHdrCreate + run against it which creates a location against it. End users + need to manage the created buffer, the location and invoke + TSHttpHdrDestroy. +*/ + +#include "ts/ts.h" + +#include + +static char const *const SLICER_MIME_FIELD_INFO = "X-Slicer-Info"; + +/** + Designed to be a cheap throwaway struct which allows a + consumer to make various calls to manipulate headers. +*/ +struct HttpHeader { + TSMBuffer const m_buffer; + TSMLoc const m_lochdr; + + explicit HttpHeader(TSMBuffer buffer, TSMLoc lochdr) : m_buffer(buffer), m_lochdr(lochdr) {} + bool + isValid() const + { + return nullptr != m_buffer && nullptr != m_lochdr; + } + + // TS_HTTP_TYPE_UNKNOWN, TS_HTTP_TYPE_REQUEST, TS_HTTP_TYPE_RESPONSE + TSHttpType type() const; + + TSHttpStatus status() const; + + bool setStatus(TSHttpStatus const newstatus); + + bool setUrl(TSMBuffer const bufurl, TSMLoc const locurl); + + typedef char const *(*CharPtrGetFunc)(TSMBuffer, TSMLoc, int *); + + // request method TS_HTTP_METHOD_* + char const * + method(int *const len = nullptr) const + { + return getCharPtr(TSHttpHdrMethodGet, len); + } + + // host + char const * + hostname(int *const len) const + { + return getCharPtr(TSHttpHdrHostGet, len); + } + + // response reason + char const * + reason(int *const len) const + { + return getCharPtr(TSHttpHdrReasonGet, len); + } + + bool setReason(char const *const valstr, int const vallen); + + bool hasKey(char const *const key, int const keylen) const; + + // returns false if header invalid or something went wrong with removal. + bool removeKey(char const *const key, int const keylen); + + bool valueForKey(char const *const keystr, int const keylen, + char *const valstr, // <-- return string value + int *const vallen, // <-- pass in capacity, returns len of string + int const index = -1 // sets all values + ) const; + + /** + Sets or adds a key/value + */ + bool setKeyVal(char const *const key, int const keylen, char const *const val, int const vallen, + int const index = -1 // sets all values + ); + + /** dump header into provided char buffer + */ + std::string toString() const; + +private: + /** + To be used with + TSHttpHdrMethodGet + TSHttpHdrHostGet + TSHttpHdrReasonGet + */ + char const *getCharPtr(CharPtrGetFunc func, int *const len) const; +}; + +struct TxnHdrMgr { + TxnHdrMgr(TxnHdrMgr const &) = delete; + TxnHdrMgr &operator=(TxnHdrMgr const &) = delete; + + TSMBuffer m_buffer{nullptr}; + TSMLoc m_lochdr{nullptr}; + + TxnHdrMgr() : m_buffer(nullptr), m_lochdr(nullptr) {} + ~TxnHdrMgr() + { + if (nullptr != m_lochdr) { + TSHandleMLocRelease(m_buffer, TS_NULL_MLOC, m_lochdr); + } + } + + typedef TSReturnCode (*HeaderGetFunc)(TSHttpTxn, TSMBuffer *, TSMLoc *); + /** use one of the following: + TSHttpTxnClientReqGet + TSHttpTxnClientRespGet + TSHttpTxnServerReqGet + TSHttpTxnServerRespGet + TSHttpTxnCachedReqGet + TSHttpTxnCachedRespGet + */ + + bool + populateFrom(TSHttpTxn const &txnp, HeaderGetFunc const &func) + { + return TS_SUCCESS == func(txnp, &m_buffer, &m_lochdr); + } + + bool + isValid() const + { + return nullptr != m_lochdr; + } +}; + +struct HdrMgr { + HdrMgr(HdrMgr const &) = delete; + HdrMgr &operator=(HdrMgr const &) = delete; + + TSMBuffer m_buffer{nullptr}; + TSMLoc m_lochdr{nullptr}; + + HdrMgr() : m_buffer(nullptr), m_lochdr(nullptr) {} + ~HdrMgr() + { + if (nullptr != m_buffer) { + if (nullptr != m_lochdr) { + TSHttpHdrDestroy(m_buffer, m_lochdr); + TSHandleMLocRelease(m_buffer, TS_NULL_MLOC, m_lochdr); + } + TSMBufferDestroy(m_buffer); + } + } + + void + resetHeader() + { + if (nullptr != m_buffer && nullptr != m_lochdr) { + TSHttpHdrDestroy(m_buffer, m_lochdr); + TSHandleMLocRelease(m_buffer, TS_NULL_MLOC, m_lochdr); + m_lochdr = nullptr; + } + } + + typedef TSParseResult (*HeaderParseFunc)(TSHttpParser, TSMBuffer, TSMLoc, char const **, char const *); + + /** Clear/create the parser before calling this and don't + use the parser on another header until done with this one. + use one of the following: + TSHttpHdrParseReq + TSHttpHdrParseResp + Call this multiple times if necessary. + */ + TSParseResult populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc); + + bool + isValid() const + { + return nullptr != m_lochdr; + } +}; diff --git a/plugins/experimental/slice/Makefile.inc b/plugins/experimental/slice/Makefile.inc new file mode 100644 index 00000000000..2f0f8f4b458 --- /dev/null +++ b/plugins/experimental/slice/Makefile.inc @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/slice/slice.la + +experimental_slice_slice_la_SOURCES = \ + experimental/slice/client.cc \ + experimental/slice/client.h \ + experimental/slice/Config.cc \ + experimental/slice/Config.h \ + experimental/slice/ContentRange.cc \ + experimental/slice/ContentRange.h \ + experimental/slice/Data.cc \ + experimental/slice/Data.h \ + experimental/slice/HttpHeader.cc \ + experimental/slice/HttpHeader.h \ + experimental/slice/intercept.cc \ + experimental/slice/intercept.h \ + experimental/slice/Range.cc \ + experimental/slice/Range.h \ + experimental/slice/response.cc \ + experimental/slice/response.h \ + experimental/slice/server.cc \ + experimental/slice/server.h \ + experimental/slice/slice.cc \ + experimental/slice/slice.h \ + experimental/slice/Stage.h \ + experimental/slice/transfer.cc \ + experimental/slice/transfer.h + +check_PROGRAMS += experimental/slice/test_content_range + +experimental_slice_test_content_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DSLICE_UNIT_TEST +experimental_slice_test_content_range_SOURCES = \ + experimental/slice/unit-tests/test_content_range.cc \ + experimental/slice/ContentRange.cc + +check_PROGRAMS += experimental/slice/test_range + +experimental_slice_test_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DSLICE_UNIT_TEST +experimental_slice_test_range_SOURCES = \ + experimental/slice/unit-tests/test_range.cc \ + experimental/slice/Range.cc + +check_PROGRAMS += experimental/slice/test_config + +experimental_slice_test_config_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DSLICE_UNIT_TEST +experimental_slice_test_config_SOURCES = \ + experimental/slice/unit-tests/test_config.cc \ + experimental/slice/Config.cc diff --git a/plugins/experimental/slice/Makefile.tsxs b/plugins/experimental/slice/Makefile.tsxs new file mode 100644 index 00000000000..b3b5d4f014d --- /dev/null +++ b/plugins/experimental/slice/Makefile.tsxs @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: slice + +slice_la_SOURCES = \ + Config.cc \ + ContentRange.cc \ + Data.cc \ + HttpHeader.cc \ + Range.cc \ + client.cc \ + intercept.cc \ + response.cc \ + server.cc \ + slice.cc \ + transfer.cc \ + +slice_la_HEADERS = \ + Config.h \ + ContentRange.h \ + Data.h \ + HttpHeader.h \ + Range.h \ + Stage.h \ + client.h \ + intercept.h \ + response.h \ + server.h \ + slice.h \ + transfer.h \ + +slice: $(slice_la_SOURCES) $(slice_la_HEADERS) + tsxs -v -o slice.so $(slice_la_SOURCES) + +install: slice $(slice_la_SOURCES) $(slice_la_HEADERS) + tsxs -v -o slice.so -i + +CXX = c++ -std=c++11 +#CXXFLAGS = -pipe -Wall -Wno-deprecated-declarations -Qunused-arguments -Wextra -Wno-ignored-qualifiers -Wno-unused-parameter -O3 -fno-strict-aliasing -Wno-invalid-offsetof -mcx16 +CXXFLAGS = -pipe -Wall -Wno-deprecated-declarations -Wextra -Wno-ignored-qualifiers -Wno-unused-parameter -O3 -fno-strict-aliasing -Wno-invalid-offsetof -mcx16 +TSINCLUDE = $(shell tsxs -q INCLUDEDIR) +#PREFIX = $(shell tsxs -q PREFIX) +#LIBS = -L$(PREFIX)/lib -latscppapi +#LIBS = $(PREFIX)/lib/libtsutil.la + +slice_test: slice_test.cc ContentRange.cc Range.cc + $(CXX) -o $@ $^ $(CXXFLAGS) -I$(TSINCLUDE) -DUNITTEST + +clean: + rm -fv *.lo *.so diff --git a/plugins/experimental/slice/README.md b/plugins/experimental/slice/README.md new file mode 100644 index 00000000000..731fd579241 --- /dev/null +++ b/plugins/experimental/slice/README.md @@ -0,0 +1,84 @@ +### Apache Traffic Server - Slicer Plugin + +The purpose of this plugin is to slice full file or range based requests +into deterministic chunks. This allows a large file to be spread across +multiple cache stripes and allows range requests to be satisfied by +stitching these chunks together. + +Deterministic chunks are requested from a parent cache or origin server +using a preconfigured block byte size. + +The plugin is an example of an intercept handler which takes a single +incoming request (range or whole asset), breaks it into a sequence +of block requests and assembles those blocks into a client response. +The plugin uses TSHttpConnect to delegate each block request to +cache_range_requests.so which handles all cache and parent interaction. + +To enable the plugin, specify the plugin library via @plugin at the end +of a remap line as follows (2MB slice in this example): + +``` +map http://ats-cache/ http://parent/ @plugin=slice.so @pparam=blockbytes:2097152 @plugin=cache_range_requests.so +``` + +for global plugins. + +``` +slice.so blockbytes:2097152 +cache_range_requests.so +``` + +**Note**: cache_range_requests **MUST** follow slice.so Put these plugins at the end of the plugin list +**Note**: blockbytes is defined in bytes. 1048576 (1MB) is the default. + +For testing purposes an unchecked value of "blockbytestest" is also available. + +Debug output can be enable by setting the debug tag: **slice** + +Debug messages related to object instance construction/deconstruction, see slice.h. + +At the current time only single range requests or the first part of a +multi part range request of the forms: +``` +Range: bytes=- +Range: bytes=- +Range: bytes=- +``` +are supported as multi part range responses are non trivial to implement. +This matches with the cache_range_requests.so plugin capability. + +--- + +Important things to note: + +Any first block server response that is not a 206 is passed down to +the client. + +Only the first server response block is used to evaluate any "If-" +headers. Subsequent server slice block requests remove these headers. + +If a client aborts mid transaction the current slice block is completed +to ensure that the block is written to cache. + +The only 416 case this plugin handles itself is if the requested range +is inside the end slice block but past the content length. Otherwise +parents seem to properly issue 416 responses themselves. + +--- + +To manually build the plugin use the "tsxs" executable that installs with +traffic_server. + +Running the following command will build the plugin + +``` +tsxs -v -o slice.so *.cc +``` + +Running the following command will build and install the plugin. +Beware this may crash a running system if the plugin is loaded +and the OS uses memory paging with plugins. + +``` +tsxs -v -i -o slice.so *.cc +``` diff --git a/plugins/experimental/slice/Range.cc b/plugins/experimental/slice/Range.cc new file mode 100644 index 00000000000..1a7e9685d46 --- /dev/null +++ b/plugins/experimental/slice/Range.cc @@ -0,0 +1,187 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Range.h" +#include "slice.h" + +#include +#include +#include +#include +#include + +bool +Range::isValid() const +{ + return m_beg < m_end && (0 <= m_beg || 0 == m_end); +} + +int64_t +Range::size() const +{ + return m_end - m_beg; +} + +bool +Range::fromStringClosed(char const *const rangestr) +{ + static char const *const BYTESTR = "bytes="; + static size_t const BYTESTRLEN = strlen(BYTESTR); + + m_beg = m_end = -1; // initialize invalid + + // make sure this is in byte units + if (0 != strncmp(BYTESTR, rangestr, BYTESTRLEN)) { + return false; + } + + // advance past any white space + char const *pstr = rangestr + BYTESTRLEN; + while ('\0' != *pstr && isblank(*pstr)) { + ++pstr; + } + + // rip out any whitespace + static int const RLEN = 1024; + char rangebuf[RLEN]; + char *pbuf = rangebuf; + while ('\0' != *pstr && (pbuf - rangebuf) < RLEN) { + if (!isblank(*pstr)) { + *pbuf++ = *pstr; + } + ++pstr; + } + *pbuf = '\0'; + + int const rlen = (pbuf - rangebuf); + + int consumed = 0; + + // last 'n' bytes - result in range with negative begin and 0 end + int64_t endbytes = 0; + char const *const fmtend = "-%" PRId64 "%n"; + int const fieldsend = sscanf(rangebuf, fmtend, &endbytes, &consumed); + if (1 == fieldsend) { + if (rlen == consumed) { + m_beg = -endbytes; + m_end = 0; + return true; + } else { + return false; + } + } + + // normal range - + char const *const fmtclosed = "%" PRId64 "-%" PRId64 "%n"; + int64_t front = 0; + int64_t back = 0; + + int const fieldsclosed = sscanf(rangebuf, fmtclosed, &front, &back, &consumed); + if (2 == fieldsclosed) { + if (0 <= front && front <= back && rlen == consumed) { + m_beg = front; + m_end = back + 1; + return true; + } else { + return false; + } + } + + front = 0; + char const *const fmtbeg = "%" PRId64 "-%n"; + int const fieldsbeg = sscanf(rangebuf, fmtbeg, &front, &consumed); + if (1 == fieldsbeg) { + if (rlen == consumed) { + m_beg = front; + m_end = Range::maxval; + return true; + } else { + return false; + } + } + + return false; +} // parseRange + +bool +Range::toStringClosed(char *const bufstr, + int *const buflen // returns actual bytes used + ) const +{ + if (!isValid()) { + if (0 < *buflen) { + bufstr[0] = '\0'; + } + *buflen = 0; + return false; + } + + int const lenin = *buflen; + + if (m_end <= Range::maxval) { + *buflen = snprintf(bufstr, lenin, "bytes=%" PRId64 "-%" PRId64, m_beg, m_end - 1); + } else { + *buflen = snprintf(bufstr, lenin, "bytes=%" PRId64 "-", m_beg); + } + + return (0 < *buflen && *buflen < lenin); +} + +int64_t +Range::firstBlockFor(int64_t const blocksize) const +{ + if (0 < blocksize && isValid()) { + return std::max((int64_t)0, m_beg / blocksize); + } else { + return -1; + } +} + +Range +Range::intersectedWith(Range const &other) const +{ + return Range(std::max(m_beg, other.m_beg), std::min(m_end, other.m_end)); +} + +bool +Range::blockIsInside(int64_t const blocksize, int64_t const blocknum) const +{ + Range const blockrange(blocksize * blocknum, blocksize * (blocknum + 1)); + + Range const isec(blockrange.intersectedWith(*this)); + + return isec.isValid(); +} + +int64_t +Range::skipBytesForBlock(int64_t const blocksize, int64_t const blocknum) const +{ + int64_t const blockstart(blocksize * blocknum); + + if (m_beg < blockstart) { + return 0; + } else { + return m_beg - blockstart; + } +} + +bool +Range::isEndBytes() const +{ + return m_beg < 0 && 0 == m_end; +} diff --git a/plugins/experimental/slice/Range.h b/plugins/experimental/slice/Range.h new file mode 100644 index 00000000000..373304db57d --- /dev/null +++ b/plugins/experimental/slice/Range.h @@ -0,0 +1,74 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +#include + +/** + represents a value parsed from a Range request header field. + Range is converted from a closed range into a half open. + */ + +struct Range { +public: + static int64_t constexpr maxval = (std::numeric_limits::max() >> 2); + + int64_t m_beg; + int64_t m_end; // half open + + Range() : m_beg(-1), m_end(-1) {} + explicit Range(int64_t const begin, int64_t const end) : m_beg(begin), m_end(end) {} + + bool isValid() const; + + int64_t size() const; + + /** parse a from a closed request range into a half open range + * This will only correctly handle the *first* range that is + * parsed via TSMimeHdrFieldValueStringGet with index '0'. + * Range representing last N bytes will be coded as (-N, 0) + */ + bool fromStringClosed(char const *const rangestr); + + /** parse a from a closed request range into a half open range + */ + bool toStringClosed(char *const rangestr, int *const rangelen) const; + + /** block number of first range block + */ + int64_t firstBlockFor(int64_t const blockbytes) const; + + /** block intersection + */ + Range intersectedWith(Range const &other) const; + + /** is the given block inside held range? + */ + bool blockIsInside(int64_t const blocksize, int64_t const blocknum) const; + + /** number of skip bytes for the given block + */ + int64_t skipBytesForBlock(int64_t const blocksize, int64_t const blocknum) const; + + /** is this coded to indicate last N bytes? + */ + bool isEndBytes() const; +}; diff --git a/plugins/experimental/slice/Stage.h b/plugins/experimental/slice/Stage.h new file mode 100644 index 00000000000..4166071e5eb --- /dev/null +++ b/plugins/experimental/slice/Stage.h @@ -0,0 +1,149 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +struct Channel { + TSVIO m_vio{nullptr}; + TSIOBuffer m_iobuf{nullptr}; + TSIOBufferReader m_reader{nullptr}; + + ~Channel() + { + if (nullptr != m_reader) { + TSIOBufferReaderFree(m_reader); + } + if (nullptr != m_iobuf) { + TSIOBufferDestroy(m_iobuf); + } + } + + void + drainReader() + { + TSAssert(nullptr != m_reader); + int64_t const bytes_avail = TSIOBufferReaderAvail(m_reader); + TSIOBufferReaderConsume(m_reader, bytes_avail); + } + + bool + setForRead(TSVConn vc, TSCont contp, int64_t const bytesin //=INT64_MAX + ) + { + TSAssert(nullptr != vc); + if (nullptr == m_iobuf) { + m_iobuf = TSIOBufferCreate(); + m_reader = TSIOBufferReaderAlloc(m_iobuf); + } else { + drainReader(); + } + m_vio = TSVConnRead(vc, contp, m_iobuf, bytesin); + return nullptr != m_vio; + } + + bool + setForWrite(TSVConn vc, TSCont contp, int64_t const bytesout //=INT64_MAX + ) + { + TSAssert(nullptr != vc); + if (nullptr == m_iobuf) { + m_iobuf = TSIOBufferCreate(); + m_reader = TSIOBufferReaderAlloc(m_iobuf); + } else { + drainReader(); + } + m_vio = TSVConnWrite(vc, contp, m_reader, bytesout); + return nullptr != m_vio; + } + + void + close() + { + if (nullptr != m_reader) { + drainReader(); + } + m_vio = nullptr; + } + + bool + isOpen() const + { + return nullptr != m_iobuf && nullptr != m_reader && nullptr != m_vio; + } +}; + +struct Stage // upstream or downstream (server or client) +{ + Stage(Stage const &) = delete; + Stage &operator=(Stage const &) = delete; + + TSVConn m_vc{nullptr}; + Channel m_read; + Channel m_write; + + Stage() {} + ~Stage() + { + if (nullptr != m_vc) { + TSVConnClose(m_vc); + } + } + + void + setupConnection(TSVConn vc) + { + if (nullptr != m_vc) { + TSVConnClose(m_vc); + } + m_vc = vc; + m_read.m_vio = nullptr; + m_write.m_vio = nullptr; + } + + void + setupVioRead(TSCont contp, int64_t const bytesin = INT64_MAX) + { + m_read.setForRead(m_vc, contp, bytesin); + } + + void + setupVioWrite(TSCont contp, int64_t const bytesout = INT64_MAX) + { + m_write.setForWrite(m_vc, contp, bytesout); + } + + void + close() + { + m_read.close(); + m_write.close(); + + if (nullptr != m_vc) { + TSVConnClose(m_vc); + m_vc = nullptr; + } + } + + bool + isOpen() const + { + return nullptr != m_vc && m_read.isOpen() && m_write.isOpen(); + } +}; diff --git a/plugins/experimental/slice/client.cc b/plugins/experimental/slice/client.cc new file mode 100644 index 00000000000..e29f9e715ac --- /dev/null +++ b/plugins/experimental/slice/client.cc @@ -0,0 +1,227 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "client.h" + +#include "transfer.h" + +namespace +{ +void +shutdown(TSCont const contp, Data *const data) +{ + DEBUG_LOG("shutting down transaction"); + delete data; + TSContDestroy(contp); +} + +// create and issue a block request +bool +requestBlock(TSCont contp, Data *const data) +{ + int64_t const blockbeg = (data->m_blockbytes_config * data->m_blocknum); + Range blockbe(blockbeg, blockbeg + data->m_blockbytes_config); + + char rangestr[1024]; + int rangelen = 1023; + bool const rpstat = blockbe.toStringClosed(rangestr, &rangelen); + TSAssert(rpstat); + + DEBUG_LOG("requestBlock: %s", rangestr); + + // reuse the incoming client header, just change the range + HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); + + // add/set sub range key and add slicer tag + bool const rangestat = header.setKeyVal(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, rangelen); + + if (!rangestat) { + ERROR_LOG("Error trying to set range request header %s", rangestr); + return false; + } + + // create virtual connection back into ATS + TSVConn const upvc = TSHttpConnect((sockaddr *)&data->m_client_ip); + + // set up connection with the HttpConnect server, maybe clear old one + data->m_upstream.setupConnection(upvc); + data->m_upstream.setupVioWrite(contp); + + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_upstream.m_write.m_iobuf); + TSVIOReenable(data->m_upstream.m_write.m_vio); + + // get ready for data back from the server + data->m_upstream.setupVioRead(contp); + + // anticipate the next server response header + TSHttpParserClear(data->m_http_parser); + data->m_resp_hdrmgr.resetHeader(); + + data->m_blockexpected = 0; + data->m_blockconsumed = 0; + data->m_iseos = false; + data->m_server_block_header_parsed = false; + + return true; +} + +} // namespace + +// this is called once per transaction when the client sends a req header +bool +handle_client_req(TSCont contp, TSEvent event, Data *const data) +{ + if (TS_EVENT_VCONN_READ_READY == event || TS_EVENT_VCONN_READ_COMPLETE == event) { + if (nullptr == data->m_http_parser) { + data->m_http_parser = TSHttpParserCreate(); + } + + // the client request header didn't fit into the input buffer: + if (TS_PARSE_DONE != + data->m_req_hdrmgr.populateFrom(data->m_http_parser, data->m_dnstream.m_read.m_reader, TSHttpHdrParseReq)) { + return false; + } + + // make the header manipulator + HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); + + // set the request url back to pristine in case of plugin stacking + header.setUrl(data->m_urlbuffer, data->m_urlloc); + + header.setKeyVal(TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST, data->m_hostname, data->m_hostlen); + + // default: whole file (unknown, wait for first server response) + Range rangebe; + + char rangestr[1024]; + int rangelen = 1024; + bool const hasRange = header.valueForKey(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, &rangelen, + 0); // <-- first range only + if (hasRange) { + // write parsed header into slicer meta tag + header.setKeyVal(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), rangestr, rangelen); + bool const isRangeGood = rangebe.fromStringClosed(rangestr); + + if (isRangeGood) { + DEBUG_LOG("Partial content request"); + data->m_statustype = TS_HTTP_STATUS_PARTIAL_CONTENT; + } else // signal a 416 needs to be formed and sent + { + DEBUG_LOG("Ill formed/unhandled range: %s", rangestr); + data->m_statustype = TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE; + + // First block will give Content-Length + rangebe = Range(0, data->m_blockbytes_config); + } + } else { + DEBUG_LOG("Full content request"); + static char const *const valstr = "full content request"; + static size_t const vallen = strlen(valstr); + header.setKeyVal(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), valstr, vallen); + data->m_statustype = TS_HTTP_STATUS_OK; + rangebe = Range(0, Range::maxval); + } + + // set to the first block in range + data->m_blocknum = rangebe.firstBlockFor(data->m_blockbytes_config); + data->m_req_range = rangebe; + + // remove ATS keys to avoid 404 loop + header.removeKey(TS_MIME_FIELD_VIA, TS_MIME_LEN_VIA); + header.removeKey(TS_MIME_FIELD_X_FORWARDED_FOR, TS_MIME_LEN_X_FORWARDED_FOR); + + // send the first block request to server + if (!requestBlock(contp, data)) { + shutdown(contp, data); + return false; + } + + // for subsequent blocks remove any conditionals which may fail + // an optimization would be to wait until the first block succeeds + header.removeKey(TS_MIME_FIELD_IF_MATCH, TS_MIME_LEN_IF_MATCH); + header.removeKey(TS_MIME_FIELD_IF_MODIFIED_SINCE, TS_MIME_LEN_IF_MODIFIED_SINCE); + header.removeKey(TS_MIME_FIELD_IF_NONE_MATCH, TS_MIME_LEN_IF_NONE_MATCH); + header.removeKey(TS_MIME_FIELD_IF_RANGE, TS_MIME_LEN_IF_RANGE); + header.removeKey(TS_MIME_FIELD_IF_UNMODIFIED_SINCE, TS_MIME_LEN_IF_UNMODIFIED_SINCE); + } + + return true; +} + +// this is when the client starts asking us for more data +void +handle_client_resp(TSCont contp, TSEvent event, Data *const data) +{ + if (TS_EVENT_VCONN_WRITE_READY == event || TS_EVENT_VCONN_WRITE_COMPLETE == event) { + transfer_content_bytes(data); + + // done transferring from server to client buffer? + if (data->m_bytestosend <= data->m_bytessent) { + // real amount transferred to client + int64_t const bytessent(TSVIONDoneGet(data->m_dnstream.m_write.m_vio)); + + // is the output buffer drained? + if (data->m_bytestosend <= bytessent) { + data->m_dnstream.close(); + if (!data->m_upstream.m_read.isOpen()) { + shutdown(contp, data); + return; + } + } + + // continue allowing the downstream to drain + return; + } + + // error condition from the server side + if (data->m_bail) { + shutdown(contp, data); + return; + } + + // check for upstream eos, maybe request next block + if (data->m_iseos) { + // still need to drain the server side + if (0 < TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader)) { + TSVIOReenable(data->m_dnstream.m_write.m_vio); + return; + } + + // if done or partial block + if (data->m_blocknum < 0 || data->m_blockconsumed < data->m_blockexpected) { + shutdown(contp, data); + return; + } + + // ready for next block + requestBlock(contp, data); + } + } + // client closed connection + else if (TS_EVENT_ERROR == event) { + DEBUG_LOG("got a TS_EVENT_ERROR from the client"); + + // allow the upstream server to drain + data->m_dnstream.close(); + if (!data->m_upstream.m_read.isOpen()) { + shutdown(contp, data); + } + } else { + DEBUG_LOG("Unhandled event: %d", event); + } +} diff --git a/plugins/experimental/slice/client.h b/plugins/experimental/slice/client.h new file mode 100644 index 00000000000..cc27bee3b1d --- /dev/null +++ b/plugins/experimental/slice/client.h @@ -0,0 +1,34 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "Data.h" + +#include "ts/ts.h" + +/** Functions to deal with the connection to the client. + * Body content transfers are handled by the client. + * New block requests are also initiated by the client. + */ + +/** returns true if the incoming vio can be turned off + */ +bool handle_client_req(TSCont contp, TSEvent event, Data *const data); + +void handle_client_resp(TSCont contp, TSEvent event, Data *const data); diff --git a/plugins/experimental/slice/intercept.cc b/plugins/experimental/slice/intercept.cc new file mode 100644 index 00000000000..f21306a9942 --- /dev/null +++ b/plugins/experimental/slice/intercept.cc @@ -0,0 +1,88 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "intercept.h" + +#include "Data.h" +#include "client.h" +#include "server.h" +#include "slice.h" + +int +intercept_hook(TSCont contp, TSEvent event, void *edata) +{ + // DEBUG_LOG("intercept_hook: %d", event); + + Data *const data = static_cast(TSContDataGet(contp)); + if (nullptr == data) { + DEBUG_LOG("Events handled after data already torn down"); + TSContDestroy(contp); + return TS_EVENT_ERROR; + } + + // After the initial TS_EVENT_NET_ACCEPT + // any "events" will be handled by the vio read or write channel handler + switch (event) { + case TS_EVENT_NET_ACCEPT: { + // set up reader from client + TSVConn const downvc = (TSVConn)edata; + data->m_dnstream.setupConnection(downvc); + data->m_dnstream.setupVioRead(contp); + } break; + + case TS_EVENT_VCONN_INACTIVITY_TIMEOUT: + case TS_EVENT_VCONN_ACTIVE_TIMEOUT: + case TS_EVENT_HTTP_TXN_CLOSE: + delete data; + TSContDestroy(contp); + break; + + default: { + // data from client -- only the initial header + if (data->m_dnstream.m_read.isOpen() && edata == data->m_dnstream.m_read.m_vio) { + if (handle_client_req(contp, event, data)) { + // DEBUG_LOG("shutting down read from client pipe"); + TSVConnShutdown(data->m_dnstream.m_vc, 1, 0); + } + } + // server wants more data from us, should never happen + // every time TSHttpConnect is called this resets + else if (data->m_upstream.m_write.isOpen() && edata == data->m_upstream.m_write.m_vio) { + // DEBUG_LOG("shutting down send to server pipe"); + TSVConnShutdown(data->m_upstream.m_vc, 0, 1); + } + // server has data for us, typically handle just the header + else if (data->m_upstream.m_read.isOpen() && edata == data->m_upstream.m_read.m_vio) { + handle_server_resp(contp, event, data); + } + // client wants more data from us, only body content + else if (data->m_dnstream.m_write.isOpen() && edata == data->m_dnstream.m_write.m_vio) { + handle_client_resp(contp, event, data); + } else { + ERROR_LOG("Unhandled event: %d", event); + /* + std::cerr << __func__ + << ": events received after intercept state torn down" + << std::endl; + */ + } + } + } + + return TS_EVENT_CONTINUE; +} diff --git a/plugins/experimental/slice/intercept.h b/plugins/experimental/slice/intercept.h new file mode 100644 index 00000000000..3b59cef30f9 --- /dev/null +++ b/plugins/experimental/slice/intercept.h @@ -0,0 +1,23 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +int intercept_hook(TSCont contp, TSEvent event, void *edata); diff --git a/plugins/experimental/slice/response.cc b/plugins/experimental/slice/response.cc new file mode 100644 index 00000000000..4b9f101bde1 --- /dev/null +++ b/plugins/experimental/slice/response.cc @@ -0,0 +1,110 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "response.h" + +#include +#include +#include + +#include "ts/ts.h" + +// canned body string for a 416, stolen from nginx +std::string const & +bodyString416() +{ + static std::string bodystr; + static std::mutex mutex; + std::lock_guard const guard(mutex); + + if (bodystr.empty()) { + bodystr.append("\n"); + bodystr.append("416 Requested Range Not Satisfiable\n"); + bodystr.append("\n"); + bodystr.append("

416 Requested Range Not Satisfiable

"); + bodystr.append("
ATS/"); + bodystr.append(TS_VERSION_STRING); + bodystr.append("
\n"); + bodystr.append("\n"); + bodystr.append("\n"); + } + + return bodystr; +} + +// Form a 502 response, preliminary +std::string const & +string502() +{ + static std::string msg; + static std::mutex mutex; + std::lock_guard const guard(mutex); + + if (msg.empty()) { + std::string bodystr; + bodystr.append("\n"); + bodystr.append("502 Bad Gateway\n"); + bodystr.append("\n"); + bodystr.append("

502 Bad Gateway: Missing/Malformed " + "Content-Range

"); + bodystr.append("
ATS/"); + bodystr.append(TS_VERSION_STRING); + bodystr.append("
\n"); + bodystr.append("\n"); + bodystr.append("\n"); + + static int const CLEN = 1024; + char clenstr[CLEN]; + int const clen = snprintf(clenstr, CLEN, "%lu", bodystr.size()); + + msg.append("HTTP/1.1 502 Bad Gateway\r\n"); + msg.append("Content-Length: "); + msg.append(clenstr, clen); + msg.append("\r\n"); + + msg.append("\r\n"); + msg.append(bodystr); + } + + return msg; +} + +void +form416HeaderAndBody(HttpHeader &header, int64_t const contentlen, std::string const &bodystr) +{ + header.removeKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED); + header.removeKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG); + header.removeKey(TS_MIME_FIELD_ACCEPT_RANGES, TS_MIME_LEN_ACCEPT_RANGES); + + header.setStatus(TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); + char const *const reason = TSHttpHdrReasonLookup(TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); + header.setReason(reason, strlen(reason)); + + char bufstr[256]; + int buflen = snprintf + // (bufstr, 255, "%" PRId64, bodystr.size()); + (bufstr, 255, "%lu", bodystr.size()); + header.setKeyVal(TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, bufstr, buflen); + + static char const *const ctypestr = "text/html"; + static int const ctypelen = strlen(ctypestr); + header.setKeyVal(TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE, ctypestr, ctypelen); + + buflen = snprintf(bufstr, 255, "*/%" PRId64, contentlen); + header.setKeyVal(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, bufstr, buflen); +} diff --git a/plugins/experimental/slice/response.h b/plugins/experimental/slice/response.h new file mode 100644 index 00000000000..925db65f725 --- /dev/null +++ b/plugins/experimental/slice/response.h @@ -0,0 +1,28 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "HttpHeader.h" +#include + +std::string const &string502(); + +std::string const &bodyString416(); + +void form416HeaderAndBody(HttpHeader &header, int64_t const contentlen, std::string const &bodystr); diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc new file mode 100644 index 00000000000..508a455ecab --- /dev/null +++ b/plugins/experimental/slice/server.cc @@ -0,0 +1,342 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "server.h" + +#include "ContentRange.h" +#include "response.h" +#include "transfer.h" + +#include + +namespace +{ +void +shutdown(TSCont const contp, Data *const data) +{ + DEBUG_LOG("shutting down transaction"); + delete data; + TSContDestroy(contp); +} + +ContentRange +contentRangeFrom(HttpHeader &header) +{ + ContentRange bcr; + + /* Pull content length off the response header + and manipulate it into a client response header + */ + static int const RLEN = 1024; + char rangestr[RLEN]; + int rangelen = RLEN - 1; + + // look for expected Content-Range field + bool const hasContentRange(header.valueForKey(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, rangestr, &rangelen)); + + if (!hasContentRange) { + DEBUG_LOG("invalid response header, no Content-Range"); + } else if (!bcr.fromStringClosed(rangestr)) { + DEBUG_LOG("invalid response header, malformed Content-Range, %s", rangestr); + } + + return bcr; +} + +bool +handleFirstServerHeader(Data *const data, TSCont const contp) +{ + HttpHeader header(data->m_resp_hdrmgr.m_buffer, data->m_resp_hdrmgr.m_lochdr); + + data->m_dnstream.setupVioWrite(contp); + + // only process a 206, everything else gets a pass through + if (TS_HTTP_STATUS_PARTIAL_CONTENT != header.status()) { + DEBUG_LOG("Non 206 response from parent: %d", header.status()); + data->m_bail = true; + + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); + + transfer_all_bytes(data); + + return false; + } + + ContentRange const blockcr = contentRangeFrom(header); + // 206 with bad content range? + if (!blockcr.isValid()) { + data->m_bail = true; + + static std::string const &msg502 = string502(); + + TSIOBufferWrite(data->m_dnstream.m_write.m_iobuf, msg502.data(), msg502.size()); + TSVIOReenable(data->m_dnstream.m_write.m_vio); + + return false; + } + + // set the resource content length from block response + data->m_contentlen = blockcr.m_length; + + // special case last N bytes + if (data->m_req_range.isEndBytes()) { + data->m_req_range.m_end += data->m_contentlen; + data->m_req_range.m_beg += data->m_contentlen; + data->m_req_range.m_beg = std::max((int64_t)0, data->m_req_range.m_beg); + } else { + // fix up request range end now that we have the content length + data->m_req_range.m_end = std::min(data->m_contentlen, data->m_req_range.m_end); + } + + int64_t const bodybytes = data->m_req_range.size(); + + // range past end of data, assume 416 needs to be sent + bool const send416 = (bodybytes <= 0 || TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE == data->m_statustype); + if (send416) { + data->m_bail = true; + std::string const &bodystr = bodyString416(); + form416HeaderAndBody(header, data->m_contentlen, bodystr); + + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); + + TSIOBufferWrite(data->m_dnstream.m_write.m_iobuf, bodystr.data(), bodystr.size()); + + TSVIOReenable(data->m_dnstream.m_write.m_vio); + + return false; + } + + // save weak cache header identifiers (rfc7232 section 2) + data->m_etaglen = sizeof(data->m_etag) - 1; + header.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, data->m_etag, &data->m_etaglen); + data->m_lastmodifiedlen = sizeof(data->m_lastmodified) - 1; + header.valueForKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED, data->m_lastmodified, &data->m_lastmodifiedlen); + + // size of the first block payload + data->m_blockexpected = blockcr.rangeSize(); + + // Now we can set up the expected client response + if (TS_HTTP_STATUS_PARTIAL_CONTENT == data->m_statustype) { + ContentRange respcr; + respcr.m_beg = data->m_req_range.m_beg; + respcr.m_end = data->m_req_range.m_end; + respcr.m_length = data->m_contentlen; + + char rangestr[1024]; + int rangelen = 1023; + bool const crstat = respcr.toStringClosed(rangestr, &rangelen); + + // corner case, return 500 ?? + if (!crstat) { + data->m_bail = true; + + data->m_upstream.close(); + data->m_dnstream.close(); + + ERROR_LOG("Bad/invalid response content range"); + return false; + } + + header.setKeyVal(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, rangestr, rangelen); + } + // fix up for 200 response + else if (TS_HTTP_STATUS_OK == data->m_statustype) { + header.setStatus(TS_HTTP_STATUS_OK); + static char const *const reason = TSHttpHdrReasonLookup(TS_HTTP_STATUS_OK); + header.setReason(reason, strlen(reason)); + header.removeKey(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE); + } + + char bufstr[1024]; + int const buflen = snprintf(bufstr, 1023, "%" PRId64, bodybytes); + header.setKeyVal(TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, bufstr, buflen); + + // add the response header length to the total bytes to send + int64_t const headerbytes = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); + + data->m_bytestosend = headerbytes + bodybytes; + + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); + + data->m_bytessent = headerbytes; + + TSVIOReenable(data->m_dnstream.m_write.m_vio); + + return true; +} + +bool +handleNextServerHeader(Data *const data, TSCont const contp) +{ + HttpHeader header(data->m_resp_hdrmgr.m_buffer, data->m_resp_hdrmgr.m_lochdr); + + // only process a 206, everything else just aborts + if (TS_HTTP_STATUS_PARTIAL_CONTENT != header.status()) { + ERROR_LOG("Non 206 internal block response from parent: %d", header.status()); + data->m_bail = true; + return false; + } + + // can't parse the content range header, abort -- might be too strict + ContentRange const blockcr = contentRangeFrom(header); + if (!blockcr.isValid()) { + ERROR_LOG("Unable to parse internal block Content-Range header"); + data->m_bail = true; + return false; + } + + // make sure the block comes from the same asset as the first block + if (data->m_contentlen != blockcr.m_length) { + ERROR_LOG("Mismatch in slice block Content-Range Len %" PRId64 " and %" PRId64, data->m_contentlen, blockcr.m_length); + data->m_bail = true; + return false; + } + + bool same = true; + + // prefer the etag but use Last-Modified if we must. + char etag[8192]; + int etaglen = sizeof(etag) - 1; + header.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, etag, &etaglen); + + if (0 < data->m_etaglen || 0 < etaglen) { + same = data->m_etaglen == etaglen && 0 == strncmp(etag, data->m_etag, etaglen); + if (!same) { + ERROR_LOG("Mismatch in slice block ETAG '%.*s' and '%.*s'", data->m_etaglen, data->m_etag, etaglen, etag); + } + } else { + char lastmodified[8192]; + int lastmodifiedlen = sizeof(lastmodified) - 1; + header.valueForKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED, lastmodified, &lastmodifiedlen); + if (0 < data->m_lastmodifiedlen || 0 != lastmodifiedlen) { + same = data->m_lastmodifiedlen == lastmodifiedlen && 0 == strncmp(lastmodified, data->m_lastmodified, lastmodifiedlen); + if (!same) { + ERROR_LOG("Mismatch in slice block Last-Modified '%.*s' and '%.*s'", data->m_lastmodifiedlen, data->m_lastmodified, + lastmodifiedlen, lastmodified); + } + } + } + + if (!same) { + data->m_bail = true; + return false; + } + + data->m_blockexpected = blockcr.rangeSize(); + + return true; +} + +} // namespace + +// this is called every time the server has data for us +void +handle_server_resp(TSCont contp, TSEvent event, Data *const data) +{ + if (TS_EVENT_VCONN_READ_READY == event || TS_EVENT_VCONN_READ_COMPLETE == event) { + // has block reponse header been parsed?? + if (!data->m_server_block_header_parsed) { + // the server response header didn't fit into the input buffer?? + if (TS_PARSE_DONE != + data->m_resp_hdrmgr.populateFrom(data->m_http_parser, data->m_upstream.m_read.m_reader, TSHttpHdrParseResp)) { + return; + } + + // very first server response header + bool headerStat = false; + if (!data->m_server_first_header_parsed) { + headerStat = handleFirstServerHeader(data, contp); + data->m_server_first_header_parsed = true; + } else { + headerStat = handleNextServerHeader(data, contp); + } + + data->m_server_block_header_parsed = true; + + // kill the upstream and allow dnstream to clean up + if (!headerStat) { + data->m_upstream.close(); + data->m_bail = true; + if (data->m_dnstream.m_write.isOpen()) { + TSVIOReenable(data->m_dnstream.m_write.m_vio); + } else { + shutdown(contp, data); + } + return; + } + + // how much to fast forward into this data block + data->m_blockskip = data->m_req_range.skipBytesForBlock(data->m_blockbytes_config, data->m_blocknum); + } + + transfer_content_bytes(data); + } else if (TS_EVENT_VCONN_EOS == event) { + // from testing as far as I can tell, if the sub transaction returns + // a valid header TS_EVENT_VCONN_READ_READY event is always called first. + // this event being called means the input stream is null. + // An upstream transaction that aborts immediately (or a few bytes) + // after it sends a header may end up here with nothing in the upstream + // buffer. + + // this is called when the upstream connection is done. + // make sure to drain all the bytes out before + // issuing the next block request + data->m_iseos = true; + + // corner condition, good source header + 0 length aborted content + // results in no header being read, just an EOS. + // trying to delete the upstream will crash ATS (??) + if (0 == data->m_blockexpected) { + shutdown(contp, data); // this will crash if first block + return; + } + + transfer_content_bytes(data); + + if (!data->m_dnstream.m_write.isOpen()) // server drain condition + { + shutdown(contp, data); + return; + } + + // all bytes left transferred to client buffer + if (0 == TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader)) { + data->m_upstream.close(); + TSVIOReenable(data->m_dnstream.m_write.m_vio); + } + + // prepare for the next request block + ++data->m_blocknum; + + // when we get a "bytes=-" last N bytes request the plugin + // issues a speculative request for the first block + // in that case fast forward to the real first in range block + // Btw this isn't implemented yet, to be handled + int64_t const firstblock(data->m_req_range.firstBlockFor(data->m_blockbytes_config)); + if (data->m_blocknum < firstblock) { + data->m_blocknum = firstblock; + } + + // done processing blocks? + if (!data->m_req_range.blockIsInside(data->m_blockbytes_config, data->m_blocknum)) { + data->m_blocknum = -1; // signal value no more blocks + } + } else { + DEBUG_LOG("Unhandled event: %d", event); + } +} diff --git a/plugins/experimental/slice/server.h b/plugins/experimental/slice/server.h new file mode 100644 index 00000000000..c0d77e48032 --- /dev/null +++ b/plugins/experimental/slice/server.h @@ -0,0 +1,37 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "Data.h" + +#include "ts/ts.h" + +/** Functions to handle the connection to the server. + * In particular slice block header responses are handled here. + * Data transfers are handled by the client code which pulls + * the data from the server side. + * + * Special case is when the client connection has been closed + * because of client data request being fulfilled or + * when the client aborts. The current slice block will + * continue reading to ensure the whole block is transferred + * to cache. + */ + +void handle_server_resp(TSCont contp, TSEvent event, Data *const data); diff --git a/plugins/experimental/slice/slice.cc b/plugins/experimental/slice/slice.cc new file mode 100644 index 00000000000..b0c0e1aba50 --- /dev/null +++ b/plugins/experimental/slice/slice.cc @@ -0,0 +1,205 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "slice.h" + +#include "Config.h" +#include "Data.h" +#include "HttpHeader.h" +#include "intercept.h" + +#include "ts/remap.h" +#include "ts/ts.h" + +#include + +namespace +{ +Config globalConfig; + +bool +read_request(TSHttpTxn txnp, Config const *const config) +{ + DEBUG_LOG("slice read_request"); + TxnHdrMgr hdrmgr; + hdrmgr.populateFrom(txnp, TSHttpTxnClientReqGet); + HttpHeader const header(hdrmgr.m_buffer, hdrmgr.m_lochdr); + + if (TS_HTTP_METHOD_GET == header.method()) { + static int const SLICER_MIME_LEN_INFO = strlen(SLICER_MIME_FIELD_INFO); + if (!header.hasKey(SLICER_MIME_FIELD_INFO, SLICER_MIME_LEN_INFO)) { + // turn off any and all transaction caching (shouldn't matter) + TSHttpTxnServerRespNoStoreSet(txnp, 1); + TSHttpTxnRespCacheableSet(txnp, 0); + TSHttpTxnReqCacheableSet(txnp, 0); + + DEBUG_LOG("slice accepting and slicing"); + // connection back into ATS + sockaddr const *const ip = TSHttpTxnClientAddrGet(txnp); + if (nullptr == ip) { + return false; + } + + Data *const data = new Data(config->m_blockbytes); + + // set up feedback connect + if (AF_INET == ip->sa_family) { + memcpy(&data->m_client_ip, ip, sizeof(sockaddr_in)); + } else if (AF_INET6 == ip->sa_family) { + memcpy(&data->m_client_ip, ip, sizeof(sockaddr_in6)); + } else { + delete data; + return false; + } + + // need to reset the HOST field for global plugin + data->m_hostlen = sizeof(data->m_hostname) - 1; + if (!header.valueForKey(TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST, data->m_hostname, &data->m_hostlen)) { + DEBUG_LOG("Unable to get hostname from header"); + delete data; + return false; + } + + // need the pristine url, especially for global plugins + TSMBuffer urlbuf; + TSMLoc urlloc; + TSReturnCode rcode = TSHttpTxnPristineUrlGet(txnp, &urlbuf, &urlloc); + + if (TS_SUCCESS == rcode) { + TSMBuffer const newbuf = TSMBufferCreate(); + TSMLoc newloc = nullptr; + rcode = TSUrlClone(newbuf, urlbuf, urlloc, &newloc); + TSHandleMLocRelease(urlbuf, TS_NULL_MLOC, urlloc); + + if (TS_SUCCESS != rcode) { + ERROR_LOG("Error cloning pristine url"); + delete data; + TSMBufferDestroy(newbuf); + return false; + } + + data->m_urlbuffer = newbuf; + data->m_urlloc = newloc; + } + + // we'll intercept this GET and do it ourselfs + TSCont const icontp(TSContCreate(intercept_hook, TSMutexCreate())); + TSContDataSet(icontp, (void *)data); + // TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, icontp); + TSHttpTxnIntercept(icontp, txnp); + return true; + } else { + DEBUG_LOG("slice passing GET request through to next plugin"); + } + } + + return false; +} + +int +global_read_request_hook(TSCont // contp + , + TSEvent // event + , + void *edata) +{ + TSHttpTxn const txnp = static_cast(edata); + read_request(txnp, &globalConfig); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +} // namespace + +///// remap plugin engine + +SLICE_EXPORT +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) +{ + Config *const config = static_cast(ih); + + if (read_request(txnp, config)) { + return TSREMAP_DID_REMAP_STOP; + } else { + return TSREMAP_NO_REMAP; + } +} + +///// remap plugin setup and teardown +SLICE_EXPORT +void +TSRemapOSResponse(void *ih, TSHttpTxn rh, int os_response_type) +{ +} + +SLICE_EXPORT +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) +{ + Config *const config = new Config; + if (2 < argc) { + config->fromArgs(argc - 2, argv + 2, errbuf, errbuf_size); + } + *ih = static_cast(config); + return TS_SUCCESS; +} + +SLICE_EXPORT +void +TSRemapDeleteInstance(void *ih) +{ + if (nullptr != ih) { + Config *const config = static_cast(ih); + delete config; + } +} + +SLICE_EXPORT +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbug, int errbuf_size) +{ + DEBUG_LOG("slice remap is successfully initialized."); + return TS_SUCCESS; +} + +///// global plugin +SLICE_EXPORT +void +TSPluginInit(int argc, char const *argv[]) +{ + TSPluginRegistrationInfo info; + info.plugin_name = (char *)PLUGIN_NAME; + info.vendor_name = (char *)"Apache Software Foundation"; + info.support_email = (char *)"dev@trafficserver.apache.org"; + + if (TS_SUCCESS != TSPluginRegister(&info)) { + ERROR_LOG("Plugin registration failed.\n"); + ERROR_LOG("Unable to initialize plugin (disabled)."); + return; + } + + if (1 < argc) { + globalConfig.fromArgs(argc - 1, argv + 1, nullptr, 0); + } + + TSCont const contp(TSContCreate(global_read_request_hook, nullptr)); + + // Called immediately after the request header is read from the client + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, contp); +} diff --git a/plugins/experimental/slice/slice.h b/plugins/experimental/slice/slice.h new file mode 100644 index 00000000000..0cb2523b18d --- /dev/null +++ b/plugins/experimental/slice/slice.h @@ -0,0 +1,54 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +#include + +#ifndef SLICE_EXPORT +#define SLICE_EXPORT extern "C" tsapi +#endif + +#ifndef PLUGIN_NAME +#define PLUGIN_NAME "slice" +#endif + +#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) + +#if !defined(UNITTEST) + +#define DEBUG_LOG(fmt, ...) \ + TSDebug(PLUGIN_NAME, "[%s:%04d] %s(): " fmt, __FILENAME__, __LINE__, __func__, \ + ##__VA_ARGS__) /* \ + ; fprintf(stderr, "[%s:%04d]: " fmt "\n" \ + , __FILENAME__ \ + , __LINE__ \ + , ##__VA_ARGS__) \ + */ + +#define ERROR_LOG(fmt, ...) \ + TSError("[%s:%04d] %s(): " fmt, __FILENAME__, __LINE__, __func__, ##__VA_ARGS__); \ + TSDebug(PLUGIN_NAME, "[%s:%04d] %s(): " fmt, __FILENAME__, __LINE__, __func__, ##__VA_ARGS__) + +#else +#define DEBUG_LOG(fmt, ...) +#define ERROR_LOG(fmt, ...) + +#endif diff --git a/plugins/experimental/slice/slice_test.cc b/plugins/experimental/slice/slice_test.cc new file mode 100644 index 00000000000..60e0246a7be --- /dev/null +++ b/plugins/experimental/slice/slice_test.cc @@ -0,0 +1,195 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/* + * These are misc unit tests for slicer + */ + +#include "ContentRange.h" +#include "Range.h" + +#include +#include +#include +#include +#include + +std::string +testContentRange() +{ + std::ostringstream oss; + + ContentRange null; + if (null.isValid()) { + oss << "fail: null isValid test" << std::endl; + } + + ContentRange const exprange(1023, 1048576, 307232768); + + if (!exprange.isValid()) { + oss << "Fail: exprange valid" << std::endl; + oss << exprange.m_beg << ' ' << exprange.m_end << ' ' << exprange.m_length << std::endl; + } + + std::string const expstr("bytes 1023-1048575/307232768"); + + char gotbuf[1024]; + int gotlen = sizeof(gotbuf); + + bool const strstat(exprange.toStringClosed(gotbuf, &gotlen)); + + if (!strstat) { + oss << "failure status toStringClosed" << std::endl; + } else if ((int)expstr.size() != gotlen) { + oss << "Fail: expected toStringClosed length" << std::endl; + oss << "got: " << gotlen << " exp: " << expstr.size() << std::endl; + oss << "Got: " << gotbuf << std::endl; + oss << "Exp: " << expstr << std::endl; + } else if (expstr != gotbuf) { + oss << "Fail: expected toStringClosed value" << std::endl; + oss << "Got: " << gotbuf << std::endl; + oss << "Exp: " << expstr << std::endl; + } + + ContentRange gotrange; + bool const gotstat(gotrange.fromStringClosed(expstr.c_str())); + if (!gotstat) { + oss << "fail: gotstat from string" << std::endl; + } else if (gotrange.m_beg != exprange.m_beg || gotrange.m_end != exprange.m_end || gotrange.m_length != exprange.m_length) { + oss << "fail: value compare gotrange and exprange" << std::endl; + } + + std::string const teststr("bytes 0-1048575/30723276"); + if (!gotrange.fromStringClosed(teststr.c_str())) { + oss << "fail: parse teststr" << std::endl; + } + + return oss.str(); +} + +std::string +testParseRange() +{ + std::ostringstream oss; + + std::vector const teststrings = { + "bytes=0-1023", + "bytes=1-1024", + "bytes=11-11", + "bytes=1-" // 2nd byte to end + , + "Range: bytes=-13" // final 13 bytes + , + "bytes=3-17" // ,23-29" // open + , + "bytes=3 -17 " //,18-29" // adjacent + , + "bytes=3- 17" //, 11-29" // overlapping + , + "bytes=3 - 11" //,13-17 , 23-29" // unsorted triplet + , + "bytes=3-11 " //,13-17, 23-29" // unsorted triplet + , + "bytes=0-0" //,-1" // first and last bytes + , + "bytes=-20" // last 20 bytes of file + + , + "bytes=-60-50" // invalid fully negative + , + "bytes=17-13" // degenerate + , + "bytes 0-1023/146515" // this should be rejected (Content-range) + }; // invalid + + std::vector const exps = {Range{0, 1023 + 1}, Range{1, 1024 + 1}, Range{11, 11 + 1}, Range{1, Range::maxval}, + Range{-1, -1}, Range{3, 17 + 1}, Range{3, 17 + 1}, Range{3, 17 + 1}, + Range{3, 11 + 1}, Range{3, 11 + 1}, Range{0, 1}, Range{-20, 0}, + Range{-1, -1}, Range{-1, -1}, Range{-1, -1}}; + + std::vector const expsres = {true, true, true, true, false, true, true, true, true, true, true, true, false, false, false}; + + assert(exps.size() == teststrings.size()); + + std::vector gots; + gots.reserve(exps.size()); + std::vector gotsres; + + for (std::string const &str : teststrings) { + Range rng; + gotsres.push_back(rng.fromStringClosed(str.c_str())); + gots.push_back(rng); + } + + assert(gots.size() == exps.size()); + + for (size_t index(0); index < gots.size(); ++index) { + if (exps[index] != gots[index] || expsres[index] != gotsres[index]) { + oss << "Eror parsing index: " << index << std::endl; + oss << "test: '" << teststrings[index] << "'" << std::endl; + oss << "exp: " << exps[index].m_beg << ' ' << exps[index].m_end << std::endl; + oss << "expsres: " << (int)expsres[index] << std::endl; + oss << "got: " << gots[index].m_beg << ' ' << gots[index].m_end << std::endl; + oss << "gotsres: " << (int)gotsres[index] << std::endl; + } + } + + return oss.str(); +} + +struct Tests { + typedef std::string (*TestFunc)(); + std::vector> funcs; + + void + add(TestFunc const &func, char const *const fname) + { + funcs.push_back(std::make_pair(func, fname)); + } + + int + run() const + { + int numfailed(0); + for (std::pair const &namefunc : funcs) { + TestFunc const &func = namefunc.first; + char const *const name = namefunc.second; + + std::cerr << name << " : "; + + std::string const fres(func()); + if (fres.empty()) { + std::cerr << "pass" << std::endl; + } else { + std::cerr << "FAIL" << std::endl; + std::cerr << fres << std::endl; + ++numfailed; + } + } + return numfailed; + } +}; + +int +main() +{ + Tests tests; + tests.add(testContentRange, "testContentRange"); + tests.add(testParseRange, "testParseRange"); + return tests.run(); +} diff --git a/plugins/experimental/slice/transfer.cc b/plugins/experimental/slice/transfer.cc new file mode 100644 index 00000000000..4ab7a8bb7c1 --- /dev/null +++ b/plugins/experimental/slice/transfer.cc @@ -0,0 +1,105 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "transfer.h" + +int64_t transfer_content_bytes(Data *const data) // , char const * const fstr) +{ + int64_t consumed(0); + + // is the downstream is fulfilled or closed + if (!data->m_dnstream.m_write.isOpen()) { + // drain the upstream + if (data->m_upstream.m_read.isOpen()) { + int64_t const avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, avail); + consumed += avail; + } + } else // if (data->m_dnstream.m_write.isOpen()) + { + if (data->m_upstream.m_read.isOpen()) { + int64_t avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); + if (0 < avail) { + int64_t const toskip = std::min(data->m_blockskip, avail); + + // consume any up front (first block) padding + if (0 < toskip) { + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, toskip); + data->m_blockskip -= toskip; + avail -= toskip; + consumed += toskip; + } + + if (0 < avail) { + int64_t const bytesleft = (data->m_bytestosend - data->m_bytessent); + int64_t const tocopy = std::min(avail, bytesleft); + + if (0 < tocopy) { + int64_t const copied(TSIOBufferCopy(data->m_dnstream.m_write.m_iobuf, data->m_upstream.m_read.m_reader, tocopy, 0)); + + data->m_bytessent += copied; + + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, copied); + + avail -= copied; + consumed += copied; + } + } + + // if hit fulfillment start bulk consuming + if (0 < avail && data->m_bytestosend <= data->m_bytessent) { + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, avail); + consumed += avail; + } + } + + if (0 < consumed) { + TSVIOReenable(data->m_dnstream.m_write.m_vio); + } + } + } + + if (0 < consumed) { + data->m_blockconsumed += consumed; + } + + return consumed; +} + +// transfer all bytes from the server (error condition) +int64_t +transfer_all_bytes(Data *const data) +{ + DEBUG_LOG("transfer_all_bytes"); + int64_t consumed = 0; + + if (data->m_dnstream.m_write.isOpen()) { + int64_t const read_avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); + + if (0 < read_avail) { + int64_t const copied(TSIOBufferCopy(data->m_dnstream.m_write.m_iobuf, data->m_upstream.m_read.m_reader, read_avail, 0)); + + if (0 < copied) { + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, copied); + consumed = copied; + } + } + } + + return consumed; +} diff --git a/plugins/experimental/slice/transfer.h b/plugins/experimental/slice/transfer.h new file mode 100644 index 00000000000..de3e1766d04 --- /dev/null +++ b/plugins/experimental/slice/transfer.h @@ -0,0 +1,34 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "Data.h" + +/** Functions to deal with the connection to the client. + * Body content transfers are handled by the client. + * New block requests are also initiated by the client. + */ + +/* transfer bytes from the server to the client + * Returns amount of bytes consumed from the reader (<= bytes written to client) + */ +int64_t transfer_content_bytes(Data *const data); // , char const * const fstr); + +// transfer all bytes from the server (error condition) +int64_t transfer_all_bytes(Data *const data); diff --git a/plugins/experimental/slice/unit-tests/slice_test.cc b/plugins/experimental/slice/unit-tests/slice_test.cc new file mode 100644 index 00000000000..96ae726a53c --- /dev/null +++ b/plugins/experimental/slice/unit-tests/slice_test.cc @@ -0,0 +1,181 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/* + * These are misc unit tests for slicer + */ + +#include "ContentRange.h" +#include "Range.h" + +#include +#include +#include +#include +#include + +std::string +testContentRange() +{ + std::ostringstream oss; + + ContentRange null; + if (null.isValid()) { + oss << "fail: null isValid test" << std::endl; + } + + ContentRange const exprange(1023, 1048576, 307232768); + + if (!exprange.isValid()) { + oss << "Fail: exprange valid" << std::endl; + oss << exprange.m_beg << ' ' << exprange.m_end << ' ' << exprange.m_length << std::endl; + } + + std::string const expstr("bytes 1023-1048575/307232768"); + + char gotbuf[1024]; + int gotlen = sizeof(gotbuf); + + bool const strstat(exprange.toStringClosed(gotbuf, &gotlen)); + + if (!strstat) { + oss << "failure status toStringClosed" << std::endl; + } else if ((int)expstr.size() != gotlen) { + oss << "Fail: expected toStringClosed length" << std::endl; + oss << "got: " << gotlen << " exp: " << expstr.size() << std::endl; + oss << "Got: " << gotbuf << std::endl; + oss << "Exp: " << expstr << std::endl; + } else if (expstr != gotbuf) { + oss << "Fail: expected toStringClosed value" << std::endl; + oss << "Got: " << gotbuf << std::endl; + oss << "Exp: " << expstr << std::endl; + } + + ContentRange gotrange; + bool const gotstat(gotrange.fromStringClosed(expstr.c_str())); + if (!gotstat) { + oss << "fail: gotstat from string" << std::endl; + } else if (gotrange.m_beg != exprange.m_beg || gotrange.m_end != exprange.m_end || gotrange.m_length != exprange.m_length) { + oss << "fail: value compare gotrange and exprange" << std::endl; + } + + std::string const teststr("bytes 0-1048575/30723276"); + if (!gotrange.fromStringClosed(teststr.c_str())) { + oss << "fail: parse teststr" << std::endl; + } + + return oss.str(); +} + +std::string +testParseRange() +{ + std::ostringstream oss; + + std::vector const teststrings = { + "bytes=0-1023", "bytes=1-1024", "bytes=11-11", + "bytes=1-", // 2nd byte to end + "Range: bytes=-13", // final 13 bytes + "bytes=3-17", // ,23-29" // open + "bytes=3 -17 ", //,18-29" // adjacent + "bytes=3- 17", //, 11-29" // overlapping + "bytes=3 - 11", //,13-17 , 23-29" // unsorted triplet + "bytes=3-11 ", //,13-17, 23-29" // unsorted triplet + "bytes=0-0", //,-1" // first and last bytes + "bytes=-20", // last 20 bytes of file + "bytes=-60-50", // invalid fully negative + "bytes=17-13", // degenerate + "bytes 0-1023/146515" // this should be rejected (Content-range) + }; // invalid + + std::vector const exps = {Range{0, 1023 + 1}, Range{1, 1024 + 1}, Range{11, 11 + 1}, Range{1, Range::maxval}, + Range{-1, -1}, Range{3, 17 + 1}, Range{3, 17 + 1}, Range{3, 17 + 1}, + Range{3, 11 + 1}, Range{3, 11 + 1}, Range{0, 1}, Range{-20, 0}, + Range{-1, -1}, Range{-1, -1}, Range{-1, -1}}; + + std::vector const expsres = {true, true, true, true, false, true, true, true, true, true, true, true, false, false, false}; + + assert(exps.size() == teststrings.size()); + + std::vector gots; + gots.reserve(exps.size()); + std::vector gotsres; + + for (std::string const &str : teststrings) { + Range rng; + gotsres.push_back(rng.fromStringClosed(str.c_str())); + gots.push_back(rng); + } + + assert(gots.size() == exps.size()); + + for (size_t index(0); index < gots.size(); ++index) { + if (exps[index] != gots[index] || expsres[index] != gotsres[index]) { + oss << "Eror parsing index: " << index << std::endl; + oss << "test: '" << teststrings[index] << "'" << std::endl; + oss << "exp: " << exps[index].m_beg << ' ' << exps[index].m_end << std::endl; + oss << "expsres: " << (int)expsres[index] << std::endl; + oss << "got: " << gots[index].m_beg << ' ' << gots[index].m_end << std::endl; + oss << "gotsres: " << (int)gotsres[index] << std::endl; + } + } + + return oss.str(); +} + +struct Tests { + typedef std::string (*TestFunc)(); + std::vector> funcs; + + void + add(TestFunc const &func, char const *const fname) + { + funcs.push_back(std::make_pair(func, fname)); + } + + int + run() const + { + int numfailed(0); + for (std::pair const &namefunc : funcs) { + TestFunc const &func = namefunc.first; + char const *const name = namefunc.second; + + std::cerr << name << " : "; + + std::string const fres(func()); + if (fres.empty()) { + std::cerr << "pass" << std::endl; + } else { + std::cerr << "FAIL" << std::endl; + std::cerr << fres << std::endl; + ++numfailed; + } + } + return numfailed; + } +}; + +int +main() +{ + Tests tests; + tests.add(testContentRange, "testContentRange"); + tests.add(testParseRange, "testParseRange"); + return tests.run(); +} diff --git a/plugins/experimental/slice/unit-tests/test_config.cc b/plugins/experimental/slice/unit-tests/test_config.cc new file mode 100644 index 00000000000..e7467cc2701 --- /dev/null +++ b/plugins/experimental/slice/unit-tests/test_config.cc @@ -0,0 +1,70 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/** + * @file test_content_range.cc + * @brief Unit test for slice ContentRange + */ + +#define CATCH_CONFIG_MAIN /* include main function */ +#include "../Config.h" +#include "catch.hpp" /* catch unit-test framework */ + +TEST_CASE("config default", "[AWS][slice][utility]") +{ + Config const config; + int64_t const defval = Config::blockbytesdefault; + CHECK(defval == config.m_blockbytes); +} + +TEST_CASE("config bytesfrom valid parsing", "[AWS][slice][utility]") +{ + std::vector const teststrings = {"1000", "1m", "5g", "2k", "3kb", "1z"}; + + std::vector const expvals = {1000, 1024 * 1024, int64_t(1024) * 1024 * 1024 * 5, 1024 * 2, 1024 * 3, 1}; + + for (size_t index = 0; index < teststrings.size(); ++index) { + std::string const &teststr = teststrings[index]; + int64_t const &exp = expvals[index]; + int64_t const got = Config::bytesFrom(teststr.c_str()); + + CHECK(got == exp); + if (got != exp) { + INFO(teststr.c_str()); + } + } +} + +TEST_CASE("config bytesfrom invalid parsing", "[AWS][slice][utility]") +{ + std::vector const badstrings = { + "abc", // alpha + "g00", // giga + "M00", // mega + "k00", // kilo + "-500" // negative + }; + + for (std::string const &badstr : badstrings) { + int64_t const val = Config::bytesFrom(badstr.c_str()); + CHECK(0 == val); + if (0 != val) { + INFO(badstr.c_str()); + } + } +} diff --git a/plugins/experimental/slice/unit-tests/test_content_range.cc b/plugins/experimental/slice/unit-tests/test_content_range.cc new file mode 100644 index 00000000000..a9876297d62 --- /dev/null +++ b/plugins/experimental/slice/unit-tests/test_content_range.cc @@ -0,0 +1,80 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/** + * @file test_content_range.cc + * @brief Unit test for slice ContentRange + */ + +#define CATCH_CONFIG_MAIN /* include main function */ +#include "catch.hpp" /* catch unit-test framework */ +#include "../ContentRange.h" + +TEST_CASE("content_range invalid state", "[AWS][slice][utility]") +{ + CHECK_FALSE(ContentRange().isValid()); // null range + CHECK_FALSE(ContentRange(1024, 1024, 4000).isValid()); // zero range + CHECK_FALSE(ContentRange(0, 1024, 1023).isValid()); // past end + CHECK_FALSE(ContentRange(-5, 13, 40).isValid()); // negative start +} + +TEST_CASE("content_range to/from string - valid", "[AWS][slice][utility]") +{ + ContentRange const exprange(1023, 1048576, 307232768); + + CHECK(exprange.isValid()); + + std::string const expstr("bytes 1023-1048575/307232768"); + + char gotbuf[1024]; + int gotlen = sizeof(gotbuf); + + bool const strstat(exprange.toStringClosed(gotbuf, &gotlen)); + + CHECK(strstat); + CHECK(gotlen == expstr.size()); + CHECK(expstr == std::string(gotbuf)); + + ContentRange gotrange; + bool const gotstat(gotrange.fromStringClosed(expstr.c_str())); + + CHECK(gotstat); + CHECK(gotrange.m_beg == exprange.m_beg); + CHECK(gotrange.m_end == exprange.m_end); + CHECK(gotrange.m_length == exprange.m_length); +} + +TEST_CASE("content_range from string - invalid", "[AWS][slice][utility]") +{ + std::vector const badstrings = { + "bytes=1024-1692", // malformed + "bytes=1023-1048575/307232768", // malformed + "bytes 1023-1022/5000", // zero size + "bytes -40-12/50", // negative start + "bytes 5-13/11" // past end + }; + + ContentRange cr; + + for (std::string const &badstr : badstrings) { + if (!cr.fromStringClosed(badstr.c_str())) { + CHECK_FALSE(cr.isValid()); + INFO(badstr.c_str()); + } + } +} diff --git a/plugins/experimental/slice/unit-tests/test_range.cc b/plugins/experimental/slice/unit-tests/test_range.cc new file mode 100644 index 00000000000..d2b1813fe3f --- /dev/null +++ b/plugins/experimental/slice/unit-tests/test_range.cc @@ -0,0 +1,99 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/** + * @file test_content_range.cc + * @brief Unit test for slice ContentRange + */ + +#define CATCH_CONFIG_MAIN /* include main function */ +#include "catch.hpp" /* catch unit-test framework */ +#include "../Range.h" + +TEST_CASE("range invalid state", "[AWS][slice][utility]") +{ + CHECK_FALSE(Range().isValid()); // null range + CHECK_FALSE(Range(1024, 1024).isValid()); // zero range + CHECK_FALSE(Range(-5, 13).isValid()); // negative start +} + +TEST_CASE("range to/from string - valid", "[AWS][slice][utility]") +{ + std::vector const teststrings = { + "bytes=0-1023", // start at zero + "bytes=1-1024", // start from non zero + "bytes=11-11", // single byte + "bytes=1-", // 2nd byte to end + "bytes=3-17", // ,23-29" // open + "bytes=3 -17 ", //,18-29" // adjacent + "bytes=3- 17", //, 11-29" // overlapping + "bytes=3 - 11", //,13-17 , 23-29" // unsorted triplet + "bytes=3-11 ", //,13-17, 23-29" // unsorted triplet + "bytes=0-0", //,-1" // first and last bytes + "bytes=-20", // last 20 bytes of file + }; + + std::vector const exps = { + Range{0, 1023 + 1}, // + Range{1, 1024 + 1}, // + Range{11, 11 + 1}, // + Range{1, Range::maxval}, // + Range{3, 17 + 1}, // + Range{3, 17 + 1}, // + Range{3, 17 + 1}, // + Range{3, 11 + 1}, // + Range{3, 11 + 1}, // + Range{0, 1}, // + Range{-20, 0} // + }; + + for (size_t index = 0; index < teststrings.size(); ++index) { + std::string const &str = teststrings[index]; + + Range got; + CHECK(got.fromStringClosed(str.c_str())); + CHECK(got.isValid()); + + if (!got.isValid()) { + INFO(str.c_str()); + } + + Range const &exp = exps[index]; + CHECK(got.m_beg == exp.m_beg); + CHECK(got.m_end == exp.m_end); + } +} + +TEST_CASE("range from string - invalid") +{ + std::vector const badstrings = { + "Range: bytes=-13", // malformed + "bytes=-60-50", // first negative, second nonzero + "bytes=17-13", // degenerate + "bytes 0-1023/146515" // malformed + }; + + Range range; + for (std::string const &badstr : badstrings) { + CHECK_FALSE(range.fromStringClosed(badstr.c_str())); + CHECK_FALSE(range.isValid()); + if (range.isValid()) { + INFO(badstr.c_str()); + } + } +} From 35ea510d3fc588fa4ec33da6b6416f983e34a730 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 10 Jan 2019 22:35:47 +0000 Subject: [PATCH 137/526] Remove TSHttpTxnRedirectRequest appears experimental and unused. --- include/ts/experimental.h | 1 - src/traffic_server/InkAPI.cc | 51 ------------------------------------ 2 files changed, 52 deletions(-) diff --git a/include/ts/experimental.h b/include/ts/experimental.h index b6cfd35e63e..8f9a818af01 100644 --- a/include/ts/experimental.h +++ b/include/ts/experimental.h @@ -195,7 +195,6 @@ tsapi TSReturnCode TSHttpTxnInfoIntGet(TSHttpTxn txnp, TSHttpTxnInfoKey key, TSM * Return: TS_SUCESS/TS_ERROR ****************************************************************************/ tsapi TSReturnCode TSHttpTxnCacheLookupCountGet(TSHttpTxn txnp, int *lookup_count); -tsapi TSReturnCode TSHttpTxnRedirectRequest(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc url_loc); tsapi TSReturnCode TSHttpTxnServerRespIgnore(TSHttpTxn txnp); tsapi TSReturnCode TSHttpTxnShutDown(TSHttpTxn txnp, TSEvent event); tsapi TSReturnCode TSHttpTxnCloseAfterResponse(TSHttpTxn txnp, int should_close); diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 61de6059a50..2969dbcbcf2 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -5122,57 +5122,6 @@ TSHttpTxnCacheLookupUrlSet(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc obj) return TS_SUCCESS; } -/* - * TSHttpTxnRedirectRequest is very odd. It is only in experimental.h. - * It is not used in any checked in code. We should probably remove this. - * SKH 1/15/2015 - */ -TSReturnCode -TSHttpTxnRedirectRequest(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc url_loc) -{ - sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); - sdk_assert(sdk_sanity_check_mbuffer(bufp) == TS_SUCCESS); - sdk_assert(sdk_sanity_check_url_handle(url_loc) == TS_SUCCESS); - - URL u, *o_url, *r_url, *client_url; - HttpSM *sm = (HttpSM *)txnp; - HttpTransact::State *s = &(sm->t_state); - - u.m_heap = ((HdrHeapSDKHandle *)bufp)->m_heap; - u.m_url_impl = (URLImpl *)url_loc; - if (!u.valid()) { - return TS_ERROR; - } - - client_url = s->hdr_info.client_request.url_get(); - if (!(client_url->valid())) { - return TS_ERROR; - } - - s->redirect_info.redirect_in_process = true; - o_url = &(s->redirect_info.original_url); - if (!o_url->valid()) { - o_url->create(nullptr); - o_url->copy(client_url); - } - client_url->copy(&u); - - r_url = &(s->redirect_info.redirect_url); - if (!r_url->valid()) { - r_url->create(nullptr); - } - r_url->copy(&u); - - s->hdr_info.server_request.destroy(); - - s->request_sent_time = 0; - s->response_received_time = 0; - s->cache_info.write_lock_state = HttpTransact::CACHE_WL_INIT; - s->next_action = HttpTransact::SM_ACTION_REDIRECT_READ; - - return TS_SUCCESS; -} - /** * timeout is in msec * overrides as proxy.config.http.transaction_active_timeout_out From 9c49e84dbbd34e6d24fd4a522699d10dc5e88fab Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 8 Jan 2019 12:48:13 -0800 Subject: [PATCH 138/526] Calls SSL child config callback after cert is loaded for both key parts This plays nicer with filesystems who's mtime can change on file read --- iocore/net/SSLUtils.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index d0ae28386c5..7c9fbaf1ff2 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -1738,10 +1738,7 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu X509_free(cert); goto fail; } - certList.push_back(cert); - if (SSLConfigParams::load_ssl_file_cb) { - SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str(), CONFIG_FLAG_UNVERSIONED); - } + // Load up any additional chain certificates SSL_CTX_add_extra_chain_cert_bio(ctx, bio); @@ -1750,6 +1747,11 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu goto fail; } + certList.push_back(cert); + if (SSLConfigParams::load_ssl_file_cb) { + SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str(), CONFIG_FLAG_UNVERSIONED); + } + // Must load all the intermediate certificates before starting the next chain // First, load any CA chains from the global chain file. This should probably From 2f1bf2549674083de74a52753ed011fd57de657f Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Thu, 10 Jan 2019 16:48:51 -0800 Subject: [PATCH 139/526] Bug in traffic_ctl, formatting output for config changes --- src/traffic_ctl/config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/traffic_ctl/config.cc b/src/traffic_ctl/config.cc index 378fb4785a6..22198d23a29 100644 --- a/src/traffic_ctl/config.cc +++ b/src/traffic_ctl/config.cc @@ -416,7 +416,7 @@ CtrlEngine::config_diff() std::cout << rec_labelof(desc->rec_class) << ' ' << desc->rec_name << ' ' << rec_typeof(desc->rec_type) << ' ' << current.c_str() << " # default: " << deflt.c_str() << std::endl; } else { - std::cout << desc->rec_name << "has changed" << std::endl; + std::cout << desc->rec_name << " has changed" << std::endl; std::cout << "\tCurrent Value: " << current.c_str() << std::endl; std::cout << "\tDefault Value: " << deflt.c_str() << std::endl; } From c929ed6fdee34a67529b55f9fe1c47d449ee854f Mon Sep 17 00:00:00 2001 From: Oknet Date: Wed, 10 Oct 2018 13:50:10 +0800 Subject: [PATCH 140/526] Assert if operations on keep_alive_queue and active_queue are not thread-safe The `vc->add_to_keep_alive_queue()` is called by: - Http1ClientSession::release() - Http2ClientSession::cleanup_streams() - Http2ClientSession::release_stream() And the `NetHandler::remove_from_active_queue()` is called by `vc->add_to_keep_alive_queue()`. The NetHandler::keep_alive_queue and NetHandler::active_queue are internal queue of NetHandler. We must have acquired the NetHandler's lock before doing anything with them. When reenable a HttpSM from plugin, the HttpSM would be run into ethread that different from client_vc lives because TSHttpSMCallback is scheduled by `eventProcessor.schedule_imm()`. --- iocore/net/UnixNet.cc | 6 ++++++ iocore/net/UnixNetVConnection.cc | 31 +++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc index eb95841f934..76a10a46620 100644 --- a/iocore/net/UnixNet.cc +++ b/iocore/net/UnixNet.cc @@ -673,6 +673,7 @@ void NetHandler::add_to_keep_alive_queue(UnixNetVConnection *vc) { Debug("net_queue", "NetVC: %p", vc); + ink_assert(mutex->thread_holding == this_ethread()); if (keep_alive_queue.in(vc)) { // already in the keep-alive queue, move the head @@ -692,6 +693,8 @@ void NetHandler::remove_from_keep_alive_queue(UnixNetVConnection *vc) { Debug("net_queue", "NetVC: %p", vc); + ink_assert(mutex->thread_holding == this_ethread()); + if (keep_alive_queue.in(vc)) { keep_alive_queue.remove(vc); --keep_alive_queue_size; @@ -704,6 +707,7 @@ NetHandler::add_to_active_queue(UnixNetVConnection *vc) Debug("net_queue", "NetVC: %p", vc); Debug("net_queue", "max_connections_per_thread_in: %d active_queue_size: %d keep_alive_queue_size: %d", max_connections_per_thread_in, active_queue_size, keep_alive_queue_size); + ink_assert(mutex->thread_holding == this_ethread()); // if active queue is over size then close inactive connections if (manage_active_queue() == false) { @@ -728,6 +732,8 @@ void NetHandler::remove_from_active_queue(UnixNetVConnection *vc) { Debug("net_queue", "NetVC: %p", vc); + ink_assert(mutex->thread_holding == this_ethread()); + if (active_queue.in(vc)) { active_queue.remove(vc); --active_queue_size; diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 2e9edfcd988..635c98f5e99 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -1451,25 +1451,48 @@ UnixNetVConnection::migrateToCurrentThread(Continuation *cont, EThread *t) void UnixNetVConnection::add_to_keep_alive_queue() { - nh->add_to_keep_alive_queue(this); + MUTEX_TRY_LOCK(lock, nh->mutex, this_ethread()); + if (lock.is_locked()) { + nh->add_to_keep_alive_queue(this); + } else { + ink_release_assert(!"BUG: It must have acquired the NetHandler's lock before doing anything on keep_alive_queue."); + } } void UnixNetVConnection::remove_from_keep_alive_queue() { - nh->remove_from_keep_alive_queue(this); + MUTEX_TRY_LOCK(lock, nh->mutex, this_ethread()); + if (lock.is_locked()) { + nh->remove_from_keep_alive_queue(this); + } else { + ink_release_assert(!"BUG: It must have acquired the NetHandler's lock before doing anything on keep_alive_queue."); + } } bool UnixNetVConnection::add_to_active_queue() { - return nh->add_to_active_queue(this); + bool result = false; + + MUTEX_TRY_LOCK(lock, nh->mutex, this_ethread()); + if (lock.is_locked()) { + result = nh->add_to_active_queue(this); + } else { + ink_release_assert(!"BUG: It must have acquired the NetHandler's lock before doing anything on active_queue."); + } + return result; } void UnixNetVConnection::remove_from_active_queue() { - nh->remove_from_active_queue(this); + MUTEX_TRY_LOCK(lock, nh->mutex, this_ethread()); + if (lock.is_locked()) { + nh->remove_from_active_queue(this); + } else { + ink_release_assert(!"BUG: It must have acquired the NetHandler's lock before doing anything on active_queue."); + } } int From 72e9db6a523e54c937f21d91bc74a608237de7e0 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Thu, 29 Nov 2018 14:58:51 -0800 Subject: [PATCH 141/526] Compare servers script for comparing responses from new and old versions of ATS --- tools/compare_servers.pl | 245 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100755 tools/compare_servers.pl diff --git a/tools/compare_servers.pl b/tools/compare_servers.pl new file mode 100755 index 00000000000..7f1bcb86bb3 --- /dev/null +++ b/tools/compare_servers.pl @@ -0,0 +1,245 @@ +#!/usr/bin/perl +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +use strict; +use warnings; +use Getopt::Long; +use Data::Dumper; +use Net::hostent; +use Socket; +use LWP::UserAgent; +use Digest::SHA1; + +my $verbose = 0; + +#---------------------------------------------------------------------------- +sub usage() { + print STDERR "USAGE: compare_hosts.pl --verbose level --host1 testing_host --host2 valid_host --file url_file\n\n"; + print STDERR "\t--host1 The host running the newest version\n"; + print STDERR "\t--host2 The host running the older version\n"; + print STDERR "\t--file A file that contains a list of URLs\n"; + print STDERR "\t--verbose verbose level 1-3, 1 is the least verbose\n\n"; + print STDERR "Example:\n"; + print STDERR "\tcompare_hosts.pl --host1 new_ats --host2 old_ats --file top_1000_urls\n"; + exit 1; +} + +#---------------------------------------------------------------------------- +sub compareHeaderNames($$) { + my($response1, $response2) = @_; + + my @names1 = $response1->header_field_names; + my @names2 = $response2->header_field_names; + + my %hash2; + $hash2{$_} = 1 for (@names2); + my %hash1; + $hash1{$_} = 1 for (@names1); + + my $return_val = 0; # header names match + + foreach my $name (@names1) { + if (!defined $hash2{$name}) { + print "\t\t- $name header not found on host2\n" if $verbose >= 2; + $return_val = 1; + } + } + + foreach my $name (@names2) { + if (!defined $hash1{$name}) { + print "\t\t- $name header not found on host1\n" if $verbose >= 2; + $return_val = 1; + } + } + + return $return_val; +} + +#---------------------------------------------------------------------------- +sub compareHeaderValues($$) { + my($response1, $response2) = @_; + + my @test_headers = qw(ETag Cache-Control Connection Accept-Ranges Server Content-Type Access-Control-Allow-Methods Access-Control-Allow-Origin Strict-Transport-Security); + my $return_val = 0; # header valuse match + + if ($verbose >= 3) { + foreach my $field ($response1->header_field_names) { + print "\t\t\t~ " . $field . ": " . $response1->header($field) . "\n"; + } + + print "\t\tHost2: \n"; + + foreach my $field ($response2->header_field_names) { + print "\t\t\t~ " . $field . ": " . $response2->header($field) . "\n"; + } + } + + # Test specific headers that are defined above + foreach my $field (@test_headers) { + my $value1 = $response1->header($field); + my $value2 = $response2->header($field); + + if (defined $value1 && defined $value2) { + if ($value1 ne $value2) { + print "\t\t- $field: $value1 ne $value2\n" if $verbose; + print "\t\t\t - Via host1: " . $response1->header('Via') . " host2: " . $response2->header('Via') . "\n" if $verbose; + print "\t\t\t - Last-Modified host1: " . $response1->header('Last-Modified') . " host2: " . $response2->header('Last-Modified') . "\n" if $verbose; + if (defined $response2->header('Content-Encoding')) { + print "\t\t\t - Content-Encoding host1: " . $response1->header('Content-Encoding') . " host2: " . $response2->header('Content-Encoding') . "\n"; + } else { + print "\t\t\t - Content-Encoding host1: " . $response1->header('Content-Encoding') . " host2: ''\n"; + } + $return_val = 1; + } else { + print "\t\t- $field: $value1 eq $value2\n" if $verbose >= 2; + } + } + } + return $return_val; +} + +#---------------------------------------------------------------------------- +{ + my %stats; + + $ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = '0'; + my($host1, $host2, $file); + GetOptions ("host1=s" => \$host1, + "host2=s" => \$host2, + "file=s" => \$file, + "verbose=f" => \$verbose) || die $!; + + usage() if (! defined $host1 || ! defined $host2 || ! defined $file); + + my $count = 0; + my $status_error = 0; + my $sha_error = 0; + my $header_names_mismatch = 0; + my $header_values_mismatch = 0; + + my $host1_addr = inet_ntoa(inet_aton($host1)); + my $host2_addr = inet_ntoa(inet_aton($host2)); + + print "Testing with host1: $host1 ($host1_addr) - host2: $host2 ($host2_addr)\n"; + print '-' x 78, "\n"; + + open(FILE, $file) || die $!; + + # Create a user agent object + my $ua1 = LWP::UserAgent->new(keep_alive => 100); + $ua1->agent("MyApp/0.1 "); + + # Create a user agent object + my $ua2 = LWP::UserAgent->new(keep_alive => 100); + $ua2->agent("MyApp/0.1 "); + + while (my $url = ) { + next if ($url =~ m|hc.l.yimg.com|); + chomp $url; + my $exit = 0; + + if ($url =~ m|(https?)://([^/]+)(.+)|) { + + my $scheme = $1; + my $host = $2; + my $path = $3; + + $count++; + print "Test $count - URL: $url\n"; + + my $port = 80; + $port = 443 if $scheme eq 'https'; + + my $request1 = HTTP::Request->new(GET => "${scheme}://${host1_addr}${path}"); + $request1->header('Host' => $host); + my $response1 = $ua1->request($request1); + + my $request2 = HTTP::Request->new(GET => "${scheme}://${host2_addr}${path}"); + $request2->header('Host' => $host); + $request2->header('Accept-Encoding' => 'deflate'); + my $response2 = $ua2->request($request2); + + print "\tStatus code for host1: " . $response1->code . " - host2: " . $response2->code . "\n" if $verbose; + + my $sha1 = Digest::SHA1->new; + $sha1->add($response1->content); + my $digest1 = $sha1->hexdigest; + open(FILE1, "> /tmp/tmp1"); + open(FILE2, "> /tmp/tmp2"); + print FILE1 $response1->content; + print FILE2 $response2->content; + close FILE1; + close FILE2; + #print $response1->content, "\n"; # for internal debugging + #print $response2->content, "\n"; # for internal debugging + + my $sha2 = Digest::SHA1->new; + $sha2->add($response2->content); + my $digest2 = $sha2->hexdigest; + + print "\tSHA hash for host1: $digest1 - host2: $digest2\n" if $verbose; + + # Build up stats + if ($response1->status_line eq $response2->status_line) { + + # Do the hashes + if ($digest1 eq $digest2) { + $stats{stat_line_match}->{$response1->code}->{sha_match}++; + print "\tResponse code: " . $response1->code . " - Status lines and SHA1 of response bodies match\n"; + } else { + $stats{stat_line_match}->{$response1->code}->{sha_nomatch}++; + print "\tResponse code: " . $response1->code . " - Status lines match SHA1 doesn't match\n"; + $sha_error++; + #$exit = 1 if $response1->code == 200; # for internal debugging + } + + # Compare the header field names + if (compareHeaderNames($response1, $response2) == 0) { + $stats{stat_line_match}->{$response1->code}->{field_names_match}++; + } else { + $stats{stat_line_match}->{$response1->code}->{field_names_nomatch}++; + $header_names_mismatch++; + } + + # Compare the values of the header fields + if (compareHeaderValues($response1, $response2) == 0) { + $stats{stat_line_match}->{$response1->code}->{field_values_match}++; + } else { + $stats{stat_line_match}->{$response1->code}->{field_values_nomatch}++; + $header_values_mismatch++; + } + } else { + $status_error++; + $stats{stat_line_nomatch}++; + print "\tERROR: status lines don't match\n"; + } + + last if $exit; + } + } + + print '-' x 78, "\n"; + print "SUMMARY:\n"; + print "URLs tested: $count\n"; + print "Status line mismatches: $status_error\n"; + print "SHA1 mismatches: $sha_error\n"; + print "Responses with header names mismatches: $header_names_mismatch\n"; + print "Responses with header values mismatches: $header_values_mismatch\n"; + print Dumper \%stats if $verbose; +} + From ed319d292053dc81ed2c968d5ce30f75db861993 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 10 Jan 2019 11:25:33 -0700 Subject: [PATCH 142/526] Adds some missing packages to the Dockerfile --- ci/docker/yum/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 82248c9e264..eeff9ca0fce 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -48,11 +48,12 @@ RUN yum -y update; \ # Autoconf autoconf automake libtool \ # Various other tools - git rpm-build distcc-server file; \ + git rpm-build distcc-server file wget openssl hwloc; \ # Devel packages that ATS needs yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel kyotocabinet-devel luajit-devel \ - brotli-devel ImageMagick-devel ImageMagick-c++-devel perl-ExtUtils-MakeMaker perl-Digest-SHA; \ + brotli-devel ImageMagick-devel ImageMagick-c++-devel \ + perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 python3-virtualenv python-virtualenv httpd-tools procps-ng nmap-ncat; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) From 7ed14f5b3b956ae0dd8b8fd43311c51d500b4401 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 9 Jan 2019 16:53:59 -0700 Subject: [PATCH 143/526] Don't allow the old body to be used when refreshing This fixes #4753. The problem happens because the previous body (with CL: > 0) is "moved", which then is served even though the body is now empty (CL: 0). --- iocore/cache/CacheWrite.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index cd873ef40a6..f81cf8da4e7 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -90,7 +90,7 @@ CacheVC::updateVector(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) return openWriteCloseDir(EVENT_IMMEDIATE, nullptr); } } - if (update_key == od->single_doc_key && (total_len || !vec)) { + if (update_key == od->single_doc_key && (total_len || f.allow_empty_doc || !vec)) { od->move_resident_alt = false; } } @@ -1113,11 +1113,11 @@ CacheVC::openWriteCloseDir(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED * } if (is_debug_tag_set("cache_update")) { if (f.update && closed > 0) { - if (!total_len && alternate_index != CACHE_ALT_REMOVED) { + if (!total_len && !f.allow_empty_doc && alternate_index != CACHE_ALT_REMOVED) { Debug("cache_update", "header only %d (%" PRIu64 ", %" PRIu64 ")", DIR_MASK_TAG(first_key.slice32(2)), update_key.b[0], update_key.b[1]); - } else if (total_len && alternate_index != CACHE_ALT_REMOVED) { + } else if ((total_len || f.allow_empty_doc) && alternate_index != CACHE_ALT_REMOVED) { Debug("cache_update", "header body, %d, (%" PRIu64 ", %" PRIu64 "), (%" PRIu64 ", %" PRIu64 ")", DIR_MASK_TAG(first_key.slice32(2)), update_key.b[0], update_key.b[1], earliest_key.b[0], earliest_key.b[1]); } else if (!total_len && alternate_index == CACHE_ALT_REMOVED) { From c77817601a8a45b9ea4f076c8e58a4e4a0b1a2f9 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Wed, 23 May 2018 09:06:33 -0700 Subject: [PATCH 144/526] Autest: Test revalidating cached objects- Chunked object with Range request and 304 (for #3413)- INM, IMS validations --- .../headers/cache_and_error_nobody.gold | 11 ++ ...cache_and_req_body-hit-stale-206-etag.gold | 13 ++ .../cache_and_req_body-hit-stale-206.gold | 13 ++ .../cache_and_req_body-hit-stale-INM.gold | 13 ++ .../headers/cache_and_req_body-hit-stale.gold | 13 ++ .../headers/cache_and_req_body-hit.gold | 3 +- .../headers/cache_and_req_body-hit_close.gold | 3 +- .../headers/cache_and_req_body-miss.gold | 6 +- .../headers/cache_and_req_body.test.py | 3 +- .../cache_and_req_nobody-hit-stale.gold | 11 ++ .../gold_tests/headers/cachedIMSRange.test.py | 158 ++++++++++++++++++ 11 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 tests/gold_tests/headers/cache_and_error_nobody.gold create mode 100644 tests/gold_tests/headers/cache_and_req_body-hit-stale-206-etag.gold create mode 100644 tests/gold_tests/headers/cache_and_req_body-hit-stale-206.gold create mode 100644 tests/gold_tests/headers/cache_and_req_body-hit-stale-INM.gold create mode 100644 tests/gold_tests/headers/cache_and_req_body-hit-stale.gold create mode 100644 tests/gold_tests/headers/cache_and_req_nobody-hit-stale.gold create mode 100644 tests/gold_tests/headers/cachedIMSRange.test.py diff --git a/tests/gold_tests/headers/cache_and_error_nobody.gold b/tests/gold_tests/headers/cache_and_error_nobody.gold new file mode 100644 index 00000000000..8e10e6fe171 --- /dev/null +++ b/tests/gold_tests/headers/cache_and_error_nobody.gold @@ -0,0 +1,11 @@ +HTTP/1.1 404 Not Found +Content-Length: 0 +Cache-Control: max-age=3 +Date: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: `` + diff --git a/tests/gold_tests/headers/cache_and_req_body-hit-stale-206-etag.gold b/tests/gold_tests/headers/cache_and_req_body-hit-stale-206-etag.gold new file mode 100644 index 00000000000..8bc8a2a488a --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_body-hit-stale-206-etag.gold @@ -0,0 +1,13 @@ +HTTP/1.1 206 Partial Content +Cache-Control: max-age=1 +Content-Length: 2 +Date: `` +Etag: myetag +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: hit-stale + +xx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body-hit-stale-206.gold b/tests/gold_tests/headers/cache_and_req_body-hit-stale-206.gold new file mode 100644 index 00000000000..03cc794eb46 --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_body-hit-stale-206.gold @@ -0,0 +1,13 @@ +HTTP/1.1 206 Partial Content +Last-Modified: `` +Cache-Control: max-age=1 +Content-Length: 2 +Date: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: hit-stale + +xx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body-hit-stale-INM.gold b/tests/gold_tests/headers/cache_and_req_body-hit-stale-INM.gold new file mode 100644 index 00000000000..2dc3eb84e90 --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_body-hit-stale-INM.gold @@ -0,0 +1,13 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=1 +Content-Length: 3 +Date: `` +Etag: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: hit-stale + +xxx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body-hit-stale.gold b/tests/gold_tests/headers/cache_and_req_body-hit-stale.gold new file mode 100644 index 00000000000..4e9f9e35c2b --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_body-hit-stale.gold @@ -0,0 +1,13 @@ +HTTP/1.1 200 OK +Last-Modified: `` +Cache-Control: max-age=1 +Content-Length: 3 +Date: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: hit-stale + +xxx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body-hit.gold b/tests/gold_tests/headers/cache_and_req_body-hit.gold index 90d4a9743af..09c06a1ccd2 100644 --- a/tests/gold_tests/headers/cache_and_req_body-hit.gold +++ b/tests/gold_tests/headers/cache_and_req_body-hit.gold @@ -1,5 +1,6 @@ HTTP/1.1 200 OK -Cache-Control: max-age=300 +Last-Modified:`` +Cache-Control: max-age=1 Content-Length: 3 Date: `` Age: `` diff --git a/tests/gold_tests/headers/cache_and_req_body-hit_close.gold b/tests/gold_tests/headers/cache_and_req_body-hit_close.gold index ed944d21a9d..9ccf980e753 100644 --- a/tests/gold_tests/headers/cache_and_req_body-hit_close.gold +++ b/tests/gold_tests/headers/cache_and_req_body-hit_close.gold @@ -1,5 +1,6 @@ HTTP/1.1 200 OK -Cache-Control: max-age=300 +Last-Modified:`` +Cache-Control: max-age=1 Content-Length: 3 Date: `` Age: `` diff --git a/tests/gold_tests/headers/cache_and_req_body-miss.gold b/tests/gold_tests/headers/cache_and_req_body-miss.gold index aa174cdc2a6..46d4e085907 100644 --- a/tests/gold_tests/headers/cache_and_req_body-miss.gold +++ b/tests/gold_tests/headers/cache_and_req_body-miss.gold @@ -1,11 +1,13 @@ HTTP/1.1 200 OK -Cache-Control: max-age=300 +Last-Modified: `` +Cache-Control: max-age=1 Content-Length: 3 Date: `` -Age: `` +Age: 0 Connection: keep-alive Via: `` Server: `` +X-Cache-Key: http://127.0.0.1`` X-Cache: miss xxx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body.test.py b/tests/gold_tests/headers/cache_and_req_body.test.py index a952cc20221..8d3afc070e1 100644 --- a/tests/gold_tests/headers/cache_and_req_body.test.py +++ b/tests/gold_tests/headers/cache_and_req_body.test.py @@ -36,7 +36,7 @@ #**testname is required** testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-Control: max-age=300\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLast-Modified: Tue, 08 May 2018 15:49:41 GMT\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} server.addResponse("sessionlog.json", request_header, response_header) # ATS Configuration @@ -82,4 +82,3 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit_close.gold" tr.StillRunningAfter = ts - diff --git a/tests/gold_tests/headers/cache_and_req_nobody-hit-stale.gold b/tests/gold_tests/headers/cache_and_req_nobody-hit-stale.gold new file mode 100644 index 00000000000..46ca77ec996 --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_nobody-hit-stale.gold @@ -0,0 +1,11 @@ +HTTP/1.1 200 OK +Content-Length: 0 +Cache-Control: max-age=3 +Date: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: `` + diff --git a/tests/gold_tests/headers/cachedIMSRange.test.py b/tests/gold_tests/headers/cachedIMSRange.test.py new file mode 100644 index 00000000000..b17a386eb89 --- /dev/null +++ b/tests/gold_tests/headers/cachedIMSRange.test.py @@ -0,0 +1,158 @@ +''' +Test cached responses and requests with bodies +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Test revalidating cached objects +''' + +testName = "RevalidateCacheObject" + +# Needs Curl +Test.SkipUnless( + Condition.HasProgram("curl", "curl needs to be installed on system for this test to work"), +) +Test.ContinueOnFail = True + +# Set up Origin server +# request_header is from ATS to origin; response from Origin to ATS +# lookup_key is to make unique response in origin for header "UID" that will pass in ATS request +server = Test.MakeOriginServer("server",lookup_key="{%UID}") +# Initial request +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\nUID: Fill\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLast-Modified: Tue, 08 May 2018 15:49:41 GMT\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request_header, response_header) +# IMS revalidation request +request_IMS_header = {"headers": "GET / HTTP/1.1\r\nUID: IMS\r\nIf-Modified-Since: Tue, 08 May 2018 15:49:41 GMT\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_IMS_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993"} +server.addResponse("sessionlog.json", request_IMS_header, response_IMS_header) + +# EtagFill +request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagFill\r\n\r\n", "timestamp": "1469733493.993"} +response_etagfill_header = {"headers": "HTTP/1.1 200 OK\r\nETag: myetag\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request_etagfill_header, response_etagfill_header) +# INM revalidation +request_INM_header = {"headers": "GET /etag HTTP/1.1\r\nUID: INM\r\nIf-None-Match: myetag\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993"} +response_INM_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nETag: myetag\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993"} +server.addResponse("sessionlog.json", request_INM_header, response_INM_header) + +# object changed to 0 byte +request_noBody_header = {"headers": "GET / HTTP/1.1\r\nUID: noBody\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_noBody_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_noBody_header, response_noBody_header) + +# etag object now is a 404. Yeah, 404s don't usually have Cache-Control, but, ATS's default is to cache 404s for a while. +request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagError\r\n\r\n", "timestamp": "1469733493.993"} +response_etagfill_header = {"headers": "HTTP/1.1 404 Not Found\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_etagfill_header, response_etagfill_header) + +# ATS Configuration +ts = Test.MakeATSProcess("ts", select_ports=False) +ts.Disk.plugin_config.AddLine('xdebug.so') +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.response_via_str': 3, + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, +}) + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Test 0 - Fill a 3 byte object with Last-Modified time into cache. +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: Fill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-miss.gold" +tr.StillRunningAfter = ts + +# Test 1 - Once it goes stale, fetch it again. We expect Origin to get IMS request, and serve a 304. We expect ATS to refresh the object, and give a 200 to user +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: IMS" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale.gold" +tr.StillRunningAfter = ts + +# Test 2 - Once it goes stale, fetch it via a range request. We expect Origin to get IMS request, and serve a 304. We expect ATS to refresh the object, and give a 206 to user +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl --range 0-1 -s -D - -v --ipv4 --http1.1 -H"UID: IMS" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-206.gold" +tr.StillRunningAfter = ts + +# Test 3 - Fill a new object with an Etag. Not checking the output here. +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagFill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +# Test 4 - Once the etag object goes stale, fetch it again. We expect Origin to get INM request, and serve a 304. We expect ATS to refresh the object, and give a 200 to user +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: INM" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-INM.gold" +tr.StillRunningAfter = ts + +# Test 5 - Once the etag object goes stale, fetch it via a range request. We expect Origin to get INM request, and serve a 304. We expect ATS to refresh the object, and give a 206 to user +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl --range 0-1 -s -D - -v --ipv4 --http1.1 -H"UID: INM" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-206-etag.gold" +tr.StillRunningAfter = ts + +# Test 6 - The origin changes the initial LMT object to 0 byte. We expect ATS to fetch and serve the new 0 byte object. +tr = Test.AddTestRun() +tr.DelayStart=3 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: noBody" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_nobody-hit-stale.gold" +tr.StillRunningAfter = ts + +# Test 7 - Fetch the new 0 byte object again when fresh in cache to ensure its still a 0 byte object. +tr = Test.AddTestRun() +tr.DelayStart=3 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: noBody" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_nobody-hit-stale.gold" +tr.StillRunningAfter = ts + +# Test 8 - The origin changes the etag object to 0 byte 404. We expect ATS to fetch and serve the 404 0 byte object. +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_error_nobody.gold" +tr.StillRunningAfter = ts + +# Test 9 - Fetch the 0 byte etag object again when fresh in cache to ensure its still a 0 byte object +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_error_nobody.gold" +tr.StillRunningAfter = ts \ No newline at end of file From 1cadfc4328ebf673e5a3e785de981157e44cbf2f Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 9 Jan 2019 16:24:58 -0600 Subject: [PATCH 145/526] TLS Bridge: Cleanup, fix TSError for more consistency. --- .../plugins/example-plugins/tls_bridge.en.rst | 31 ++++++----- plugins/experimental/tls_bridge/tls_bridge.cc | 54 +++++++++---------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst index 57a4661d7cd..29821e65d5b 100644 --- a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst +++ b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst @@ -66,7 +66,9 @@ Configuration ============= |Name| requires at least two instances of |TS| (Ingress and Peer). The client connects to the -ingress |TS|, and the peer |TS| connects to the service. +ingress |TS|, and the peer |TS| connects to the service. The Peer could in theory be configured to +connect on to a further |TS| instance, acting as the ingress to that peer, but that doesn't seem a +useful case. #. Disable caching on |TS| in ``records.config``:: @@ -87,12 +89,12 @@ ingress |TS|, and the peer |TS| connects to the service. The Ingress |TS| also needs ``proxy.config.http.server_ports`` configured to have proxy ports to which the Client can connect. -#. Remap on the ingress is not required, however, |TS| requires remap in order to accept the - request. This can be done by disabling the remap requirement:: +#. By default |TS| requires remap in order to allow to outbound request to the peer. To disable this + requirement and allow all connections, use the setting :: CONFIG proxy.config.url_remap.remap_required INT 0 - In this case |TS| will act as an open proxy which is unlikely to be a good idea, therefore if + In this case |TS| will act as an open proxy which is unlikely to be a good idea. Therefore if this approach is used |TS| will need to run in a restricted environment or use access control (via ``ip_allow.config`` or ``iptables``). @@ -104,22 +106,23 @@ ingress |TS|, and the peer |TS| connects to the service. Remapping will be disabled for the user agent connection and so it will not need a rule. #. If remap is required on the peer to enable the outbound connection from the peer to the service - (it is not explicitly disabled) the destination port must be + (e.g. required remapping is not explicitly disabled) the destination port must be explicitly stated [#]_. E.g. :: map https://service:4443 https://service:4443 Note this remap rule cannot alter the actual HTTP transactions between the client and service - because those happen inside what is effectively a tunnel between the client and service, supported - by the two |TS| instances. This rule just allows the ``CONNECT`` sent from the ingress to cause a - tunnel connection from the peer to the service. + because those happen inside what is effectively a tunnel between the client and service, + supported by the two |TS| instances. This rule serves to allows the ``CONNECT`` sent from the + ingress to cause a tunnel connection from the peer to the service. #. Configure the Ingress |TS| to verify the Peer server certificate:: CONFIG proxy.config.ssl.client.verify.server.policy STRING ENFORCED -#. Configure Certificate Authority used by the Ingress |TS| to verify the Peer server certificate. If this - is a directory all of the certificates in the directory are treated as Certificate Authorites. :: +#. Configure Certificate Authority used by the Ingress |TS| to verify the Peer server certificate. + If this is a directory, all of the certificates in the directory are treated as Certificate + Authorities. :: CONFIG proxy.config.ssl.client.CA.cert.filename STRING @@ -150,9 +153,9 @@ ingress |TS|, and the peer |TS| connects to the service. tls_bridge.so .*[.]service[.]com peer.ats:4443 .*[.]altsvc.ats altpeer.ats:4443 - Mappings can also be specified in an external file. For instance, if the file named "bridge.config" in the default |TS| - configuration directory contained mappings, the ``plugin.config`` configuration line could look - like :: + Mappings can also be specified in an external file. For instance, if there was file named + "bridge.config" in the default |TS| configuration directory which contained mappings, the + ``plugin.config`` configuration line could look like :: tls_bridge.so .*[.]service[.]com peer.ats:4443 --file bridge.config @@ -162,7 +165,7 @@ ingress |TS|, and the peer |TS| connects to the service. These are not identical - direct mappings and file mappings are processed in order. This means in the first example, the direct mapping is checked before any mappping in "bridge.config", and in - the latter example the mappings in "bridge.config" are checked before the direct mapping. There + the latter example the mappings in "bridge.config" are checked before the direct mappings. There can be multiple "--file" arguments, which are processed in the order they appear in "plugin.config". The file name can be absolute, or relative. If the file name is relative, it is relative to the |TS| configuration directory. Therefore, in these examples, "bridge.config" must diff --git a/plugins/experimental/tls_bridge/tls_bridge.cc b/plugins/experimental/tls_bridge/tls_bridge.cc index e8b78b6ef91..d295d08d346 100644 --- a/plugins/experimental/tls_bridge/tls_bridge.cc +++ b/plugins/experimental/tls_bridge/tls_bridge.cc @@ -1,19 +1,16 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. */ #include "ts/ts.h" @@ -24,19 +21,21 @@ #include "tscore/ts_file.h" #include "regex.h" -#define PLUGIN_NAME "TLS Bridge" -#define PLUGIN_TAG "tls_bridge" - using ts::TextView; namespace { +constexpr char const PLUGIN_NAME[] = "TLS Bridge"; +constexpr char const PLUGIN_TAG[] = "tls_bridge"; + // Base format string for making the internal CONNECT. char const CONNECT_FORMAT[] = "CONNECT https:%.*s HTTP/1.1\r\n\r\n"; +// TextView of the 'CONNECT' method string. const TextView METHOD_CONNECT{TS_HTTP_METHOD_CONNECT, TS_HTTP_LEN_CONNECT}; constexpr TextView CONFIG_FILE_ARG{"--file"}; const std::string TS_CONFIG_DIR{TSConfigDirGet()}; + }; // namespace /* ------------------------------------------------------------------------------------ */ @@ -70,6 +69,8 @@ class BridgeConfig using self_type = BridgeConfig; /// Construct an item. + /// @internal Pass in the compiled regex because no instance of this is created if + /// the regex doesn't compile successfully. Item(std::string_view pattern, Regex &&r, std::string_view service) : _pattern(pattern), _r(std::move(r)), _service(service) {} std::string _pattern; ///< Original configuration regular expression. @@ -114,13 +115,12 @@ BridgeConfig::load_pair(std::string_view rxp, std::string_view service, ts::file if (r.compile(pattern.c_str(), Regex::ANCHORED)) { _items.emplace_back(rxp, std::move(r), service); } else { - char buff[std::numeric_limits::digits10 + 2]; - char const *place = ""; + char buff[std::numeric_limits::digits10 + 2] = ""; if (ln) { snprintf(buff, sizeof(buff), " on line %d", ln); - place = buff; } - TSError("%s: Failed to compile regular expression '%.*s' in %s%s", PLUGIN_TAG, int(rxp.size()), rxp.data(), src.c_str(), place); + TSError("[%s] Failed to compile regular expression '%.*s' in %s%s", PLUGIN_NAME, int(rxp.size()), rxp.data(), src.c_str(), + buff); } } @@ -132,7 +132,7 @@ BridgeConfig::load_config(int argc, const char *argv[]) for (int i = 0; i < argc; i += 2) { if (argv[i] == CONFIG_FILE_ARG) { if (i + 1 >= argc) { - TSError("%s: Invalid '%.*s' argument - no file name found.", PLUGIN_TAG, int(CONFIG_FILE_ARG.size()), + TSError("[%s] Invalid '%.*s' argument - no file name found.", PLUGIN_NAME, int(CONFIG_FILE_ARG.size()), CONFIG_FILE_ARG.data()); } else { ts::file::path fp(argv[i + 1]); @@ -143,7 +143,7 @@ BridgeConfig::load_config(int argc, const char *argv[]) // bulk load the file. std::string content{ts::file::load(fp, ec)}; if (ec) { - TSError("%s: Invalid '%.*s' argument - unable to read file '%s' : %s.", PLUGIN_TAG, int(CONFIG_FILE_ARG.size()), + TSError("[%s] Invalid '%.*s' argument - unable to read file '%s' : %s.", PLUGIN_NAME, int(CONFIG_FILE_ARG.size()), CONFIG_FILE_ARG.data(), fp.c_str(), ec.message().c_str()); } else { @@ -162,7 +162,7 @@ BridgeConfig::load_config(int argc, const char *argv[]) service.ltrim_if(&isspace); // dump extra separating space. // Only need to check service, as if the line isn't empty rxp will also be non-empty. if (service.empty()) { - TSError("%s: Invalid line %d in '%s' - no destination service found.", PLUGIN_TAG, line_no, fp.c_str()); + TSError("[%s] Invalid line %d in '%s' - no destination service found.", PLUGIN_NAME, line_no, fp.c_str()); } else { this->load_pair(rxp, service, fp, line_no); } @@ -170,11 +170,11 @@ BridgeConfig::load_config(int argc, const char *argv[]) } } } else if (argv[i][0] == '-') { - TSError("%s: Unrecognized option '%s'", PLUGIN_TAG, argv[i]); + TSError("[%s] Unrecognized option '%s'", PLUGIN_NAME, argv[i]); i -= 1; // Don't skip next arg. } else { if (i + 1 >= argc) { - TSError("%s: Regular expression '%s' without destination service", PLUGIN_TAG, argv[i]); + TSError("[%s] Regular expression '%s' without destination service", PLUGIN_NAME, argv[i]); } else { this->load_pair(argv[i], argv[i + 1], plugin_config_fp); } @@ -301,7 +301,7 @@ void Bridge::net_accept(TSVConn vc) { char buff[1024]; - int64_t n = snprintf(buff, sizeof(buff), CONNECT_FORMAT, static_cast(_peer.size()), _peer.data()); + int64_t n = snprintf(buff, sizeof(buff), CONNECT_FORMAT, int(_peer.size()), _peer.data()); TSDebug(PLUGIN_TAG, "Received UA VConn, connecting to peer %.*s", int(_peer.size()), _peer.data()); // UA side intercepted. @@ -685,12 +685,12 @@ TSPluginInit(int argc, char const *argv[]) TSPluginRegistrationInfo info{PLUGIN_NAME, "Oath:", "solidwallofcode@oath.com"}; if (TSPluginRegister(&info) != TS_SUCCESS) { - TSError(PLUGIN_NAME ": plugin registration failed."); + TSError("[%s] plugin registration failed.", PLUGIN_NAME); } Config.load_config(argc - 1, argv + 1); if (Config.count() <= 0) { - TSError("%s: No destinations defined, plugin disabled", PLUGIN_TAG); + TSError("[%s] No destinations defined, plugin disabled", PLUGIN_NAME); } TSCont contp = TSContCreate(CB_Read_Request_Hdr, TSMutexCreate()); From 845a9a4c840943cabd53b2eb16843d953d614032 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Fri, 4 Jan 2019 18:20:37 +0000 Subject: [PATCH 146/526] Add unit tests for the URI signing plugin --- .gitignore | 2 + plugins/experimental/uri_signing/Makefile.inc | 14 +++ plugins/experimental/uri_signing/README.md | 3 + plugins/experimental/uri_signing/common.c | 32 +++++++ .../uri_signing/{uri_signing.h => common.h} | 12 +++ plugins/experimental/uri_signing/config.c | 2 +- plugins/experimental/uri_signing/cookie.c | 2 +- plugins/experimental/uri_signing/jwt.c | 2 +- plugins/experimental/uri_signing/match.c | 2 +- plugins/experimental/uri_signing/parse.c | 2 +- .../unit_tests/uri_signing_test.cc | 92 +++++++++++++++++++ .../experimental/uri_signing/uri_signing.c | 2 +- 12 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 plugins/experimental/uri_signing/common.c rename plugins/experimental/uri_signing/{uri_signing.h => common.h} (72%) create mode 100644 plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc diff --git a/.gitignore b/.gitignore index f721723dc9a..a63e882a585 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,8 @@ plugins/esi/processor_test plugins/esi/utils_test plugins/esi/vars_test +plugins/experimental/uri_signing/test_uri_signing + mgmt/api/traffic_api_cli_remote mgmt/tools/traffic_mcast_snoop mgmt/tools/traffic_net_config diff --git a/plugins/experimental/uri_signing/Makefile.inc b/plugins/experimental/uri_signing/Makefile.inc index 9499479b4a6..e9807baf678 100644 --- a/plugins/experimental/uri_signing/Makefile.inc +++ b/plugins/experimental/uri_signing/Makefile.inc @@ -26,3 +26,17 @@ experimental_uri_signing_uri_signing_la_SOURCES = \ experimental/uri_signing/timing.c experimental_uri_signing_uri_signing_la_LIBADD = @LIBJANSSON@ @LIBCJOSE@ @LIBPCRE@ -lm -lcrypto + +check_PROGRAMS += experimental/uri_signing/test_uri_signing + +experimental_uri_signing_test_uri_signing_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DURI_SIGNING_UNIT_TEST +experimental_uri_signing_test_uri_signing_LDADD = @LIBJANSSON@ @LIBCJOSE@ @LIBPCRE@ -lm -lcrypto +experimental_uri_signing_test_uri_signing_SOURCES = \ + experimental/uri_signing/unit_tests/uri_signing_test.cc \ + experimental/uri_signing/jwt.c \ + experimental/uri_signing/common.c \ + experimental/uri_signing/parse.c \ + experimental/uri_signing/cookie.c \ + experimental/uri_signing/config.c \ + experimental/uri_signing/timing.c \ + experimental/uri_signing/match.c diff --git a/plugins/experimental/uri_signing/README.md b/plugins/experimental/uri_signing/README.md index fe242d8d98f..f4fef0c85bd 100644 --- a/plugins/experimental/uri_signing/README.md +++ b/plugins/experimental/uri_signing/README.md @@ -147,6 +147,9 @@ This builds in-tree with the rest of the ATS plugins. Of special note, however, are the first two libraries: cjose and jansson. These libraries are not currently used anywhere else, so they may not be installed. +Note that the default prefix value for cjose is /usr/local. Ensure this is visible to +any executables that are being run using this library. + As of this writing, both libraries install a dynamic library and a static archive. However, by default, the static archive is not compiled with Position Independent Code. The build script will detect this and build a dynamic diff --git a/plugins/experimental/uri_signing/common.c b/plugins/experimental/uri_signing/common.c new file mode 100644 index 00000000000..bae8b6d911a --- /dev/null +++ b/plugins/experimental/uri_signing/common.c @@ -0,0 +1,32 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "common.h" + +#ifdef URI_SIGNING_UNIT_TEST + +void +PrintToStdErr(const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); +} + +#endif diff --git a/plugins/experimental/uri_signing/uri_signing.h b/plugins/experimental/uri_signing/common.h similarity index 72% rename from plugins/experimental/uri_signing/uri_signing.h rename to plugins/experimental/uri_signing/common.h index 6cb5046c2df..c9304085f58 100644 --- a/plugins/experimental/uri_signing/uri_signing.h +++ b/plugins/experimental/uri_signing/common.h @@ -18,5 +18,17 @@ #define PLUGIN_NAME "uri_signing" +#ifdef URI_SIGNING_UNIT_TEST +#include +#include + +#define PluginDebug(fmt, ...) PrintToStdErr("(%s) %s:%d:%s() " fmt "\n", PLUGIN_NAME, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define PluginError(fmt, ...) PrintToStdErr("(%s) %s:%d:%s() " fmt "\n", PLUGIN_NAME, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +void PrintToStdErr(const char *fmt, ...); + +#else + #define PluginDebug(...) TSDebug("uri_signing", PLUGIN_NAME " " __VA_ARGS__) #define PluginError(...) PluginDebug(__VA_ARGS__), TSError(PLUGIN_NAME " " __VA_ARGS__) + +#endif diff --git a/plugins/experimental/uri_signing/config.c b/plugins/experimental/uri_signing/config.c index 83083f8fbda..b52b94470b3 100644 --- a/plugins/experimental/uri_signing/config.c +++ b/plugins/experimental/uri_signing/config.c @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "config.h" #include "timing.h" #include "jwt.h" diff --git a/plugins/experimental/uri_signing/cookie.c b/plugins/experimental/uri_signing/cookie.c index 70dd1aa8469..45a2e43b961 100644 --- a/plugins/experimental/uri_signing/cookie.c +++ b/plugins/experimental/uri_signing/cookie.c @@ -17,7 +17,7 @@ */ #include "cookie.h" -#include "uri_signing.h" +#include "common.h" #include #include diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index 9324ca866ec..997448bfb2e 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "jwt.h" #include "match.h" #include "ts/ts.h" diff --git a/plugins/experimental/uri_signing/match.c b/plugins/experimental/uri_signing/match.c index ad376a251e4..b665dc47bce 100644 --- a/plugins/experimental/uri_signing/match.c +++ b/plugins/experimental/uri_signing/match.c @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "ts/ts.h" #include #include diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index 603c4ec9f53..99ded7b71ab 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "parse.h" #include "config.h" #include "jwt.h" diff --git a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc new file mode 100644 index 00000000000..d89bf9c1898 --- /dev/null +++ b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc @@ -0,0 +1,92 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * These are misc unit tests for uri signing + */ + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +extern "C" { +#include +#include +#include "../jwt.h" +} + +bool +jwt_parsing_helper(const char *jwt_string) +{ + fprintf(stderr, "Parsing JWT from string: %s\n", jwt_string); + bool resp; + json_error_t jerr = {}; + size_t pt_ct = strlen(jwt_string); + struct jwt *jwt = parse_jwt(json_loadb(jwt_string, pt_ct, 0, &jerr)); + + if (jwt) { + resp = jwt_validate(jwt); + } else { + resp = false; + } + + jwt_delete(jwt); + return resp; +} + +TEST_CASE("1", "[JWSParsingTest]") +{ + INFO("TEST 1, Test JWT Parsing From Token Strings"); + + SECTION("Standard JWT Parsing") + { + REQUIRE(jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"exp\":7284188499,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); + } + + SECTION("JWT Parsing With Unknown Claim") + { + REQUIRE(jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"exp\":7284188499,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/" + "*\",\"jamesBond\":\"Something,Something_else\"}")); + } + + SECTION("JWT Parsing with unsupported crit claim passed") + { + REQUIRE(!jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"exp\":7284188499,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/" + "*\",\"cdnicrit\":\"Something,Something_else\"}")); + } + + SECTION("JWT Parsing with empty exp claim") + { + REQUIRE(jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); + } + + SECTION("JWT Parsing with unsupported cdniip claim") + { + REQUIRE(!jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"cdniip\":\"123.123.123.123\",\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); + } + + SECTION("JWT Parsing with unsupported value for cdnistd claim") + { + REQUIRE(!jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"cdnistd\":4,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); + } +} diff --git a/plugins/experimental/uri_signing/uri_signing.c b/plugins/experimental/uri_signing/uri_signing.c index e9a2a81579e..044731b98a4 100644 --- a/plugins/experimental/uri_signing/uri_signing.c +++ b/plugins/experimental/uri_signing/uri_signing.c @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "config.h" #include "parse.h" #include "jwt.h" From f9d962b19742146c21c306c9cd804aa54a83301b Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 11 Jan 2019 19:37:57 -0700 Subject: [PATCH 147/526] Validates return values from TSHttpTxnClientAddrGet() --- plugins/header_rewrite/conditions.cc | 142 ++++++++++++++------------- plugins/header_rewrite/lulu.cc | 4 + 2 files changed, 80 insertions(+), 66 deletions(-) diff --git a/plugins/header_rewrite/conditions.cc b/plugins/header_rewrite/conditions.cc index 2920172e76f..16082da045f 100644 --- a/plugins/header_rewrite/conditions.cc +++ b/plugins/header_rewrite/conditions.cc @@ -840,57 +840,59 @@ ConditionGeo::get_geo_string(const sockaddr *addr) const const char *ret = nullptr; int v = 4; - switch (_geo_qual) { - // Country database - case GEO_QUAL_COUNTRY: - switch (addr->sa_family) { - case AF_INET: - if (gGeoIP[GEOIP_COUNTRY_EDITION]) { - uint32_t ip = ntohl(reinterpret_cast(addr)->sin_addr.s_addr); - - ret = GeoIP_country_code_by_ipnum(gGeoIP[GEOIP_COUNTRY_EDITION], ip); + if (addr) { + switch (_geo_qual) { + // Country database + case GEO_QUAL_COUNTRY: + switch (addr->sa_family) { + case AF_INET: + if (gGeoIP[GEOIP_COUNTRY_EDITION]) { + uint32_t ip = ntohl(reinterpret_cast(addr)->sin_addr.s_addr); + + ret = GeoIP_country_code_by_ipnum(gGeoIP[GEOIP_COUNTRY_EDITION], ip); + } + break; + case AF_INET6: { + if (gGeoIP[GEOIP_COUNTRY_EDITION_V6]) { + geoipv6_t ip = reinterpret_cast(addr)->sin6_addr; + + v = 6; + ret = GeoIP_country_code_by_ipnum_v6(gGeoIP[GEOIP_COUNTRY_EDITION_V6], ip); + } + } break; + default: + break; } + TSDebug(PLUGIN_NAME, "eval(): Client IPv%d seems to come from Country: %s", v, ret); break; - case AF_INET6: { - if (gGeoIP[GEOIP_COUNTRY_EDITION_V6]) { - geoipv6_t ip = reinterpret_cast(addr)->sin6_addr; - v = 6; - ret = GeoIP_country_code_by_ipnum_v6(gGeoIP[GEOIP_COUNTRY_EDITION_V6], ip); + // ASN database + case GEO_QUAL_ASN_NAME: + switch (addr->sa_family) { + case AF_INET: + if (gGeoIP[GEOIP_ASNUM_EDITION]) { + uint32_t ip = ntohl(reinterpret_cast(addr)->sin_addr.s_addr); + + ret = GeoIP_name_by_ipnum(gGeoIP[GEOIP_ASNUM_EDITION], ip); + } + break; + case AF_INET6: { + if (gGeoIP[GEOIP_ASNUM_EDITION_V6]) { + geoipv6_t ip = reinterpret_cast(addr)->sin6_addr; + + v = 6; + ret = GeoIP_name_by_ipnum_v6(gGeoIP[GEOIP_ASNUM_EDITION_V6], ip); + } + } break; + default: + break; } - } break; - default: + TSDebug(PLUGIN_NAME, "eval(): Client IPv%d seems to come from ASN Name: %s", v, ret); break; - } - TSDebug(PLUGIN_NAME, "eval(): Client IPv%d seems to come from Country: %s", v, ret); - break; - - // ASN database - case GEO_QUAL_ASN_NAME: - switch (addr->sa_family) { - case AF_INET: - if (gGeoIP[GEOIP_ASNUM_EDITION]) { - uint32_t ip = ntohl(reinterpret_cast(addr)->sin_addr.s_addr); - - ret = GeoIP_name_by_ipnum(gGeoIP[GEOIP_ASNUM_EDITION], ip); - } - break; - case AF_INET6: { - if (gGeoIP[GEOIP_ASNUM_EDITION_V6]) { - geoipv6_t ip = reinterpret_cast(addr)->sin6_addr; - v = 6; - ret = GeoIP_name_by_ipnum_v6(gGeoIP[GEOIP_ASNUM_EDITION_V6], ip); - } - } break; default: break; } - TSDebug(PLUGIN_NAME, "eval(): Client IPv%d seems to come from ASN Name: %s", v, ret); - break; - - default: - break; } return ret ? ret : "(unknown)"; @@ -902,6 +904,10 @@ ConditionGeo::get_geo_int(const sockaddr *addr) const int64_t ret = -1; int v = 4; + if (!addr) { + return 0; + } + switch (_geo_qual) { // Country Databse case GEO_QUAL_COUNTRY_ISO: @@ -1212,32 +1218,36 @@ ConditionCidr::append_value(std::string &s, const Resources &res) { struct sockaddr const *addr = TSHttpTxnClientAddrGet(res.txnp); - switch (addr->sa_family) { - case AF_INET: { - char res[INET_ADDRSTRLEN]; - struct in_addr ipv4 = reinterpret_cast(addr)->sin_addr; - - ipv4.s_addr &= _v4_mask.s_addr; - inet_ntop(AF_INET, &ipv4, res, INET_ADDRSTRLEN); - if (res[0]) { - s += res; - } - } break; - case AF_INET6: { - char res[INET6_ADDRSTRLEN]; - struct in6_addr ipv6 = reinterpret_cast(addr)->sin6_addr; + if (addr) { + switch (addr->sa_family) { + case AF_INET: { + char res[INET_ADDRSTRLEN]; + struct in_addr ipv4 = reinterpret_cast(addr)->sin_addr; + + ipv4.s_addr &= _v4_mask.s_addr; + inet_ntop(AF_INET, &ipv4, res, INET_ADDRSTRLEN); + if (res[0]) { + s += res; + } + } break; + case AF_INET6: { + char res[INET6_ADDRSTRLEN]; + struct in6_addr ipv6 = reinterpret_cast(addr)->sin6_addr; - if (_v6_zero_bytes > 0) { - memset(&ipv6.s6_addr[16 - _v6_zero_bytes], 0, _v6_zero_bytes); - } - if (_v6_mask != 0xff) { - ipv6.s6_addr[16 - _v6_zero_bytes] &= _v6_mask; - } - inet_ntop(AF_INET6, &ipv6, res, INET6_ADDRSTRLEN); - if (res[0]) { - s += res; + if (_v6_zero_bytes > 0) { + memset(&ipv6.s6_addr[16 - _v6_zero_bytes], 0, _v6_zero_bytes); + } + if (_v6_mask != 0xff) { + ipv6.s6_addr[16 - _v6_zero_bytes] &= _v6_mask; + } + inet_ntop(AF_INET6, &ipv6, res, INET6_ADDRSTRLEN); + if (res[0]) { + s += res; + } + } break; } - } break; + } else { + s += "0.0.0.0"; // No client addr for some reason ... } } diff --git a/plugins/header_rewrite/lulu.cc b/plugins/header_rewrite/lulu.cc index 78b70f0c7eb..b4eb9f6341c 100644 --- a/plugins/header_rewrite/lulu.cc +++ b/plugins/header_rewrite/lulu.cc @@ -71,6 +71,10 @@ getIP(sockaddr const *s_sockaddr) uint16_t getPort(sockaddr const *s_sockaddr) { + if (!s_sockaddr) { + return 0; + } + switch (s_sockaddr->sa_family) { case AF_INET: { const struct sockaddr_in *s_sockaddr_in = reinterpret_cast(s_sockaddr); From 58b154bb6f253ac8660bd778dd2bf43cb7b60687 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 14 Jan 2019 13:41:50 -0700 Subject: [PATCH 148/526] Eliminates frequent librecords lookup This also avoids up to two malloc(1) calls, even when the congestion configurations aren't used. I'm making this as simple as possible, for backporting to 7.x and 8.x, but it also means these configs are no longer reloadable (they were never marked as reloadable in the Docs though). For v9.0.0, I think we should change these configurations to be in the ports specifications, rather than the globals for all of ATS. E.g. 80:ip-ccp-in=bbr:ip-ccp-out=reno --- iocore/net/I_Net.h | 3 +++ iocore/net/Net.cc | 18 ++++++++++++++++ iocore/net/P_UnixNetVConnection.h | 35 ------------------------------- iocore/net/UnixNetVConnection.cc | 30 ++++++++++++++++++++++++++ mgmt/RecordsConfig.cc | 4 ++-- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/iocore/net/I_Net.h b/iocore/net/I_Net.h index e4e82a64592..81ed6c3edbe 100644 --- a/iocore/net/I_Net.h +++ b/iocore/net/I_Net.h @@ -61,6 +61,9 @@ extern int net_accept_period; extern int net_retry_delay; extern int net_throttle_delay; +extern std::string_view net_ccp_in; +extern std::string_view net_ccp_out; + #define NET_EVENT_OPEN (NET_EVENT_EVENTS_START) #define NET_EVENT_OPEN_FAILED (NET_EVENT_EVENTS_START + 1) #define NET_EVENT_ACCEPT (NET_EVENT_EVENTS_START + 2) diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc index af1d1f75bcd..a2c5f5f03a7 100644 --- a/iocore/net/Net.cc +++ b/iocore/net/Net.cc @@ -40,6 +40,10 @@ int net_accept_period = 10; int net_retry_delay = 10; int net_throttle_delay = 50; /* milliseconds */ +// For the in/out congestion control: ToDo: this probably would be better as ports: specifications +std::string_view net_ccp_in; +std::string_view net_ccp_out; + static inline void configure_net() { @@ -52,6 +56,20 @@ configure_net() // These are not reloadable REC_ReadConfigInteger(net_event_period, "proxy.config.net.event_period"); REC_ReadConfigInteger(net_accept_period, "proxy.config.net.accept_period"); + + // This is kinda fugly, but better than it was before (on every connection in and out) + // Note that these would need to be ats_free()'d if we ever want to clean that up, but + // we have no good way of dealing with that on such globals I think? + RecString ccp; + + REC_ReadConfigStringAlloc(ccp, "proxy.config.net.tcp_congestion_control_in"); + if (ccp && *ccp != '\0') { + net_ccp_in = {ccp}; + } + REC_ReadConfigStringAlloc(ccp, "proxy.config.net.tcp_congestion_control_out"); + if (ccp && *ccp != '\0') { + net_ccp_out = {ccp}; + } } static inline void diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 94c141801b9..ba5709171ac 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -398,41 +398,6 @@ UnixNetVConnection::set_tcp_init_cwnd(int init_cwnd) #endif } -inline int -UnixNetVConnection::set_tcp_congestion_control(int side) -{ -#ifdef TCP_CONGESTION - RecString congestion_control; - int ret; - - if (side == CLIENT_SIDE) { - ret = REC_ReadConfigStringAlloc(congestion_control, "proxy.config.net.tcp_congestion_control_in"); - } else { - ret = REC_ReadConfigStringAlloc(congestion_control, "proxy.config.net.tcp_congestion_control_out"); - } - - if (ret == REC_ERR_OKAY) { - int len = strlen(congestion_control); - if (len > 0) { - int rv = 0; - rv = setsockopt(con.fd, IPPROTO_TCP, TCP_CONGESTION, reinterpret_cast(congestion_control), len); - if (rv < 0) { - Error("Unable to set TCP congestion control on socket %d to \"%.*s\", errno=%d (%s)", con.fd, len, congestion_control, - errno, strerror(errno)); - } else { - Debug("socket", "Setting TCP congestion control on socket [%d] to \"%.*s\" -> %d", con.fd, len, congestion_control, rv); - } - } - ats_free(congestion_control); - return 0; - } - return -1; -#else - Debug("socket", "Setting TCP congestion control is not supported on this platform."); - return -1; -#endif -} - inline UnixNetVConnection::~UnixNetVConnection() {} inline SOCKET diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 635c98f5e99..f005f3d9948 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -1524,3 +1524,33 @@ UnixNetVConnection::protocol_contains(std::string_view tag) const } return retval.data(); } + +int +UnixNetVConnection::set_tcp_congestion_control(int side) +{ +#ifdef TCP_CONGESTION + std::string_view ccp; + + if (side == CLIENT_SIDE) { + ccp = net_ccp_in; + } else { + ccp = net_ccp_out; + } + + if (!ccp.empty()) { + int rv = setsockopt(con.fd, IPPROTO_TCP, TCP_CONGESTION, reinterpret_cast(ccp.data()), ccp.length()); + + if (rv < 0) { + Error("Unable to set TCP congestion control on socket %d to \"%s\", errno=%d (%s)", con.fd, ccp.data(), errno, + strerror(errno)); + } else { + Debug("socket", "Setting TCP congestion control on socket [%d] to \"%s\" -> %d", con.fd, ccp.data(), rv); + } + return 0; + } + return -1; +#else + Debug("socket", "Setting TCP congestion control is not supported on this platform."); + return -1; +#endif +} diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index d4a212262ad..896f171f3d3 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -793,9 +793,9 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.net.sock_option_tfo_queue_size_in", RECD_INT, "10000", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.net.tcp_congestion_control_in", RECD_STRING, "", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.net.tcp_congestion_control_in", RECD_STRING, "", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.net.tcp_congestion_control_out", RECD_STRING, "", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.net.tcp_congestion_control_out", RECD_STRING, "", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , //############################################################################## From acc47efd083bd48ceecdb0beaf75c98ca07e2973 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Sun, 25 Nov 2018 16:39:04 -0600 Subject: [PATCH 149/526] Issue #4637 - Clean up / unify hex conversions Clean up port hex conversion in SSLAddressLookupKey. --- .gitignore | 1 + include/tscore/BufferWriter.h | 45 +++++++++++++++++++ include/tscore/bwf_std_format.h | 3 +- include/tscore/ink_inet.h | 10 +++++ iocore/net/SSLCertLookup.cc | 33 ++++++-------- src/tscore/BufferWriterFormat.cc | 41 ++++++++++------- src/tscore/ink_inet.cc | 9 ++++ .../unit_tests/test_BufferWriterFormat.cc | 20 +++++++++ src/tscore/unit_tests/test_ink_inet.cc | 8 ++++ 9 files changed, 135 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index a63e882a585..e56e2fef90a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ Makefile.in .project .cproject +.idea .diags.log.meta diff --git a/include/tscore/BufferWriter.h b/include/tscore/BufferWriter.h index 6da3e12c21b..f5fce06db73 100644 --- a/include/tscore/BufferWriter.h +++ b/include/tscore/BufferWriter.h @@ -906,6 +906,51 @@ FixedBufferWriter::printv(BWFormat const &fmt, std::tuple const &args) return static_cast(this->super_type::printv(fmt, args)); } +// Basic format wrappers - these are here because they're used internally. +namespace bwf +{ + namespace detail + { + /** Write out raw memory in hexadecimal wrapper. + * + * This wrapper indicates the contained view should be dumped as raw memory in hexadecimal format. + * This is intended primarily for internal use by other formatting logic. + * + * @see Hex_Dump + */ + struct MemDump { + std::string_view _view; + + /** Dump @a n bytes starting at @a mem as hex. + * + * @param mem First byte of memory to dump. + * @param n Number of bytes. + */ + MemDump(void const *mem, size_t n) : _view(static_cast(mem), n) {} + }; + } // namespace detail + + /** Treat @a t as raw memory and dump the memory as hexadecimal. + * + * @tparam T Type of argument. + * @param t Object to dump. + * @return @a A wrapper to do a hex dump. + * + * This is the standard way to do a hexadecimal memory dump of an object. + * + * @internal This function exists so that other types can overload it for special processing, + * which would not be possible with just @c HexDump. + */ + template + detail::MemDump + Hex_Dump(T const &t) + { + return {&t, sizeof(T)}; + } +} // namespace bwf + +BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, bwf::detail::MemDump const &hex); + } // end namespace ts namespace std diff --git a/include/tscore/bwf_std_format.h b/include/tscore/bwf_std_format.h index ce7f4f7080a..1e4a26741e6 100644 --- a/include/tscore/bwf_std_format.h +++ b/include/tscore/bwf_std_format.h @@ -121,7 +121,8 @@ namespace bwf } } }; -} // namespace bwf + +}; // namespace bwf BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, bwf::Errno const &e); BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, bwf::Date const &date); diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index 4cbf14c1d41..b72aefa37b6 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -1558,4 +1558,14 @@ bwformat(BufferWriter &w, BWFSpec const &spec, IpEndpoint const &addr) { return bwformat(w, spec, &addr.sa); } + +namespace bwf +{ + namespace detail + { + struct MemDump; + } // namespace detail + + detail::MemDump Hex_Dump(IpEndpoint const &addr); +} // namespace bwf } // namespace ts diff --git a/iocore/net/SSLCertLookup.cc b/iocore/net/SSLCertLookup.cc index 2c06ca61eb3..508ea2ffe6c 100644 --- a/iocore/net/SSLCertLookup.cc +++ b/iocore/net/SSLCertLookup.cc @@ -31,6 +31,8 @@ #include "tscore/MatcherUtils.h" #include "tscore/Regex.h" #include "tscore/Trie.h" +#include "tscore/BufferWriter.h" +#include "tscore/bwf_std_format.h" #include "tscore/TestBox.h" #include @@ -45,26 +47,19 @@ #endif /* SSL_CTX_set_tlsext_ticket_key_cb */ struct SSLAddressLookupKey { - explicit SSLAddressLookupKey(const IpEndpoint &ip) : sep(0) + explicit SSLAddressLookupKey(const IpEndpoint &ip) { - static const char hextab[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - - int nbytes; - uint16_t port = ntohs(ip.port()); - - // For IP addresses, the cache key is the hex address with the port concatenated. This makes the lookup - // insensitive to address formatting and also allow the longest match semantic to produce different matches - // if there is a certificate on the port. - nbytes = ats_ip_to_hex(&ip.sa, key, sizeof(key)); - if (port) { - sep = nbytes; - key[nbytes++] = '.'; - key[nbytes++] = hextab[(port >> 12) & 0x000F]; - key[nbytes++] = hextab[(port >> 8) & 0x000F]; - key[nbytes++] = hextab[(port >> 4) & 0x000F]; - key[nbytes++] = hextab[(port)&0x000F]; + // For IP addresses, the cache key is the hex address with the port concatenated. This makes the + // lookup insensitive to address formatting and also allow the longest match semantic to produce + // different matches if there is a certificate on the port. + + ts::FixedBufferWriter w{key, sizeof(key)}; + w.print("{}", ts::bwf::Hex_Dump(ip)); // dump raw bytes in hex, don't format as IP address. + if (in_port_t port = ip.host_order_port(); port) { + sep = unsigned(w.size()); + w.print(".{:x}", port); } - key[nbytes++] = 0; + w.print("\0"); } const char * @@ -85,7 +80,7 @@ struct SSLAddressLookupKey { private: char key[(TS_IP6_SIZE * 2) /* hex addr */ + 1 /* dot */ + 4 /* port */ + 1 /* nullptr */]; - unsigned char sep; // offset of address/port separator + unsigned char sep = 0; // offset of address/port separator }; struct SSLContextStorage { diff --git a/src/tscore/BufferWriterFormat.cc b/src/tscore/BufferWriterFormat.cc index 7efd5e04eae..70a5e016281 100644 --- a/src/tscore/BufferWriterFormat.cc +++ b/src/tscore/BufferWriterFormat.cc @@ -32,6 +32,7 @@ #include #include #include +#include using namespace std::literals; @@ -587,7 +588,7 @@ namespace bw_fmt /// Write out the @a data as hexadecimal, using @a digits as the conversion. void - Hex_Dump(BufferWriter &w, std::string_view data, const char *digits) + Format_As_Hex(BufferWriter &w, std::string_view data, const char *digits) { const char *ptr = data.data(); for (auto n = data.size(); n > 0; --n) { @@ -608,14 +609,7 @@ bwformat(BufferWriter &w, BWFSpec const &spec, std::string_view sv) } if ('x' == spec._type || 'X' == spec._type) { - const char *digits = 'x' == spec._type ? bw_fmt::LOWER_DIGITS : bw_fmt::UPPER_DIGITS; - width -= sv.size() * 2; - if (spec._radix_lead_p) { - w.write('0'); - w.write(spec._type); - width -= 2; - } - bw_fmt::Write_Aligned(w, [&w, &sv, digits]() { bw_fmt::Hex_Dump(w, sv, digits); }, spec._align, width, spec._fill, 0); + return bwformat(w, spec, bwf::detail::MemDump(sv.data(), sv.size())); } else { width -= sv.size(); bw_fmt::Write_Aligned(w, [&w, &sv]() { w.write(sv); }, spec._align, width, spec._fill, 0); @@ -628,12 +622,7 @@ bwformat(BufferWriter &w, BWFSpec const &spec, MemSpan const &span) { static const BWFormat default_fmt{"{:#x}@{:p}"}; if (spec._ext.size() && 'd' == spec._ext.front()) { - const char *digits = 'X' == spec._type ? bw_fmt::UPPER_DIGITS : bw_fmt::LOWER_DIGITS; - if (spec._radix_lead_p) { - w.write('0'); - w.write(digits[33]); - } - bw_fmt::Hex_Dump(w, span.view(), digits); + bwformat(w, spec, bwf::detail::MemDump(span.data(), span.size())); } else { w.print(default_fmt, span.size(), span.data()); } @@ -958,6 +947,28 @@ bwformat(BufferWriter &w, BWFSpec const &spec, bwf::OptionalAffix const &opts) return w.write(opts._prefix).write(opts._text).write(opts._suffix); } +BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, bwf::detail::MemDump const &hex) +{ + char fmt_type = spec._type; + const char *digits = bw_fmt::UPPER_DIGITS; + + if ('X' != fmt_type) { + fmt_type = 'x'; + digits = bw_fmt::LOWER_DIGITS; + } + + int width = int(spec._min) - hex._view.size() * 2; // amount left to fill. + if (spec._radix_lead_p) { + w.write('0'); + w.write(fmt_type); + width -= 2; + } + bw_fmt::Write_Aligned(w, [&w, &hex, digits]() { bw_fmt::Format_As_Hex(w, hex._view, digits); }, spec._align, width, spec._fill, + 0); + return w; +} + } // namespace ts namespace diff --git a/src/tscore/ink_inet.cc b/src/tscore/ink_inet.cc index 12524878b77..c52c316c180 100644 --- a/src/tscore/ink_inet.cc +++ b/src/tscore/ink_inet.cc @@ -937,4 +937,13 @@ bwformat(BufferWriter &w, BWFSpec const &spec, sockaddr const *addr) return w; } +namespace bwf +{ + detail::MemDump + Hex_Dump(IpEndpoint const &addr) + { + return detail::MemDump(ats_ip_addr8_cast(&addr), ats_ip_addr_size(&addr)); + } +} // namespace bwf + } // namespace ts diff --git a/src/tscore/unit_tests/test_BufferWriterFormat.cc b/src/tscore/unit_tests/test_BufferWriterFormat.cc index 1e3a70b8cca..6e0062ee214 100644 --- a/src/tscore/unit_tests/test_BufferWriterFormat.cc +++ b/src/tscore/unit_tests/test_BufferWriterFormat.cc @@ -25,6 +25,7 @@ #include "../../../tests/include/catch.hpp" #include #include +#include #include "tscore/BufferWriter.h" #include "tscore/bwf_std_format.h" #include "tscpp/util/MemSpan.h" @@ -574,6 +575,25 @@ TEST_CASE("bwstring std formats", "[libts][bwprint]") REQUIRE(w.view() == "name = Evil Dave"); w.reset().print("name = {}", ts::bwf::FirstOf(empty, empty, s3, empty, s2, s1)); REQUIRE(w.view() == "name = Leif"); + + unsigned v = ntohl(0xdeadbeef); + w.reset().print("{}", ts::bwf::Hex_Dump(v)); + REQUIRE(w.view() == "deadbeef"); + w.reset().print("{:x}", ts::bwf::Hex_Dump(v)); + REQUIRE(w.view() == "deadbeef"); + w.reset().print("{:X}", ts::bwf::Hex_Dump(v)); + REQUIRE(w.view() == "DEADBEEF"); + w.reset().print("{:#X}", ts::bwf::Hex_Dump(v)); + REQUIRE(w.view() == "0XDEADBEEF"); + w.reset().print("{} bytes {} digits {}", sizeof(double), std::numeric_limits::digits10, ts::bwf::Hex_Dump(2.718281828)); + REQUIRE(w.view() == "8 bytes 15 digits 9b91048b0abf0540"); + + INK_MD5 md5; + w.reset().print("{}", ts::bwf::Hex_Dump(md5)); + REQUIRE(w.view() == "00000000000000000000000000000000"); + CryptoContext().hash_immediate(md5, s2.data(), s2.size()); + w.reset().print("{}", ts::bwf::Hex_Dump(md5)); + REQUIRE(w.view() == "f240ccd7a95c7ec66d6c111e2925b23e"); } // Normally there's no point in running the performance tests, but it's worth keeping the code diff --git a/src/tscore/unit_tests/test_ink_inet.cc b/src/tscore/unit_tests/test_ink_inet.cc index bd5749818b8..4fd47475610 100644 --- a/src/tscore/unit_tests/test_ink_inet.cc +++ b/src/tscore/unit_tests/test_ink_inet.cc @@ -219,6 +219,10 @@ TEST_CASE("inet formatting", "[libts][ink_inet][bwformat]") REQUIRE(w.view() == "172. 17. 99.231"); w.reset().print("{::=a}", ep); REQUIRE(w.view() == "172.017.099.231"); + w.reset().print("{}", ts::bwf::Hex_Dump(ep)); + REQUIRE(w.view() == "ac1163e7"); + w.reset().print("{:#X}", ts::bwf::Hex_Dump(ep)); + REQUIRE(w.view() == "0XAC1163E7"); // Documentation examples REQUIRE(0 == ats_ip_pton(addr_7, &ep.sa)); @@ -247,4 +251,8 @@ TEST_CASE("inet formatting", "[libts][ink_inet][bwformat]") w.reset().print("{:p}", reinterpret_cast(0x1337beef)); REQUIRE(w.view() == "0x1337beef"); + + ats_ip_pton(addr_1, &ep.sa); + w.reset().print("{}", ts::bwf::Hex_Dump(ep)); + REQUIRE(w.view() == "ffee00000000000024c333493cee0143"); } From fe70d57b3dffa6d00b8e1ef8984e031cc4e7b3ec Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 15 Jan 2019 13:17:44 -0600 Subject: [PATCH 150/526] i4637: A tiny bit of cleanup. --- iocore/net/SSLCertLookup.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iocore/net/SSLCertLookup.cc b/iocore/net/SSLCertLookup.cc index 508ea2ffe6c..f94f5150986 100644 --- a/iocore/net/SSLCertLookup.cc +++ b/iocore/net/SSLCertLookup.cc @@ -54,12 +54,12 @@ struct SSLAddressLookupKey { // different matches if there is a certificate on the port. ts::FixedBufferWriter w{key, sizeof(key)}; - w.print("{}", ts::bwf::Hex_Dump(ip)); // dump raw bytes in hex, don't format as IP address. + w.print("{}", ts::bwf::Hex_Dump(ip)); // dump as raw hex bytes, don't format as IP address. if (in_port_t port = ip.host_order_port(); port) { - sep = unsigned(w.size()); + sep = static_cast(w.size()); w.print(".{:x}", port); } - w.print("\0"); + w.write('\0'); // force C-string termination. } const char * From b917405d814ba0ded54b83c6a619ce2c0ce75b3a Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 14 Jan 2019 16:55:37 -0600 Subject: [PATCH 151/526] Regex: Clean up constructors. --- include/tscore/Regex.h | 23 +++++++++++++++-------- src/tscore/Regex.cc | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/include/tscore/Regex.h b/include/tscore/Regex.h index 6c1572c837f..8888ae60094 100644 --- a/include/tscore/Regex.h +++ b/include/tscore/Regex.h @@ -31,16 +31,23 @@ #include #endif +/// Match flags for regular expression evaluation. enum REFlags { - RE_CASE_INSENSITIVE = 0x0001, // default is case sensitive - RE_UNANCHORED = 0x0002, // default (for DFA) is to anchor at the first matching position - RE_ANCHORED = 0x0004, // default (for Regex) is unanchored + RE_CASE_INSENSITIVE = 0x0001, ///< Ignore case (default: case sensitive). + RE_UNANCHORED = 0x0002, ///< Unanchored (DFA defaults to anchored). + RE_ANCHORED = 0x0004, ///< Anchored (Regex defaults to unanchored). }; +/** Wrapper for PCRE evaluation. + * + */ class Regex { public: - Regex() : regex(nullptr), regex_extra(nullptr) {} + /// Default number of capture groups. + static constexpr size_t DEFAULT_GROUP_COUNT = 30; + + Regex() = default; bool compile(const char *pattern, const unsigned flags = 0); // It is safe to call exec() concurrently on the same object instance bool exec(const char *str); @@ -50,8 +57,8 @@ class Regex ~Regex(); private: - pcre *regex; - pcre_extra *regex_extra; + pcre *regex = nullptr; + pcre_extra *regex_extra = nullptr; }; typedef struct __pat { @@ -64,7 +71,7 @@ typedef struct __pat { class DFA { public: - DFA() : _my_patterns(nullptr) {} + DFA() = default; ~DFA(); int compile(const char *pattern, unsigned flags = 0); @@ -76,5 +83,5 @@ class DFA private: dfa_pattern *build(const char *pattern, unsigned flags = 0); - dfa_pattern *_my_patterns; + dfa_pattern *_my_patterns = nullptr; }; diff --git a/src/tscore/Regex.cc b/src/tscore/Regex.cc index 1a6cbedff00..79028927e8b 100644 --- a/src/tscore/Regex.cc +++ b/src/tscore/Regex.cc @@ -109,7 +109,7 @@ Regex::exec(const char *str) bool Regex::exec(const char *str, int length) { - int ovector[30]; + int ovector[DEFAULT_GROUP_COUNT]; return exec(str, length, ovector, countof(ovector)); } From b8ddcb3a3ac54212cc69a0ee454b26d6b277bbd1 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 10 Jan 2019 11:04:36 -0600 Subject: [PATCH 152/526] Doc: Repair build errors, primarily with some new events and configurations. --- doc/admin-guide/files/records.config.en.rst | 11 +++- .../functions/TSHttpOverridableConfig.en.rst | 4 +- .../api/functions/TSLifecycleHookAdd.en.rst | 25 +++++----- doc/developer-guide/api/types/TSEvent.en.rst | 50 +++++++++++++++++-- .../api/types/TSOverridableConfigKey.en.rst | 10 +++- 5 files changed, 80 insertions(+), 20 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index c4c02a805ea..b7d85c932b0 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -1097,6 +1097,15 @@ mptcp .. ts:cv:: CONFIG proxy.config.http.default_buffer_water_mark INT 32768 +.. ts:cv:: CONFIG proxy.config.http.request_buffer_enabled INT 0 + :overridable: + + This is a configuration value that is overridable but not configurable. This is most likely an + implementation error. + + This enables buffering the content for incoming ``POST`` requests. If enabled no outbound + connection is made until the entire ``POST`` request has been buffered. + .. ts:cv:: CONFIG proxy.config.http.request_header_max_size INT 131072 Controls the maximum size, in bytes, of an HTTP header in requests. Headers @@ -3580,7 +3589,7 @@ Client-Related Configuration .. ts:cv:: CONFIG proxy.config.ssl.client.sni_policy STRING NULL :overridable: - Indicate how the SNI value for the TLS connection to the origin is selected. By default it is + Indicate how the SNI value for the TLS connection to the origin is selected. By default it is `host` which means the host header field value is used for the SNI. If `remap` is specified, the remapped origin name is used for the SNI value. diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index a38fd566786..a30b5c29676 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -173,10 +173,10 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_NET_SOCK_SEND_BUFFER_SIZE_OUT` :ts:cv:`proxy.config.net.sock_send_buffer_size_out` :c:macro:`TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB` :ts:cv:`proxy.config.http.parent_proxy.mark_down_hostdb` :c:macro:`TS_CONFIG_SRV_ENABLED` :ts:cv:`proxy.config.srv_enabled` -:c:macro:`TS_CONFIG_SSL_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` +:c:macro:`TS_CONFIG_SSL_CLIENT_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` :c:macro:`TS_CONFIG_SSL_CERT_FILEPATH` :ts:cv:`proxy.config.ssl.client.cert.path` :c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER` :ts:cv:`proxy.config.ssl.client.verify.server` -:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES` :ts:cv:`proxy.config.ssl.client.verify.server,properties` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES` :ts:cv:`proxy.config.ssl.client.verify.server.properties` :c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY` :ts:cv:`proxy.config.ssl.client.verify.server.policy` :c:macro:`TS_CONFIG_SSL_CLIENT_SNI_POLICY` :ts:cv:`proxy.config.ssl.client.sni_policy` :c:macro:`TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS` :ts:cv:`proxy.config.ssl.hsts_include_subdomains` diff --git a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst index 82b22d46cb6..ced169bbaab 100644 --- a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst @@ -1,19 +1,18 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. .. include:: ../../../common.defs diff --git a/doc/developer-guide/api/types/TSEvent.en.rst b/doc/developer-guide/api/types/TSEvent.en.rst index 3104c1392d0..8261cfcef1d 100644 --- a/doc/developer-guide/api/types/TSEvent.en.rst +++ b/doc/developer-guide/api/types/TSEvent.en.rst @@ -57,6 +57,22 @@ Enumeration Members .. c:macro:: TS_EVENT_VCONN_ACTIVE_TIMEOUT +.. c:macro:: TS_EVENT_VCONN_START + + An inbound connection has started. + +.. c:macro:: TS_EVENT_VCONN_CLOSE + + An inbound connection has closed. + +.. c:macro:: TS_EVENT_OUTBOUND_START + + An outbound connection has started. + +.. c:macro:: TS_EVENT_OUTBOUND_CLOSE + + An outbound connection has closed. + .. c:macro:: TS_EVENT_NET_CONNECT .. c:macro:: TS_EVENT_NET_CONNECT_FAILED @@ -167,21 +183,31 @@ Enumeration Members .. c:macro:: TS_EVENT_LIFECYCLE_PORTS_INITIALIZED + The internal data structures for the proxy ports have been initialized. + .. c:macro:: TS_EVENT_LIFECYCLE_PORTS_READY + The proxy ports are now open for inbound connections. + .. c:macro:: TS_EVENT_LIFECYCLE_CACHE_READY + The cache is ready. + .. c:macro:: TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED .. c:macro:: TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED .. c:macro:: TS_EVENT_LIFECYCLE_MSG -.. c:macro:: TS_EVENT_VCONN_START + A message from an external source has arrived. -.. c:macro:: TS_EVENT_VCONN_CLOSE +.. c:macro:: TS_EVENT_LIFECYCLE_TASK_THREADS_READY -.. c:macro:: TS_EVENT_MGMT_UPDATE + The ``ET_TASK`` threads are running. + +.. c:macro:: TS_EVENT_LIFECYCLE_SHUTDOWN + + The |TS| process has is shutting down. .. c:macro:: TS_EVENT_INTERNAL_60200 @@ -189,6 +215,24 @@ Enumeration Members .. c:macro:: TS_EVENT_INTERNAL_60202 +.. c:macro:: TS_EVENT_SSL_CERT + + Preparing to present a server certificate to an inbound TLS connection. + +.. c:macro:: TS_EVENT_SSL_SERVERNAME + + The SNI name for an Inbound TLS connection has become available. + +.. c:macro:: TS_EVENT_SSL_VERIFY_SERVER + + Outbound TLS connection certificate verification (verifying the server certificate). + +.. c:macro:: TS_EVENT_SSL_VERIFY_CLIENT + + Inbound TLS connection certificate verification (verifying the client certificate). + +.. c:macro:: TS_EVENT_MGMT_UPDATE + Description =========== diff --git a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst index 55e2b4ea285..938254e99f5 100644 --- a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst +++ b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst @@ -119,6 +119,7 @@ Enumeration Members .. c:macro:: TS_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS .. c:macro:: TS_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES .. c:macro:: TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY + .. c:macro:: TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED .. c:macro:: TS_CONFIG_HTTP_ATTACH_SERVER_SESSION_TO_CLIENT .. c:macro:: TS_CONFIG_HTTP_ORIGIN_MAX_CONNECTIONS_QUEUE .. c:macro:: TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT @@ -128,7 +129,7 @@ Enumeration Members .. c:macro:: TS_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_IN .. c:macro:: TS_CONFIG_SRV_ENABLED .. c:macro:: TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD - .. c:macro:: TS_CONFIG_SSL_CERT_FILENAME + .. c:macro:: TS_CONFIG_SSL_CLIENT_CERT_FILENAME .. c:macro:: TS_CONFIG_SSL_CERT_FILEPATH .. c:macro:: TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB .. c:macro:: TS_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER @@ -149,6 +150,13 @@ Enumeration Members .. c:macro:: TS_CONFIG_HTTP_ALLOW_HALF_OPEN .. c:macro:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX .. c:macro:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH + .. c:macro:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER + .. c:macro:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY + .. c:macro:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES + .. c:macro:: TS_CONFIG_SSL_CLIENT_SNI_POLICY + .. c:macro:: TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME + .. c:macro:: TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME + Description =========== From 97b35113fc77f155737cf18206a0dea87c9f90fe Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Thu, 25 Oct 2018 16:13:03 -0500 Subject: [PATCH 153/526] Add Session and SSL hooks to test_hooks Au test. --- .../api/functions/TSHttpHookAdd.en.rst | 8 +++ .../gold_tests/pluginTest/test_hooks/log.gold | 19 ++++++ tests/gold_tests/pluginTest/test_hooks/one.in | 4 -- .../pluginTest/test_hooks/spurious.in | 4 ++ .../pluginTest/test_hooks/ssl/server.key | 15 +++++ .../pluginTest/test_hooks/ssl/server.pem | 32 +++++++++ .../pluginTest/test_hooks/test_hooks.test.py | 65 +++++++++++++------ tests/tools/plugins/test_hooks.cc | 53 +++++++++++++++ 8 files changed, 175 insertions(+), 25 deletions(-) delete mode 100644 tests/gold_tests/pluginTest/test_hooks/one.in create mode 100644 tests/gold_tests/pluginTest/test_hooks/spurious.in create mode 100644 tests/gold_tests/pluginTest/test_hooks/ssl/server.key create mode 100644 tests/gold_tests/pluginTest/test_hooks/ssl/server.pem diff --git a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst index 283e7e28bae..d21bb4b7ce8 100644 --- a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst @@ -113,6 +113,11 @@ transaction hooks:: TSHttpTxnHookAdd(txnp, TS_HTTP_READ_REQUEST_HDR_HOOK, contp); TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); return 0; + case TS_EVENT_HTTP_READ_REQUEST_HDR: + txnp = (TSHttpTxn) edata; + // ... + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; default: break; } @@ -128,6 +133,9 @@ transaction hooks:: TSHttpHookAdd(TS_HTTP_SSN_START_HOOK, contp); } +For more example code using hooks, see the test_hooks plugin in tests/tools/plugins (used by the test_hooks.test.py +Gold test). + See Also ======== diff --git a/tests/gold_tests/pluginTest/test_hooks/log.gold b/tests/gold_tests/pluginTest/test_hooks/log.gold index 5d5be50a427..b798539374a 100644 --- a/tests/gold_tests/pluginTest/test_hooks/log.gold +++ b/tests/gold_tests/pluginTest/test_hooks/log.gold @@ -9,3 +9,22 @@ Session: event=TS_EVENT_HTTP_TXN_CLOSE Transaction: event=TS_EVENT_HTTP_TXN_CLOSE Global: event=TS_EVENT_HTTP_SSN_CLOSE Session: event=TS_EVENT_HTTP_SSN_CLOSE +Global: event=TS_EVENT_VCONN_START +Global: ssl flag=1 +Global: event=TS_EVENT_SSL_SERVERNAME +Global: ssl flag=1 +Global: event=TS_EVENT_SSL_CERT +Global: ssl flag=1 +Global: event=TS_EVENT_HTTP_SSN_START +Global: event=TS_EVENT_HTTP_TXN_START +Session: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Session: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Global: event=TS_EVENT_HTTP_TXN_CLOSE +Session: event=TS_EVENT_HTTP_TXN_CLOSE +Transaction: event=TS_EVENT_HTTP_TXN_CLOSE +Global: event=TS_EVENT_HTTP_SSN_CLOSE +Session: event=TS_EVENT_HTTP_SSN_CLOSE +Global: event=TS_EVENT_VCONN_CLOSE +Global: ssl flag=1 diff --git a/tests/gold_tests/pluginTest/test_hooks/one.in b/tests/gold_tests/pluginTest/test_hooks/one.in deleted file mode 100644 index 48306bed26c..00000000000 --- a/tests/gold_tests/pluginTest/test_hooks/one.in +++ /dev/null @@ -1,4 +0,0 @@ -GET /argh HTTP/1.1 -Host: one -X-Debug: X-Remap - diff --git a/tests/gold_tests/pluginTest/test_hooks/spurious.in b/tests/gold_tests/pluginTest/test_hooks/spurious.in new file mode 100644 index 00000000000..a88748bbbdf --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/spurious.in @@ -0,0 +1,4 @@ +Global: event=TS_EVENT_VCONN_START +Global: ssl flag=1 +Global: event=TS_EVENT_VCONN_CLOSE +Global: ssl flag=1 diff --git a/tests/gold_tests/pluginTest/test_hooks/ssl/server.key b/tests/gold_tests/pluginTest/test_hooks/ssl/server.key new file mode 100644 index 00000000000..4c7a661a6bd --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/ssl/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/test_hooks/ssl/server.pem b/tests/gold_tests/pluginTest/test_hooks/ssl/server.pem new file mode 100644 index 00000000000..58b9b9715b7 --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/ssl/server.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICszCCAhwCCQD4jSkztmlO1TANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTcwODI4MDM0NDQ1WhcNMjcwODI2MDM0NDQ1WjCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 +uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE +lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 +Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEATX7975NdhIbJ +glda+sXI9a86GgOpiuKO+vKubRJQZA+UlPf2vHEONjC2+7Y1aZvZYaKYL74vxGky +zkgp6ANSPl45lqD632x0e1Z7vzW5TkqK1JB2/xH2WgDcQZmP0FuQHzVNs4GjghDr +HCp1+sQDhfPB4aLmLFeyN0TkhdH1N3M= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py index 6f79daef44f..ae413defe3f 100644 --- a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py +++ b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py @@ -18,6 +18,12 @@ Test TS API Hooks. ''' +Test.SkipUnless( + Condition.HasATSFeature('TS_USE_TLS_ALPN'), + Condition.HasCurlFeature('http2'), +) +Test.ContinueOnFail = True + # test_hooks.so will output test logging to this file. Test.Env["OUTPUT_FILE"] = Test.RunDirectory + "/log.txt" @@ -28,41 +34,58 @@ response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "" } server.addResponse("sessionlog.json", request_header, response_header) -ts = Test.MakeATSProcess("ts") +ts = Test.MakeATSProcess("ts", select_ports=False) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ + 'proxy.config.http.cache.http': 0, # Make sure each request is forwarded to the origin server. + 'proxy.config.proxy_name': 'Poxy_Proxy', # This will be the server name. + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': ( + 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port)), 'proxy.config.url_remap.remap_required': 0, - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'test_hooks' - # 'proxy.config.diags.debug.tags': 'http|test_hooks' + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|test_hooks', }) -ts.Disk.remap_config.AddLine( - "map http://one http://127.0.0.1:{0}".format(server.Variables.Port) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' ) Test.PreparePlugin(Test.Variables.AtsTestToolsDir + '/plugins/test_hooks.cc', ts) +ts.Disk.remap_config.AddLine( + "map http://one http://127.0.0.1:{0}".format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + "map https://one http://127.0.0.1:{0}".format(server.Variables.Port) +) + tr = Test.AddTestRun() -tr.Processes.Default.StartBefore(Test.Processes.ts) -tr.Processes.Default.StartBefore(Test.Processes.server) -tr.Processes.Default.Command = "cp {}/tcp_client.py {}/tcp_client.py".format( - Test.Variables.AtsTestToolsDir, Test.RunDirectory) +# Wait for the micro server +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +# Delay on readiness of our ssl ports +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +# +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --header "Host: one" http://localhost:{0}/argh'.format(ts.Variables.port) +) tr.Processes.Default.ReturnCode = 0 -def sendMsg(msgFile): - - tr = Test.AddTestRun() - tr.Processes.Default.Command = ( - "python {}/tcp_client.py 127.0.0.1 {} {}/{}.in".format( - Test.RunDirectory, ts.Variables.port, Test.TestDirectory, msgFile) - ) - tr.Processes.Default.ReturnCode = 0 - -sendMsg('one') +tr = Test.AddTestRun() +# A small delay so the test_hooks test plugin can assume there is only one HTTP transaction in progress at a time. +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --http2 --insecure --header "Host: one" https://localhost:{0}/argh'.format(ts.Variables.ssl_port) +) +tr.Processes.Default.ReturnCode = 0 tr = Test.AddTestRun() -tr.Processes.Default.Command = "echo test log.gold" +tr.Processes.Default.Command = "echo check log" tr.Processes.Default.ReturnCode = 0 f = tr.Disk.File("log.txt") f.Content = "log.gold" diff --git a/tests/tools/plugins/test_hooks.cc b/tests/tools/plugins/test_hooks.cc index a369148d56f..f5c3897bb20 100644 --- a/tests/tools/plugins/test_hooks.cc +++ b/tests/tools/plugins/test_hooks.cc @@ -44,6 +44,8 @@ char PIName[] = PINAME; // std::fstream logFile; +TSVConn activeVConn; + TSHttpSsn activeSsn; TSHttpTxn activeTxn; @@ -167,6 +169,41 @@ globalContFunc(TSCont, TSEvent event, void *eventData) TSDebug(PIName, "Global: event=%s(%d) eventData=%p", TSHttpEventNameLookup(event), event, eventData); switch (event) { + case TS_EVENT_VCONN_START: { + ALWAYS_ASSERT(!activeVConn) + + auto vConn = static_cast(eventData); + + activeVConn = vConn; + + logFile << "Global: ssl flag=" << TSVConnIsSsl(vConn) << std::endl; + + TSVConnReenable(vConn); + } break; + + case TS_EVENT_SSL_CERT: + case TS_EVENT_SSL_SERVERNAME: { + auto vConn = static_cast(eventData); + + ALWAYS_ASSERT(vConn == activeVConn) + + logFile << "Global: ssl flag=" << TSVConnIsSsl(vConn) << std::endl; + + TSVConnReenable(vConn); + } break; + + case TS_EVENT_VCONN_CLOSE: { + auto vConn = static_cast(eventData); + + ALWAYS_ASSERT(vConn == activeVConn) + + logFile << "Global: ssl flag=" << TSVConnIsSsl(vConn) << std::endl; + + TSVConnReenable(vConn); + + activeVConn = nullptr; + } break; + case TS_EVENT_HTTP_SSN_START: { ALWAYS_ASSERT(!activeSsn) @@ -288,6 +325,22 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, gCont); TSHttpHookAdd(TS_HTTP_TXN_START_HOOK, gCont); TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, gCont); + TSHttpHookAdd(TS_SSL_CERT_HOOK, gCont); + TSHttpHookAdd(TS_SSL_SERVERNAME_HOOK, gCont); + + // NOTE: as of January 2019 these two hooks are only triggered for TLS connections. It seems that, at trafficserver + // startup, spurious data on the TLS TCP port may cause trafficserver to attempt (and fail) to create a TLS + // connection. If this happens, it will result in TS_VCONN_START_HOOK being triggered, and then TS_VCONN_CLOSE_HOOK + // will be triggered when the connection closes due to failure. + // + TSHttpHookAdd(TS_VCONN_START_HOOK, gCont); + TSHttpHookAdd(TS_VCONN_CLOSE_HOOK, gCont); + + // TSHttpHookAdd(TS_SSL_SESSION_HOOK, gCont); -- Event is TS_EVENT_SSL_SESSION_NEW -- Event data is TSHttpSsn + // TSHttpHookAdd(TS_SSL_SERVER_VERIFY_HOOK, gCont); + // TSHttpHookAdd(TS_SSL_VERIFY_CLIENT_HOOK, gCont); + // TSHttpHookAdd(TS_VCONN_OUTBOUND_START_HOOK, gCont); + // TSHttpHookAdd(TS_VCONN_OUTBOUND_CLOSE_HOOK, gCont); sCont = TSContCreate(sessionContFunc, mtx); From fc0ddfa3bd0d2191d92035bc9ec8fc127d0d1d21 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 16 Jan 2019 08:47:51 -0700 Subject: [PATCH 154/526] Changes the string_view::length to ::size --- iocore/net/Net.cc | 4 ++-- iocore/net/UnixNetVConnection.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc index a2c5f5f03a7..3ac1c8208ff 100644 --- a/iocore/net/Net.cc +++ b/iocore/net/Net.cc @@ -64,11 +64,11 @@ configure_net() REC_ReadConfigStringAlloc(ccp, "proxy.config.net.tcp_congestion_control_in"); if (ccp && *ccp != '\0') { - net_ccp_in = {ccp}; + net_ccp_in = ccp; } REC_ReadConfigStringAlloc(ccp, "proxy.config.net.tcp_congestion_control_out"); if (ccp && *ccp != '\0') { - net_ccp_out = {ccp}; + net_ccp_out = ccp; } } diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index f005f3d9948..8b117f1ae36 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -1538,7 +1538,7 @@ UnixNetVConnection::set_tcp_congestion_control(int side) } if (!ccp.empty()) { - int rv = setsockopt(con.fd, IPPROTO_TCP, TCP_CONGESTION, reinterpret_cast(ccp.data()), ccp.length()); + int rv = setsockopt(con.fd, IPPROTO_TCP, TCP_CONGESTION, reinterpret_cast(ccp.data()), ccp.size()); if (rv < 0) { Error("Unable to set TCP congestion control on socket %d to \"%s\", errno=%d (%s)", con.fd, ccp.data(), errno, From 646b2d9802202dee850f52a177bb6f6ef870c65b Mon Sep 17 00:00:00 2001 From: scw00 Date: Wed, 16 Jan 2019 15:01:40 +0800 Subject: [PATCH 155/526] Fixes unmatched port when looking up HostDB --- proxy/http/HttpSM.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 875dba887c3..e9ea12a5e38 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -4096,10 +4096,14 @@ HttpSM::do_hostdb_lookup() // If there is not a current server, we must be looking up the origin // server at the beginning of the transaction - int server_port = t_state.current.server ? - t_state.current.server->dst_addr.host_order_port() : - t_state.server_info.dst_addr.isValid() ? t_state.server_info.dst_addr.host_order_port() : - t_state.hdr_info.client_request.port_get(); + int server_port = 0; + if (t_state.current.server && t_state.current.server->dst_addr.isValid()) { + server_port = t_state.current.server->dst_addr.host_order_port(); + } else if (t_state.server_info.dst_addr.isValid()) { + server_port = t_state.server_info.dst_addr.host_order_port(); + } else { + server_port = t_state.hdr_info.client_request.port_get(); + } if (t_state.api_txn_dns_timeout_value != -1) { SMDebug("http_timeout", "beginning DNS lookup. allowing %d mseconds for DNS lookup", t_state.api_txn_dns_timeout_value); From 8434893eb77e7cddf92bd8907a276c45cb6bdb02 Mon Sep 17 00:00:00 2001 From: Gianfranco Costamagna Date: Sat, 12 Jan 2019 12:58:56 +0100 Subject: [PATCH 156/526] Update common.m4 to correctly use $LIBS instead of $LDFLAGS With Ubuntu, the wl,asneeded flags i injected automatically. This causes the link fail, because of LDFLAGS being put before common.c when autoconf runs. $CC $CPPFLAGS $CFLAGS $LDFLAGS sourcefile.c $LIBS ^^ gcc strips the library because it is put before the file being compiled and then fails. failure: configure:26271: $? = 1 configure:26590: checking for LUAJIT configure:26597: $PKG_CONFIG --exists --print-errors "luajit >= 2.0.4" configure:26600: $? = 0 configure:26614: $PKG_CONFIG --exists --print-errors "luajit >= 2.0.4" configure:26617: $? = 0 configure:27005: result: yes configure:27061: checking jansson.h usability configure:27061: cc -c -g -O2 -fdebug-prefix-map=/trafficserver-8.0.1=. -fstack-protector-strong -Wformat -Werror=format-security -O3 -Wdate-time -D_FORTIFY_SOURCE=2 -D_GNU_SOURCE -DOPENSSL_NO_SSL_INTERN conftest.c >&5 configure:27061: $? = 0 configure:27061: result: yes configure:27061: checking jansson.h presence configure:27061: cc -E -Wdate-time -D_FORTIFY_SOURCE=2 -D_GNU_SOURCE -DOPENSSL_NO_SSL_INTERN conftest.c configure:27061: $? = 0 configure:27061: result: yes configure:27061: checking for jansson.h configure:27061: result: yes configure:27067: checking whether jansson is dynamic configure:27082: cc -o conftest -g -O2 -fdebug-prefix-map=/trafficserver-8.0.1=. -fstack-protector-strong -Wformat -Werror=format-security -O3 -Wdate-time -D_FORTIFY_SOURCE=2 -D_GNU_SOURCE -DOPENSSL_NO_SSL_INTERN -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now -fPIC -ljansson conftest.c -lbrotlienc -lpthread -ldl >&5 /usr/bin/ld: /tmp/cc2Ctsj4.o: in function `main': ./conftest.c:96: undefined reference to `json_object' collect2: error: ld returned 1 exit status configure:27082: $? = 1 with this patch: configure:26271: $? = 1 configure:26590: checking for LUAJIT configure:26597: $PKG_CONFIG --exists --print-errors "luajit >= 2.0.4" configure:26600: $? = 0 configure:26614: $PKG_CONFIG --exists --print-errors "luajit >= 2.0.4" configure:26617: $? = 0 configure:27005: result: yes configure:27061: checking jansson.h usability configure:27061: cc -c -g -O2 -fdebug-prefix-map=/trafficserver-8.0.1=. -fstack-protector-strong -Wformat -Werror=format-security -O3 -Wdate-time -D_FORTIFY_SOURCE=2 -D_GNU_SOURCE -DOPENSSL_NO_SSL_INTERN conftest.c >&5 configure:27061: $? = 0 configure:27061: result: yes configure:27061: checking jansson.h presence configure:27061: cc -E -Wdate-time -D_FORTIFY_SOURCE=2 -D_GNU_SOURCE -DOPENSSL_NO_SSL_INTERN conftest.c configure:27061: $? = 0 configure:27061: result: yes configure:27061: checking for jansson.h configure:27061: result: yes configure:27067: checking whether jansson is dynamic configure:27082: cc -o conftest -g -O2 -fdebug-prefix-map=/trafficserver-8.0.1=. -fstack-protector-strong -Wformat -Werror=format-security -O3 -Wdate-time -D_FORTIFY_SOURCE=2 -D_GNU_SOURCE -DOPENSSL_NO_SSL_INTERN -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now conftest.c -lbrotlienc -lpthread -ldl -fPIC -ljansson >&5 configure:27082: $? = 0 configure:27083: result: yes --- build/common.m4 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/common.m4 b/build/common.m4 index d66b002646d..6f5148a9aac 100644 --- a/build/common.m4 +++ b/build/common.m4 @@ -196,17 +196,17 @@ AC_DEFUN([TS_TRY_COMPILE_NO_WARNING], ]) dnl -dnl TS_LINK_WITH_FLAGS_IFELSE(LDFLAGS, FUNCTION-BODY, +dnl TS_LINK_WITH_FLAGS_IFELSE(LIBS, FUNCTION-BODY, dnl [ACTIONS-IF-LINKS], [ACTIONS-IF-LINK-FAILS]) dnl dnl Tries a link test with the provided flags. dnl AC_DEFUN([TS_LINK_WITH_FLAGS_IFELSE], -[ats_save_LDFLAGS=$LDFLAGS - LDFLAGS="$LDFLAGS $1" +[ats_save_LIBS=$LIBS + LIBS="$LIBS $1" AC_LINK_IFELSE([$2],[$3],[$4]) - LDFLAGS=$ats_save_LDFLAGS + LIBS=$ats_save_LIBS ]) From 6a3c5198ad5498f836986c7c29132f5ecf803a58 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 15 Jan 2019 16:50:18 -0700 Subject: [PATCH 157/526] Allows to run make clean multiple times --- lib/perl/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/perl/Makefile.am b/lib/perl/Makefile.am index 591ee2b709e..4a7c4de2b6b 100644 --- a/lib/perl/Makefile.am +++ b/lib/perl/Makefile.am @@ -32,7 +32,7 @@ Makefile-pl: Makefile.PL $(top_builddir)/config.status $(PERL) Makefile.PL INSTALLDIRS=$(INSTALLDIRS) INSTALL_BASE=$(prefix) PREFIX= clean-local: - -rm Makefile-pl + -rm -f Makefile-pl distclean-local: -rm -rf Makefile-pl MYMETA.* blip From ac123b78962a6eafb05402eb1d7006207382baf9 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 16 Jan 2019 11:47:04 +0900 Subject: [PATCH 158/526] Set "Language" of po files under locale/ja/ --- .../admin-guide/configuration/explicit-forward-proxying.en.po | 2 +- .../admin-guide/configuration/hierachical-caching.en.po | 2 +- .../configuration/transparent-forward-proxying.en.po | 2 +- .../ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po | 2 +- .../ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po | 2 +- .../monitoring/statistics/core/http-transaction.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po | 2 +- .../ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po | 2 +- doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po | 2 +- .../admin-guide/transparent-proxy/router-inline.en.po | 2 +- .../admin-guide/transparent-proxy/wccp-configuration.en.po | 2 +- .../LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po | 2 +- .../ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po | 2 +- .../LC_MESSAGES/appendices/command-line/traffic_logstats.en.po | 2 +- .../LC_MESSAGES/appendices/command-line/traffic_manager.en.po | 2 +- .../ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po | 2 +- doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po | 2 +- doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po | 2 +- doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po | 2 +- .../developer-guide/api/functions/TSActionCancel.en.po | 2 +- .../developer-guide/api/functions/TSActionDone.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po | 2 +- .../developer-guide/api/functions/TSCacheRemove.en.po | 2 +- .../developer-guide/api/functions/TSCacheWrite.en.po | 2 +- .../developer-guide/api/functions/TSConfigDataGet.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po | 2 +- .../developer-guide/api/functions/TSConfigRelease.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po | 2 +- .../developer-guide/api/functions/TSContCreate.en.po | 2 +- .../developer-guide/api/functions/TSContDataGet.en.po | 2 +- .../developer-guide/api/functions/TSContDataSet.en.po | 2 +- .../developer-guide/api/functions/TSContDestroy.en.po | 2 +- .../developer-guide/api/functions/TSContMutexGet.en.po | 2 +- .../developer-guide/api/functions/TSContSchedule.en.po | 2 +- .../developer-guide/api/functions/TSHostLookup.en.po | 2 +- .../api/functions/TSHostLookupResultAddrGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpConnect.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrClone.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrCreate.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrDestroy.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrHostGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrLengthGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrMethodGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrMethodSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrPrint.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrReasonGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrReasonLookup.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrReasonSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrStatusGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrStatusSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrTypeGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrTypeSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrUrlGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrUrlSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrVersionGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHdrVersionSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpHookAdd.en.po | 2 +- .../developer-guide/api/functions/TSHttpParserCreate.en.po | 2 +- .../developer-guide/api/functions/TSHttpSsnClientFdGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpSsnReenable.en.po | 2 +- .../api/functions/TSHttpTxnCacheLookupStatusGet.en.po | 2 +- .../api/functions/TSHttpTxnCacheLookupUrlGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnClientFdGet.en.po | 2 +- .../api/functions/TSHttpTxnClientPacketDscpSet.en.po | 2 +- .../api/functions/TSHttpTxnClientPacketMarkSet.en.po | 2 +- .../api/functions/TSHttpTxnClientPacketTosSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnClientReqGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnClientRespGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po | 2 +- .../api/functions/TSHttpTxnIncomingAddrGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnIntercept.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnIsInternal.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po | 2 +- .../api/functions/TSHttpTxnOutgoingAddrGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnParentProxySet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnReenable.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnServerFdGet.en.po | 2 +- .../api/functions/TSHttpTxnServerIntercept.en.po | 2 +- .../api/functions/TSHttpTxnServerPacketDscpSet.en.po | 2 +- .../api/functions/TSHttpTxnServerPacketMarkSet.en.po | 2 +- .../api/functions/TSHttpTxnServerPacketTosSet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnServerReqGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnServerRespGet.en.po | 2 +- .../developer-guide/api/functions/TSHttpTxnSsnGet.en.po | 2 +- .../api/functions/TSHttpTxnTransformRespGet.en.po | 2 +- .../api/functions/TSHttpTxnTransformedRespCache.en.po | 2 +- .../api/functions/TSHttpTxnUntransformedRespCache.en.po | 2 +- .../api/functions/TSIOBufferBlockReadStart.en.po | 2 +- .../developer-guide/api/functions/TSIOBufferCopy.en.po | 2 +- .../developer-guide/api/functions/TSIOBufferCreate.en.po | 2 +- .../developer-guide/api/functions/TSMBufferCreate.en.po | 2 +- .../developer-guide/api/functions/TSMgmtCounterGet.en.po | 2 +- .../developer-guide/api/functions/TSMgmtFloatGet.en.po | 2 +- .../developer-guide/api/functions/TSMgmtIntGet.en.po | 2 +- .../developer-guide/api/functions/TSMgmtStringGet.en.po | 2 +- .../developer-guide/api/functions/TSMgmtUpdateRegister.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrClone.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrCopy.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrCreate.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrDestroy.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldAppend.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldClone.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldCopy.en.po | 2 +- .../api/functions/TSMimeHdrFieldCopyValues.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldCreate.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldFind.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldGet.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldNext.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldRemove.en.po | 2 +- .../api/functions/TSMimeHdrFieldValueAppend.en.po | 2 +- .../api/functions/TSMimeHdrFieldValueDateInsert.en.po | 2 +- .../api/functions/TSMimeHdrFieldValueDateSet.en.po | 2 +- .../api/functions/TSMimeHdrFieldValueIntSet.en.po | 2 +- .../api/functions/TSMimeHdrFieldValueStringInsert.en.po | 2 +- .../api/functions/TSMimeHdrFieldValueStringSet.en.po | 2 +- .../api/functions/TSMimeHdrFieldValueUintInsert.en.po | 2 +- .../api/functions/TSMimeHdrFieldValueUintSet.en.po | 2 +- .../api/functions/TSMimeHdrFieldValuesClear.en.po | 2 +- .../api/functions/TSMimeHdrFieldValuesCount.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldsClear.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrFieldsCount.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrLengthGet.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrParse.en.po | 2 +- .../developer-guide/api/functions/TSMimeHdrPrint.en.po | 2 +- .../developer-guide/api/functions/TSMimeParserClear.en.po | 2 +- .../developer-guide/api/functions/TSMimeParserCreate.en.po | 2 +- .../developer-guide/api/functions/TSMimeParserDestroy.en.po | 2 +- .../developer-guide/api/functions/TSMutexCreate.en.po | 2 +- .../developer-guide/api/functions/TSMutexDestroy.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po | 2 +- .../developer-guide/api/functions/TSMutexLockTry.en.po | 2 +- .../developer-guide/api/functions/TSMutexUnlock.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po | 2 +- .../api/functions/TSNetAcceptNamedProtocol.en.po | 2 +- .../developer-guide/api/functions/TSNetConnect.en.po | 2 +- .../developer-guide/api/functions/TSTextLogObjectCreate.en.po | 2 +- .../developer-guide/api/functions/TSThreadCreate.en.po | 2 +- .../developer-guide/api/functions/TSThreadDestroy.en.po | 2 +- .../developer-guide/api/functions/TSThreadInit.en.po | 2 +- .../developer-guide/api/functions/TSThreadSelf.en.po | 2 +- .../developer-guide/api/functions/TSTransformCreate.en.po | 2 +- .../api/functions/TSTransformOutputVConnGet.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po | 2 +- .../developer-guide/api/functions/TSUrlFtpTypeGet.en.po | 2 +- .../developer-guide/api/functions/TSUrlFtpTypeSet.en.po | 2 +- .../developer-guide/api/functions/TSUrlHostGet.en.po | 2 +- .../developer-guide/api/functions/TSUrlHostSet.en.po | 2 +- .../developer-guide/api/functions/TSUrlPercentEncode.en.po | 2 +- .../developer-guide/api/functions/TSUrlStringGet.en.po | 2 +- .../developer-guide/api/functions/TSVConnAbort.en.po | 2 +- .../api/functions/TSVConnCacheObjectSizeGet.en.po | 2 +- .../developer-guide/api/functions/TSVConnClose.en.po | 2 +- .../developer-guide/api/functions/TSVConnClosedGet.en.po | 2 +- .../developer-guide/api/functions/TSVConnFdCreate.en.po | 2 +- .../developer-guide/api/functions/TSVConnIsSsl.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po | 2 +- .../developer-guide/api/functions/TSVConnReadVIOGet.en.po | 2 +- .../developer-guide/api/functions/TSVConnReenable.en.po | 2 +- .../developer-guide/api/functions/TSVConnShutdown.en.po | 2 +- .../developer-guide/api/functions/TSVConnSslConnectionGet.en.po | 2 +- .../developer-guide/api/functions/TSVConnTunnel.en.po | 2 +- .../developer-guide/api/functions/TSVConnWrite.en.po | 2 +- .../developer-guide/api/functions/TSVConnWriteVIOGet.en.po | 2 +- .../developer-guide/api/functions/TSVIOBufferGet.en.po | 2 +- .../developer-guide/api/functions/TSVIOContGet.en.po | 2 +- .../developer-guide/api/functions/TSVIOMutexGet.en.po | 2 +- .../developer-guide/api/functions/TSVIONBytesGet.en.po | 2 +- .../developer-guide/api/functions/TSVIONBytesSet.en.po | 2 +- .../developer-guide/api/functions/TSVIONDoneGet.en.po | 2 +- .../developer-guide/api/functions/TSVIONDoneSet.en.po | 2 +- .../developer-guide/api/functions/TSVIONTodoGet.en.po | 2 +- .../developer-guide/api/functions/TSVIOReaderGet.en.po | 2 +- .../developer-guide/api/functions/TSVIOReenable.en.po | 2 +- .../developer-guide/api/functions/TSVIOVConnGet.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po | 2 +- .../developer-guide/api/types/TSCacheLookupResult.en.po | 2 +- .../developer-guide/api/types/TSCacheScanResult.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po | 2 +- .../developer-guide/api/types/TSFetchWakeUpOptions.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po | 2 +- .../developer-guide/api/types/TSIOBuffersSizeIndex.en.po | 2 +- .../developer-guide/api/types/TSLifecycleHookID.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po | 2 +- .../developer-guide/api/types/TSMilestonesType.en.po | 2 +- .../developer-guide/api/types/TSOverridableConfigKey.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po | 2 +- .../developer-guide/api/types/TSRecordAccessType.en.po | 2 +- .../developer-guide/api/types/TSRecordCheckType.en.po | 2 +- .../developer-guide/api/types/TSRecordDataType.en.po | 2 +- .../developer-guide/api/types/TSRecordModeType.en.po | 2 +- .../developer-guide/api/types/TSRecordPersistType.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po | 2 +- .../developer-guide/api/types/TSRecordUpdateType.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po | 2 +- .../api/types/TSServerSessionSharingMatchType.en.po | 2 +- .../api/types/TSServerSessionSharingPoolType.en.po | 2 +- .../LC_MESSAGES/developer-guide/api/types/TSServerState.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po | 2 +- .../developer-guide/api/types/TSVConnCloseFlags.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/architecture/index.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po | 2 +- .../LC_MESSAGES/developer-guide/documentation/structure.en.po | 2 +- .../LC_MESSAGES/developer-guide/host-resolution-proposal.en.po | 2 +- doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po | 2 +- .../plugins/continuations/activating-continuations.en.po | 2 +- .../developer-guide/plugins/continuations/index.en.po | 2 +- .../plugins/example-plugins/blacklist/index.en.po | 2 +- .../developer-guide/plugins/example-plugins/index.en.po | 2 +- .../plugins/example-plugins/query-remap/index.en.po | 2 +- .../plugins/getting-started/a-simple-plugin.en.po | 2 +- .../developer-guide/plugins/getting-started/index.en.po | 2 +- .../plugins/getting-started/naming-conventions.en.po | 2 +- .../developer-guide/plugins/hooks-and-transactions/index.en.po | 2 +- .../developer-guide/plugins/http-transformations/index.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po | 2 +- .../developer-guide/plugins/new-protocol-plugins.en.po | 2 +- .../LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po | 2 +- .../developer-guide/plugins/plugin-management/index.en.po | 2 +- .../ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po | 2 +- .../troubleshooting-tips/unable-to-load-plugins.en.po | 2 +- doc/locale/ja/LC_MESSAGES/index.po | 2 +- 256 files changed, 256 insertions(+), 256 deletions(-) diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/explicit-forward-proxying.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/explicit-forward-proxying.en.po index 898d5d34232..25e8b422934 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/explicit-forward-proxying.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/explicit-forward-proxying.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po index 5da856087b6..50452ac181b 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-forward-proxying.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-forward-proxying.en.po index 98980e657cc..13f8b27e02b 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-forward-proxying.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-forward-proxying.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po index fadf1bf7ebe..9b81249152c 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po index 6458c172ad4..8ea799a0141 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po index abb6c957a89..f866d7640e7 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po index cd5965100d0..143016aab78 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/statistics/core/http-transaction.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/statistics/core/http-transaction.en.po index 3781d5d0f96..471363d1de7 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/statistics/core/http-transaction.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/statistics/core/http-transaction.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po index 95b232c44ca..c9e6dbaf5fb 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po index 0360eae81b2..b8074d755e8 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po index 0ec13c7536e..85b8a87e4c9 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po index 42dce50464c..e03365d616f 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po index d8d2b71a070..884b3edf271 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po index 849046a921a..e7adaa156c2 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po index 426ff9e7c83..4bbd60e970e 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po index b487d7dfa2f..ac912880bc6 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po index 57949aa6aab..0b180032772 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po index 5ca8406e580..3a88821fa8e 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/router-inline.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/router-inline.en.po index 810df020eed..887c4215b53 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/router-inline.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/router-inline.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-configuration.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-configuration.en.po index ed17753417a..d4c7fed49e0 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-configuration.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-configuration.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po index 54968ff9e52..66f8354ff51 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po index bf0353a79d5..a5545be26fd 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logstats.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logstats.en.po index 4be0c589fdc..f6b8e1c1d7e 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logstats.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logstats.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_manager.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_manager.en.po index ac9dff52e23..582f35806a5 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_manager.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_manager.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po index 0905d2f3302..e2ba0406339 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po index 746e4503add..6b48c36fe25 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po index 2d5236118de..4f95157fdd9 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po b/doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po index 42803e27e66..4ab407ab18b 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po index 2723918c9ce..d74a3862f4d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionCancel.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionCancel.en.po index be14cbdf122..1180534fe00 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionCancel.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionCancel.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionDone.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionDone.en.po index ea5c22a1a4d..0aeeea82444 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionDone.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionDone.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po index f7ee312ee3b..7e5924002ad 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRemove.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRemove.en.po index 60feac862b4..b522dc19708 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRemove.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRemove.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheWrite.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheWrite.en.po index ddf7af075dc..02da2025a9d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheWrite.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheWrite.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigDataGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigDataGet.en.po index 014db9ff5e0..d96fe31285d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigDataGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigDataGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po index 62e198cc6df..6f3fb508c6c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigRelease.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigRelease.en.po index ed00a3ebbb9..89955262cee 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigRelease.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigRelease.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po index 85c9af32fa1..72fca4c4e13 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po index 8004e6299d4..1076f0e3a81 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCreate.en.po index 5bb4a37b726..12106d4eb34 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataGet.en.po index 83488fbf677..30c359004b1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataSet.en.po index 702e290a661..f9e9e59e631 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDestroy.en.po index 335d4e66df6..5cc42583257 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContMutexGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContMutexGet.en.po index f3822765e81..b9f6c1ad207 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContMutexGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContMutexGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContSchedule.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContSchedule.en.po index 5e4612a3108..e35760b7bd3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContSchedule.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContSchedule.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookup.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookup.en.po index 214e1bf5ca7..c2819e57ed6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookup.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookup.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookupResultAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookupResultAddrGet.en.po index aa2427ec68b..affce1a85a1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookupResultAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookupResultAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpConnect.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpConnect.en.po index ed111ffed28..889c2a586ce 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpConnect.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpConnect.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrClone.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrClone.en.po index 81265bd73db..63296925cdd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrClone.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrClone.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrCreate.en.po index 5b8739b6ce4..feadb855576 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrDestroy.en.po index 834d7eeb8a2..d598b386a4b 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrHostGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrHostGet.en.po index 5b144f1538e..944086dd43c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrHostGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrHostGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrLengthGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrLengthGet.en.po index 02c092e40c9..d1221964437 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrLengthGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrLengthGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodGet.en.po index a12b8d9875b..037932cb44f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodSet.en.po index 47bc6b34bcf..7c1cd9f97d1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrPrint.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrPrint.en.po index d5adb998de8..3959eb46623 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrPrint.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrPrint.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonGet.en.po index 559c0b0f777..d715b5fe8a9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonLookup.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonLookup.en.po index dc314ae68e6..1a0d5cdda15 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonLookup.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonLookup.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonSet.en.po index 0aa7fe08a1c..3bf0ef3adcc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusGet.en.po index 1a247fe2566..1cf6c1365cb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusSet.en.po index 14ca8b2d27f..3e7a9c11920 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeGet.en.po index e7e194bd994..8dfb2bf8814 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeSet.en.po index 8dfd75c4ec3..08bc78171d4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlGet.en.po index 4277d5ca39d..3ad4f3bbbe8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlSet.en.po index 4da5c060b6c..115321d97f6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionGet.en.po index 32f98824ef4..5129d1b6b23 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionSet.en.po index c0d18a515ef..36b53b7ea25 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHookAdd.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHookAdd.en.po index ee687e7d558..cb2010b4cf5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHookAdd.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHookAdd.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpParserCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpParserCreate.en.po index 0013a3867ac..6236c69515c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpParserCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpParserCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnClientFdGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnClientFdGet.en.po index ee240b013f8..ab881f83d05 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnClientFdGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnClientFdGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnReenable.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnReenable.en.po index 8cefe6ad19b..ca5ea4613f3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnReenable.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnReenable.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.po index 7501beaa18f..5c69d1f134f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.po index b324ee81fe9..f8e1fa7e53c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po index 5c1d0a2f0ba..f520c0d5801 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po index 098f74abfd2..97ea157bd56 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientFdGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientFdGet.en.po index ff2386085ed..6bec3a2adb3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientFdGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientFdGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.po index 9b9bcc62201..022856be3d7 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.po index e63224ed3f0..d4d83b5e699 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.po index 9362cc48efb..29925d9fe8c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientReqGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientReqGet.en.po index 324f13d0d29..89b382e6ebf 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientReqGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientReqGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientRespGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientRespGet.en.po index 18da60ccd6c..200cf20209c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientRespGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientRespGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po index a558e512192..863aa156e33 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.po index 1c7ba0a9b25..54f651dae7f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po index 408ed32e26b..554289bbda5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIntercept.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIntercept.en.po index 9ed5853f481..560784f3b1d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIntercept.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIntercept.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIsInternal.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIsInternal.en.po index 4f5aa078cd4..fa33da6884b 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIsInternal.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIsInternal.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po index ecbdfe5c1a3..162fc54df3f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po index a088669f51e..bc181380886 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.po index 6fb3791abf1..23814b8ffca 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnParentProxySet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnParentProxySet.en.po index 7c33927868c..394c897f3ff 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnParentProxySet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnParentProxySet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnReenable.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnReenable.en.po index 4b02b02847c..5bc3c2f31f4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnReenable.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnReenable.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po index 48b3bfe1b0b..cda8bb323e0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po index a414840be8f..c14330ad47a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerFdGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerFdGet.en.po index dccc704cff0..031bac5d8e1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerFdGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerFdGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerIntercept.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerIntercept.en.po index ab74435d3a2..aa1e8788658 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerIntercept.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerIntercept.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.po index 4ca59ec09c8..5ed0331fb2a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.po index 7893dcf6502..c07c7caf058 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.po index 8ee389dc9c7..9546528d691 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerReqGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerReqGet.en.po index 59cba69f1f3..dba9ef156c0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerReqGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerReqGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerRespGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerRespGet.en.po index 996aff17ddd..d1a9029232c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerRespGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerRespGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnSsnGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnSsnGet.en.po index b46c95dc130..fb19f76c31e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnSsnGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnSsnGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.po index c1fa76cef9a..1e2423afcf3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.po index bbda7be18b6..be17d5da91e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.po index 0f0da58bb1f..9efa034ce76 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferBlockReadStart.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferBlockReadStart.en.po index c45fb4ce668..a473e8bfa71 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferBlockReadStart.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferBlockReadStart.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCopy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCopy.en.po index facfded5b92..47e27dd3a9e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCopy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCopy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCreate.en.po index 46b957b9e95..a40b1d38040 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMBufferCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMBufferCreate.en.po index 53ddabcb43a..7d64116d4e1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMBufferCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMBufferCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtCounterGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtCounterGet.en.po index 9ed77495b84..213dbc79024 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtCounterGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtCounterGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtFloatGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtFloatGet.en.po index 36081786af7..402d466201d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtFloatGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtFloatGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtIntGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtIntGet.en.po index 4aea5a51386..05a5d1b344a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtIntGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtIntGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtStringGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtStringGet.en.po index 2d8283e070e..bf7a54abf60 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtStringGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtStringGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtUpdateRegister.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtUpdateRegister.en.po index 2f9b1db44ee..e6dbb47a86e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtUpdateRegister.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtUpdateRegister.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrClone.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrClone.en.po index b0ca55e8f1c..aa523453f4e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrClone.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrClone.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCopy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCopy.en.po index 264b1c2ef48..3f13bcffd54 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCopy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCopy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCreate.en.po index 701a28552c6..809851fb4b5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrDestroy.en.po index e968302b303..c92bd1f53cb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldAppend.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldAppend.en.po index 4a290cbfde7..c0198a11c32 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldAppend.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldAppend.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldClone.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldClone.en.po index 3e0f6296830..8782f5df73b 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldClone.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldClone.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopy.en.po index 67c688da239..b549a3f4cf3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.po index 9fa22ecf7d8..5c2d6dff50e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCreate.en.po index e8dc678c0aa..02daf7b85d5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po index ca754cca6d4..0b1507a8419 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldFind.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldFind.en.po index d1533279aa8..b7cd17f7055 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldFind.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldFind.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldGet.en.po index d19a37fb28d..dd229aec9b0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po index 09d23117e4a..e1b21c26e0f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po index 35ad3216d1d..03bf2308415 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po index 9ffff34b4d2..4377d251ced 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNext.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNext.en.po index 5664e84b6c6..558a049a439 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNext.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNext.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po index 4d79471123d..1ada0cf1ff0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldRemove.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldRemove.en.po index 907edb0c5d1..82aed0aa798 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldRemove.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldRemove.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.po index d578a04355e..bb61ea2f1ce 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.po index 19bd71ec3ea..8df96b02c60 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.po index c52895de53c..d5168474f62 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.po index 880bbd0a168..b89f05b1e79 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.po index 22cc1836ccc..90693fd77f5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.po index 10e987915fd..603088b6cc0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.po index 2cb98952505..f4fcd8e9aa1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.po index 9f12f14b16e..877fd6f40d6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.po index 40382e70e65..54ed4ddb809 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.po index e390d4e2471..cb5513e6b71 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsClear.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsClear.en.po index cbd49890b68..71973810788 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsClear.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsClear.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsCount.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsCount.en.po index 60264b26624..b611f86780a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsCount.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsCount.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrLengthGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrLengthGet.en.po index 2a0bc003ccb..b6721a940c0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrLengthGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrLengthGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrParse.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrParse.en.po index e4ea44a47db..98ed132fafb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrParse.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrParse.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrPrint.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrPrint.en.po index b3219eb820b..de3397f56c4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrPrint.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrPrint.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserClear.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserClear.en.po index 4d85af6e934..86eeeaca0de 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserClear.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserClear.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserCreate.en.po index 1b7ec03d56f..346898f1dfe 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserDestroy.en.po index f304f552162..522d683cfbc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexCreate.en.po index e9e447c29a0..be93a2ac6f7 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexDestroy.en.po index cbbdfcefcb4..138c1834dfc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po index 9b0f45be2f9..1854e16b486 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLockTry.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLockTry.en.po index 153fb0ecbcf..145bb30e221 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLockTry.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLockTry.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexUnlock.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexUnlock.en.po index 4ed40b45892..163690e6a62 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexUnlock.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexUnlock.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po index 659236f87e9..06c8c920484 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.po index 465db35570a..2d360ab8e5e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetConnect.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetConnect.en.po index 00fa4fde68e..aeaf0944e9c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetConnect.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetConnect.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTextLogObjectCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTextLogObjectCreate.en.po index 9e68077b1a4..549163b79bd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTextLogObjectCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTextLogObjectCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadCreate.en.po index 95c8fac6a64..87f0ec5c1c4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadDestroy.en.po index acb0f844946..1e2a3ea2347 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadInit.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadInit.en.po index 4d5fdb0ef23..64986a4d4aa 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadInit.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadInit.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadSelf.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadSelf.en.po index 909ab34a0c4..d9ce3fa435a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadSelf.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadSelf.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformCreate.en.po index d1180cb7f8a..dcc72fbf831 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformOutputVConnGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformOutputVConnGet.en.po index dab9314e330..4f95e5f8b59 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformOutputVConnGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformOutputVConnGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po index 5e0d01e034f..b178f485e0c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po index 0b92a480ab3..c7b0459c322 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeGet.en.po index 94950ab3900..fd0eb10791d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeSet.en.po index bbf9a4da6b4..d3dc4710da5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostGet.en.po index a40ab5522be..f0b30ffe67e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostSet.en.po index cea8d0ea864..81bfd6be1c6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlPercentEncode.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlPercentEncode.en.po index c1d14de78e9..c69a79ba069 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlPercentEncode.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlPercentEncode.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlStringGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlStringGet.en.po index fe7f7219878..465bde2ab19 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlStringGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlStringGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnAbort.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnAbort.en.po index 1c2f614f314..da1cb51f0e3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnAbort.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnAbort.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.po index eac92163cbe..d2a5f610607 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClose.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClose.en.po index c4254188f66..77b78ac4520 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClose.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClose.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClosedGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClosedGet.en.po index bbbf4a842a5..233b739a48d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClosedGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClosedGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnFdCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnFdCreate.en.po index 01cf14618c6..29acfa16b47 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnFdCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnFdCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnIsSsl.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnIsSsl.en.po index f91c31c7694..150264e3bdd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnIsSsl.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnIsSsl.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po index bd1fc967676..4a36fc97fcd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReadVIOGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReadVIOGet.en.po index 20ea65afb50..32a02bb70b2 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReadVIOGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReadVIOGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReenable.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReenable.en.po index c002158cf2d..6e9c9351d06 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReenable.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReenable.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnShutdown.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnShutdown.en.po index b87fc4915ca..d8be0c8405c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnShutdown.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnShutdown.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnSslConnectionGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnSslConnectionGet.en.po index 929d058ca92..1e2ac0f60a8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnSslConnectionGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnSslConnectionGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnTunnel.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnTunnel.en.po index 6304684a202..68307704c40 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnTunnel.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnTunnel.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWrite.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWrite.en.po index de9281ded29..cff36010ad8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWrite.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWrite.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWriteVIOGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWriteVIOGet.en.po index 4b07b214005..050e028564f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWriteVIOGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWriteVIOGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOBufferGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOBufferGet.en.po index 6cdcca43e77..fe278eac324 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOBufferGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOBufferGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOContGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOContGet.en.po index 3f068224b9b..ceddc36b193 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOContGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOContGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOMutexGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOMutexGet.en.po index 9f465c82768..d80ca2e497a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOMutexGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOMutexGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesGet.en.po index 094954074a4..86563695ac0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesSet.en.po index b177fc4b70b..d11a794ab47 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneGet.en.po index 1a4d431e103..acc886e7efa 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneSet.en.po index 81e57516584..f2f9e09f492 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONTodoGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONTodoGet.en.po index 8b1ad29a9ea..bf09ac13e0d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONTodoGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONTodoGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReaderGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReaderGet.en.po index 822e3a3fd19..4bf20af7edd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReaderGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReaderGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReenable.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReenable.en.po index 9055af66aa1..b770bf0f858 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReenable.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReenable.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOVConnGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOVConnGet.en.po index 167b93c8be4..4dc76a9189c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOVConnGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOVConnGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po index 7238ddcce93..28b796d2e50 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po index b1b4421a65d..7c7c5f0c0cc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po index aa63d2e2a40..0678fb183ed 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po index ee7557b01f1..7e6addbf3bb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po index 317c448c049..ab88ed1dc6e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po index ec5884e043e..4d8b048bdb4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po index 12a6b400c0f..ddfd9d60fb0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po index 22f1086adc9..9426813bb9e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheLookupResult.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheLookupResult.en.po index 709e91f3931..d60ee32340a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheLookupResult.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheLookupResult.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheScanResult.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheScanResult.en.po index f0fe83cab7f..c35316efaa1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheScanResult.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheScanResult.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po index 45ae9667fff..bd06fad7205 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSFetchWakeUpOptions.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSFetchWakeUpOptions.en.po index 6a6845d9f69..2ede3ea4d18 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSFetchWakeUpOptions.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSFetchWakeUpOptions.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po index 24cdad96eb2..f1ddb8e72dc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po index e9bbb77261a..487d0c1e237 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po index 37232140986..5d2cc4dc6dc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSIOBuffersSizeIndex.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSIOBuffersSizeIndex.en.po index de6366ea265..0d0176dee21 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSIOBuffersSizeIndex.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSIOBuffersSizeIndex.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLifecycleHookID.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLifecycleHookID.en.po index b61b6c23d42..2e7e011ffc9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLifecycleHookID.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLifecycleHookID.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po index 8b21d732676..0a0ca36af14 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSMilestonesType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSMilestonesType.en.po index a7f157af159..511f51e0356 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSMilestonesType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSMilestonesType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSOverridableConfigKey.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSOverridableConfigKey.en.po index 2dfdc7d1c34..29033da0c1b 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSOverridableConfigKey.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSOverridableConfigKey.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po index 3a6ce4bf797..f02c0911fef 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordAccessType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordAccessType.en.po index 7d74efeff81..ff3bd7d50a0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordAccessType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordAccessType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordCheckType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordCheckType.en.po index 3176194b3b1..88d87e2dff2 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordCheckType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordCheckType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordDataType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordDataType.en.po index 3114d74f88f..999e22e1f44 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordDataType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordDataType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordModeType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordModeType.en.po index 4e524de3df9..089ce2b5326 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordModeType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordModeType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordPersistType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordPersistType.en.po index 39c68c1eb1f..17d441b9ff6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordPersistType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordPersistType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po index e75e01ae60d..519c25a8290 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordUpdateType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordUpdateType.en.po index 7e133c66806..def18f162d7 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordUpdateType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordUpdateType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po index e5fb4a23ca1..246775c46e5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po index 8c3387863b0..70b1fa1bebf 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingMatchType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingMatchType.en.po index 68b2eb5bd9f..06da651d222 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingMatchType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingMatchType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingPoolType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingPoolType.en.po index 81baed7c984..8fc8c227132 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingPoolType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingPoolType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerState.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerState.en.po index 866e80a2755..66a04503aa8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerState.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerState.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po index bb99e5d28b0..3f850f17c11 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po index 9535782f2a4..fe13d2455a5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSVConnCloseFlags.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSVConnCloseFlags.en.po index c7dcc7ad455..ef28d6361a9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSVConnCloseFlags.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSVConnCloseFlags.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/index.en.po index bb9118fe45e..31725636b4d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po index 82a18e0f1f1..c9fc94d2a98 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/documentation/structure.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/documentation/structure.en.po index 26a72e578f2..a8ae68bc858 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/documentation/structure.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/documentation/structure.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/host-resolution-proposal.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/host-resolution-proposal.en.po index 9d41993ec25..1beb235f46d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/host-resolution-proposal.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/host-resolution-proposal.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po index 904e8e50153..ee15a182a60 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/activating-continuations.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/activating-continuations.en.po index db401a5e678..5e8bbaabdd9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/activating-continuations.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/activating-continuations.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/index.en.po index 03bb83b9b69..b2595d96af8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po index 211ea63547a..b614f19d572 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/index.en.po index 045c695bff7..3a9ea8559eb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/query-remap/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/query-remap/index.en.po index 52efbb28012..2a8da8e0553 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/query-remap/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/query-remap/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/a-simple-plugin.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/a-simple-plugin.en.po index 561d4ead1e4..12156f19633 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/a-simple-plugin.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/a-simple-plugin.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/index.en.po index 4698548af7e..c89a6c4051e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/naming-conventions.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/naming-conventions.en.po index b615a8086a8..412bf43cf76 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/naming-conventions.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/naming-conventions.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/hooks-and-transactions/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/hooks-and-transactions/index.en.po index bf7bae4deb2..9d19d647d6d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/hooks-and-transactions/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/hooks-and-transactions/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/http-transformations/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/http-transformations/index.en.po index 972a0b5e444..c8bc94fade5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/http-transformations/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/http-transformations/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po index 25e4dc67c63..555960c2bc5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po index f0fdc426cce..8bdf0260136 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/new-protocol-plugins.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/new-protocol-plugins.en.po index 7d685845fd0..4f6fb77df81 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/new-protocol-plugins.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/new-protocol-plugins.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po index b2cb92a5127..7d3bfa686b6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-management/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-management/index.en.po index 0a01c411adc..847744c496c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-management/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-management/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po index 32dead14747..41d0ccbb9b9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips/unable-to-load-plugins.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips/unable-to-load-plugins.en.po index 4db6599dc5a..0d8b906e4ad 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips/unable-to-load-plugins.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips/unable-to-load-plugins.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/index.po b/doc/locale/ja/LC_MESSAGES/index.po index bde91362236..4dc43f6a175 100644 --- a/doc/locale/ja/LC_MESSAGES/index.po +++ b/doc/locale/ja/LC_MESSAGES/index.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" From 726c2d3a239a544c57b93badc6635454e85a703b Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 16 Jan 2019 13:13:08 -0600 Subject: [PATCH 159/526] Add ifdef so tls test client will compile on non-openssl-1.1.x --- tests/gold_tests/tls/ssl-post.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/gold_tests/tls/ssl-post.c b/tests/gold_tests/tls/ssl-post.c index 00dc3a07a3f..1b3a2fb2f68 100644 --- a/tests/gold_tests/tls/ssl-post.c +++ b/tests/gold_tests/tls/ssl-post.c @@ -89,7 +89,10 @@ spawn_same_session_send(void *arg) SSL_CTX *client_ctx = SSL_CTX_new(SSLv23_client_method()); SSL *ssl = SSL_new(client_ctx); +#if OPENSSL_VERSION_NUMBER >= 0x10100000 SSL_set_max_proto_version(ssl, TLS1_2_VERSION); +#endif + SSL_set_session(ssl, tinfo->session); SSL_set_fd(ssl, sfd); @@ -283,7 +286,9 @@ main(int argc, char *argv[]) SSL_CTX *client_ctx = SSL_CTX_new(SSLv23_client_method()); SSL *ssl = SSL_new(client_ctx); +#if OPENSSL_VERSION_NUMBER >= 0x10100000 SSL_set_max_proto_version(ssl, TLS1_2_VERSION); +#endif SSL_set_fd(ssl, sfd); int ret = SSL_connect(ssl); From 04bfb458c8390130f5d5cb93b8117ae4b9de15ea Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 16 Jan 2019 13:09:32 -0600 Subject: [PATCH 160/526] Fix potential unterminated string in logging in state machine. --- proxy/http/HttpSM.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index e9ea12a5e38..f59a2c20548 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -49,6 +49,7 @@ #include #include #include +#include #define DEFAULT_RESPONSE_BUFFER_SIZE_INDEX 6 // 8K #define DEFAULT_REQUEST_BUFFER_SIZE_INDEX 6 // 8K @@ -5217,9 +5218,12 @@ HttpSM::mark_host_failure(HostDBInfo *info, time_t time_down) if (info->app.http_data.last_failure == 0) { char *url_str = t_state.hdr_info.client_request.url_string_get(&t_state.arena, nullptr); Log::error("%s", lbw() - .print("CONNECT: could not connect to {} for '{}' (setting last failure time) connect_result={}\0", - t_state.current.server->dst_addr, url_str ? url_str : "", - ts::bwf::Errno(t_state.current.server->connect_result)) + .clip(1) + .print("CONNECT Error: {} connecting to {} for '{}' (setting last failure time)", + ts::bwf::Errno(t_state.current.server->connect_result), t_state.current.server->dst_addr, + ts::bwf::FirstOf(url_str, "")) + .extend(1) + .write('\0') .data()); if (url_str) { From c3c3a9c667120ab044b3c23938989f4a7bbaa497 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 16 Jan 2019 21:58:39 +0000 Subject: [PATCH 161/526] Fix a content issue and a couple format issues. --- doc/admin-guide/files/records.config.en.rst | 8 ++++---- doc/admin-guide/plugins/index.en.rst | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index b7d85c932b0..ab814014f3c 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -1759,7 +1759,7 @@ Proxy User Variables ``10.0.2.123`` A single IP Address. ``10.0.3.1-10.0.3.254`` A range of IP address. ``10.0.4.0/24`` A range of IP address specified by CIDR notation. - ======================= ============================================================ + ======================= =========================================================== .. important:: @@ -1768,7 +1768,7 @@ Proxy User Variables information. See :ts:cv:`proxy.config.http.server_ports` for information on how to enable Proxy Protocol on a port. - See :ref:`proxy-protocol` for more discussion on how |TS| tranforms the `Forwarded: header. + See :ref:`proxy-protocol` for more discussion on how |TS| tranforms the `Forwarded: header`. .. ts:cv:: CONFIG proxy.config.http.normalize_ae INT 1 :reloadable: @@ -3507,9 +3507,9 @@ Client-Related Configuration :code:`DISABLED` Server Certificate will not be verified :code:`PERMISSIVE` - Certificate will be verified and the connection will not be established if verification fails. -:code:`ENFORCED` The provided certificate will be verified and the connection will be established irrespective of the verification result. If verification fails the name of the server will be logged. +:code:`ENFORCED` + Certificate will be verified and the connection will not be established if verification fails. .. ts:cv:: CONFIG proxy.config.ssl.client.verify.server.properties STRING ALL :reloadable: diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index 9d8d9b0c5f5..4f750a31a65 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -62,6 +62,7 @@ Plugins that are considered stable are installed by default in |TS| releases. Regex Remap Regex Revalidate Remap Purge + Slice Stats over HTTP TCPInfo XDebug From d9e01f278424e1b520fa8b805dcc1008d838361d Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 16 Jan 2019 18:46:33 -0700 Subject: [PATCH 162/526] Eliminates expensive librecords and malloc on drain --- proxy/ProxyClientSession.h | 9 ++++----- src/traffic_server/traffic_server.cc | 8 ++++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/proxy/ProxyClientSession.h b/proxy/ProxyClientSession.h index 9637a9a4c50..7e1d03974ff 100644 --- a/proxy/ProxyClientSession.h +++ b/proxy/ProxyClientSession.h @@ -69,6 +69,9 @@ struct ProxyError { uint32_t code = 0; }; +// A little ugly, but this global is tracked by traffic_server. +extern bool ts_is_draining; + class ProxyClientSession : public VConnection { public: @@ -152,11 +155,7 @@ class ProxyClientSession : public VConnection bool is_draining() const { - RecInt draining; - if (RecGetRecordInt("proxy.node.config.draining", &draining) != REC_ERR_OKAY) { - return false; - } - return draining != 0; + return ts_is_draining; } // Initiate an API hook invocation. diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index b2753c83331..79e534f400b 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -170,6 +170,9 @@ static bool signal_received[NSIG]; // -1: cache is already initialized, don't delay. static int delay_listen_for_cache_p; +// Keeps track if the server is in draining state, follows the proxy.node.config.draining metric +bool ts_is_draining = false; + AppVersionInfo appVersionInfo; // Build info for this application static ArgumentDescription argument_descriptions[] = { @@ -284,6 +287,7 @@ class SignalContinuation : public Continuation RecInt timeout = 0; if (RecGetRecordInt("proxy.config.stop.shutdown_timeout", &timeout) == REC_ERR_OKAY && timeout) { RecSetRecordInt("proxy.node.config.draining", 1, REC_SOURCE_DEFAULT); + ts_is_draining = true; if (!remote_management_flag) { // Close listening sockets here only if TS is running standalone RecInt close_sockets = 0; @@ -2036,8 +2040,8 @@ mgmt_restart_shutdown_callback(void *, char *, int /* data_len ATS_UNUSED */) static void * mgmt_drain_callback(void *, char *arg, int len) { - ink_assert(len > 1 && (arg[0] == '0' || arg[0] == '1')); - RecSetRecordInt("proxy.node.config.draining", arg[0] == '1', REC_SOURCE_DEFAULT); + ts_is_draining = (len == 2 && arg[0] == '1'); + RecSetRecordInt("proxy.node.config.draining", ts_is_draining ? 1 : 0, REC_SOURCE_DEFAULT); return nullptr; } From 2bcc9eba77703ab03b164b0c4e9ed47eac3eb672 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Thu, 17 Jan 2019 17:40:34 -0600 Subject: [PATCH 163/526] Chamge src/tscpp/api/TransformationPlugin.cc to avoid error for -Werror=subobject-linkage . --- src/tscpp/api/TransformationPlugin.cc | 31 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/tscpp/api/TransformationPlugin.cc b/src/tscpp/api/TransformationPlugin.cc index 435e498d6ab..5d8507539be 100644 --- a/src/tscpp/api/TransformationPlugin.cc +++ b/src/tscpp/api/TransformationPlugin.cc @@ -34,28 +34,27 @@ #define INT64_MAX (9223372036854775807LL) #endif -using namespace atscppapi; -using atscppapi::TransformationPlugin; - -namespace +namespace atscppapi { -class ResumeAfterPauseCont : public Continuation +namespace detail { -public: - ResumeAfterPauseCont() : Continuation() {} + class ResumeAfterPauseCont : public Continuation + { + public: + ResumeAfterPauseCont() : Continuation() {} - ResumeAfterPauseCont(Continuation::Mutex m) : Continuation(m) {} + ResumeAfterPauseCont(Continuation::Mutex m) : Continuation(m) {} -protected: - int _run(TSEvent event, void *edata) override; -}; + protected: + int _run(TSEvent event, void *edata) override; + }; -} // end anonymous namespace +} // end namespace detail /** * @private */ -struct atscppapi::TransformationPluginState : noncopyable, public ResumeAfterPauseCont { +struct TransformationPluginState : noncopyable, public detail::ResumeAfterPauseCont { TSVConn vconn_; Transaction &transaction_; TransformationPlugin &transformation_plugin_; @@ -107,8 +106,14 @@ struct atscppapi::TransformationPluginState : noncopyable, public ResumeAfterPau } }; +} // end namespace atscppapi + +using namespace atscppapi; + namespace { +using ResumeAfterPauseCont = atscppapi::detail::ResumeAfterPauseCont; + void cleanupTransformation(TSCont contp) { From f4944ee494316245da5bc1decffbf0af87f991e4 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 17 Jan 2019 10:21:44 -0600 Subject: [PATCH 164/526] MIME: Update MIMEField::name_get and MIMEField::value_get to return string_view. --- proxy/hdrs/MIME.cc | 65 +++++++++++++++--------------------- proxy/hdrs/MIME.h | 28 +++++++++------- src/traffic_server/InkAPI.cc | 6 ++-- 3 files changed, 46 insertions(+), 53 deletions(-) diff --git a/proxy/hdrs/MIME.cc b/proxy/hdrs/MIME.cc index c2a22c8c257..33ab602c265 100644 --- a/proxy/hdrs/MIME.cc +++ b/proxy/hdrs/MIME.cc @@ -1476,9 +1476,8 @@ mime_hdr_field_attach(MIMEHdrImpl *mh, MIMEField *field, int check_for_dups, MIM ////////////////////////////////////////////////// if (check_for_dups || (prev_dup && (!prev_dup->is_dup_head()))) { - int length; - const char *name = mime_field_name_get(field, &length); - prev_dup = mime_hdr_field_find(mh, name, length); + std::string_view name{field->name_get()}; + prev_dup = mime_hdr_field_find(mh, name.data(), static_cast(name.size())); ink_assert((prev_dup == nullptr) || (prev_dup->is_dup_head())); } @@ -1595,9 +1594,8 @@ mime_hdr_field_detach(MIMEHdrImpl *mh, MIMEField *field, bool detach_all_dups) } } else // need to walk list to find and patch out from predecessor { - int name_length; - const char *name = mime_field_name_get(field, &name_length); - MIMEField *prev = mime_hdr_field_find(mh, name, name_length); + std::string_view name{field->name_get()}; + MIMEField *prev = mime_hdr_field_find(mh, name.data(), static_cast(name.size())); while (prev && (prev->m_next_dup != field)) { prev = prev->m_next_dup; @@ -1726,15 +1724,13 @@ mime_field_destroy(MIMEHdrImpl * /* mh ATS_UNUSED */, MIMEField *field) field->m_readiness = MIME_FIELD_SLOT_READINESS_DELETED; } -const char * -mime_field_name_get(const MIMEField *field, int *length) +std::string_view +MIMEField::name_get() const { - *length = field->m_len_name; - if (field->m_wks_idx >= 0) { - return hdrtoken_index_to_wks(field->m_wks_idx); - } else { - return field->m_ptr_name; + if (m_wks_idx >= 0) { + return {hdrtoken_index_to_wks(m_wks_idx), m_len_name}; } + return {m_ptr_name, m_len_name}; } void @@ -1781,45 +1777,42 @@ MIMEField::value_get_index(const char *value, int length) const return retval; } -const char * -mime_field_value_get(const MIMEField *field, int *length) +std::string_view +MIMEField::value_get() const { - *length = field->m_len_value; - return field->m_ptr_value; + return {m_ptr_value, m_len_value}; } int32_t mime_field_value_get_int(const MIMEField *field) { - int length; - const char *str = mime_field_value_get(field, &length); + std::string_view value{field->value_get()}; - return mime_parse_int(str, str + length); + return mime_parse_int(value.data(), value.data() + value.size()); } uint32_t mime_field_value_get_uint(const MIMEField *field) { - int length; - const char *str = mime_field_value_get(field, &length); - return mime_parse_uint(str, str + length); + std::string_view value{field->value_get()}; + + return mime_parse_uint(value.data(), value.data() + value.size()); } int64_t mime_field_value_get_int64(const MIMEField *field) { - int length; - const char *str = mime_field_value_get(field, &length); + std::string_view value{field->value_get()}; - return mime_parse_int64(str, str + length); + return mime_parse_int64(value.data(), value.data() + value.size()); } time_t mime_field_value_get_date(const MIMEField *field) { - int length; - const char *str = mime_field_value_get(field, &length); - return mime_parse_date(str, str + length); + std::string_view value{field->value_get()}; + + return mime_parse_date(value.data(), value.data() + value.size()); } const char * @@ -1828,10 +1821,9 @@ mime_field_value_get_comma_val(const MIMEField *field, int *length, int idx) // some fields (like Date) contain commas but should not be ripped apart if (!field->supports_commas()) { if (idx == 0) { - return mime_field_value_get(field, length); - } else { - return nullptr; + return field->value_get(length); } + return nullptr; } else { Str *str; StrList list(false); @@ -1864,16 +1856,13 @@ mime_field_value_get_comma_val_count(const MIMEField *field) int mime_field_value_get_comma_list(const MIMEField *field, StrList *list) { - const char *str; - int len; - - str = mime_field_value_get(field, &len); + std::string_view value{field->value_get()}; // if field doesn't support commas, don't rip apart. if (!field->supports_commas()) { - list->append_string(str, len); + list->append_string(value.data(), static_cast(value.size())); } else { - HttpCompat::parse_tok_list(list, 1, str, len, ','); + HttpCompat::parse_tok_list(list, 1, value.data(), static_cast(value.size()), ','); } return list->count; diff --git a/proxy/hdrs/MIME.h b/proxy/hdrs/MIME.h index 59431a67394..5e78834efd7 100644 --- a/proxy/hdrs/MIME.h +++ b/proxy/hdrs/MIME.h @@ -141,6 +141,8 @@ struct MIMEField { return true; // by default, assume supports commas } + /// @return The name of @a this field. + std::string_view name_get() const; const char *name_get(int *length) const; /** Find the index of the value in the multi-value field. @@ -157,7 +159,10 @@ struct MIMEField { */ int value_get_index(const char *value, int length) const; + /// @return The value of @a this field. + std::string_view value_get() const; const char *value_get(int *length) const; + int32_t value_get_int() const; uint32_t value_get_uint() const; int64_t value_get_int64() const; @@ -655,11 +660,9 @@ inkcoreapi MIMEField *mime_hdr_prepare_for_value_set(HdrHeap *heap, MIMEHdrImpl void mime_field_destroy(MIMEHdrImpl *mh, MIMEField *field); -const char *mime_field_name_get(const MIMEField *field, int *length); void mime_field_name_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int16_t name_wks_idx_or_neg1, const char *name, int length, bool must_copy_string); -inkcoreapi const char *mime_field_value_get(const MIMEField *field, int *length); int32_t mime_field_value_get_int(const MIMEField *field); uint32_t mime_field_value_get_uint(const MIMEField *field); int64_t mime_field_value_get_int64(const MIMEField *field); @@ -744,7 +747,9 @@ int mime_parse_integer(const char *&buf, const char *end, int *integer); inline const char * MIMEField::name_get(int *length) const { - return (mime_field_name_get(this, length)); + auto name{this->name_get()}; + *length = int(name.size()); + return name.data(); } /*------------------------------------------------------------------------- @@ -788,7 +793,9 @@ MIMEField::name_is_valid() const inline const char * MIMEField::value_get(int *length) const { - return (mime_field_value_get(this, length)); + auto value{this->value_get()}; + *length = int(value.size()); + return value.data(); } inline int32_t @@ -1253,20 +1260,17 @@ MIMEHdr::value_get_index(const char *name, int name_length, const char *value, i inline const char * MIMEHdr::value_get(const char *name, int name_length, int *value_length_return) const { - // ink_assert(valid()); - const MIMEField *field = field_find(name, name_length); - - if (field) - return (mime_field_value_get(field, value_length_return)); - else - return (nullptr); + if (const MIMEField *field = field_find(name, name_length); field) { + return field->value_get(value_length_return); + } + return nullptr; } inline std::string_view MIMEHdr::value_get(std::string_view const &name) const { if (MIMEField const *field = field_find(name.data(), name.size()); field) { - return {field->m_ptr_value, field->m_len_value}; + return field->value_get(); } return {}; } diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 2969dbcbcf2..b92e42c19d3 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -2684,7 +2684,7 @@ TSMimeFieldValueGet(TSMBuffer /* bufp ATS_UNUSED */, TSMLoc field_obj, int idx, if (idx >= 0) { return mime_field_value_get_comma_val(handle->field_ptr, value_len_ptr, idx); } else { - return mime_field_value_get(handle->field_ptr, value_len_ptr); + return handle->field_ptr->value_get(value_len_ptr); } } @@ -3135,8 +3135,8 @@ TSMimeHdrFieldNameGet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int *length) sdk_assert(sdk_sanity_check_field_handle(field, hdr) == TS_SUCCESS); sdk_assert(sdk_sanity_check_null_ptr((void *)length) == TS_SUCCESS); - MIMEFieldSDKHandle *handle = (MIMEFieldSDKHandle *)field; - return mime_field_name_get(handle->field_ptr, length); + MIMEFieldSDKHandle *handle = reinterpret_cast(field); + return handle->field_ptr->name_get(length); } TSReturnCode From 1e471d22a1cc105b20cb4ff3b492e78ca060e669 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 16 Jan 2019 13:44:48 -0600 Subject: [PATCH 165/526] Add clang-format build target to CMake editor file. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fa91bc3303..9eb3dd032af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,3 +122,4 @@ file(GLOB plugin_files ) add_library(plugins SHARED ${plugin_files}) +add_custom_target(clang-format WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} COMMAND make clang-format) From e18d65a147ad91acd6baccadd2f641e1f7e25304 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 16 Jan 2019 13:13:05 -0600 Subject: [PATCH 166/526] Add log method overload for string_view to LogObject. --- proxy/logging/LogObject.cc | 19 ++++++++++++++----- proxy/logging/LogObject.h | 11 +++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/proxy/logging/LogObject.cc b/proxy/logging/LogObject.cc index 9f765dafcd9..336815cc507 100644 --- a/proxy/logging/LogObject.cc +++ b/proxy/logging/LogObject.cc @@ -532,6 +532,13 @@ LogObject::va_log(LogAccess *lad, const char *fmt, va_list ap) int LogObject::log(LogAccess *lad, const char *text_entry) +{ + // Clang doesn't like initializing a view with nullptr, have to check. + return this->log(lad, std::string_view{text_entry ? text_entry : ""}); +} + +int +LogObject::log(LogAccess *lad, std::string_view text_entry) { LogBuffer *buffer; size_t offset = 0; // prevent warning @@ -546,7 +553,7 @@ LogObject::log(LogAccess *lad, const char *text_entry) } // this verification must be done here in order to avoid 'dead' LogBuffers // with none zero 'in usage' counters (see _checkout_write for more details) - if (!lad && !text_entry) { + if (!lad && text_entry.empty()) { Note("Call to LogAccess without LAD or text entry; skipping"); return Log::FAIL; } @@ -596,8 +603,8 @@ LogObject::log(LogAccess *lad, const char *text_entry) bytes_needed = m_format->field_count() * INK_MIN_ALIGN; } else if (lad) { bytes_needed = m_format->m_field_list.marshal_len(lad); - } else if (text_entry) { - bytes_needed = LogAccess::strlen(text_entry); + } else if (!text_entry.empty()) { + bytes_needed = INK_ALIGN_DEFAULT(text_entry.size() + 1); // must include null terminator. } if (bytes_needed == 0) { @@ -630,8 +637,10 @@ LogObject::log(LogAccess *lad, const char *text_entry) } else if (lad) { bytes_used = m_format->m_field_list.marshal(lad, &(*buffer)[offset]); ink_assert(bytes_needed >= bytes_used); - } else if (text_entry) { - ink_strlcpy(&(*buffer)[offset], text_entry, bytes_needed); + } else if (!text_entry.empty()) { + char *dst = &(*buffer)[offset]; + memcpy(dst, text_entry.data(), text_entry.size()); + memset(dst + text_entry.size(), 0, bytes_needed - text_entry.size()); } buffer->checkin_write(offset); diff --git a/proxy/logging/LogObject.h b/proxy/logging/LogObject.h index 3dc50e278d0..9c59b849b70 100644 --- a/proxy/logging/LogObject.h +++ b/proxy/logging/LogObject.h @@ -119,6 +119,17 @@ class LogObject : public RefCountObj } int log(LogAccess *lad, const char *text_entry = nullptr); + + /** Log the @a text_entry. + * + * @param lad Log accessor. + * @param text_entry Literal text to log. + * @return Result - value from Log::ReturnCodeFlags. + * + * @see Log::ReturnCodeFlags. + */ + int log(LogAccess *lad, std::string_view text_entry); + int va_log(LogAccess *lad, const char *fmt, va_list ap); unsigned roll_files(long time_now = 0); From a752168978895be766850022cebfe6192e68d26c Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 17 Jan 2019 11:28:50 -0700 Subject: [PATCH 167/526] Improves on the ripgrep .rc file --- .ripgreprc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.ripgreprc b/.ripgreprc index 5286a8a9c5a..7a5ea0ac2f9 100644 --- a/.ripgreprc +++ b/.ripgreprc @@ -2,6 +2,4 @@ --no-heading --smart-case --line-number ---multiline ---trim ---one-file-system +--column From 0ee2a26d735c8047688e660f134a6c2cd99ee913 Mon Sep 17 00:00:00 2001 From: "cichang.chen" Date: Mon, 21 Jan 2019 20:44:42 +0800 Subject: [PATCH 168/526] resolve stack-use-after-scope in YamlLogConfig.cc fix warning in YamlLogConfig.cc --- proxy/logging/YamlLogConfig.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc index b3fd5bcdc8f..cc4c6ace7b8 100644 --- a/proxy/logging/YamlLogConfig.cc +++ b/proxy/logging/YamlLogConfig.cc @@ -206,10 +206,10 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } for (auto &&filter : filters) { - const char *filter_name = filter.as().c_str(); - LogFilter *f = cfg->filter_list.find_by_name(filter_name); + std::string filter_name = filter.as().c_str(); + LogFilter *f = cfg->filter_list.find_by_name(filter_name.c_str()); if (!f) { - Warning("Filter %s is not a known filter; cannot add to this LogObject", filter_name); + Warning("Filter %s is not a known filter; cannot add to this LogObject", filter_name.c_str()); } else { logObject->add_filter(f); } From f7f6d7e4a2de4c2d3e26e3fd9aea93c9974a23f8 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Mon, 21 Jan 2019 20:48:22 +0800 Subject: [PATCH 169/526] Do not call dns_result repeatedly for a valid dns result. --- iocore/dns/DNS.cc | 93 ++++++++++++++++++++----------------- iocore/dns/P_DNSProcessor.h | 3 +- 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 5db7f67d0d8..6deb3dc1a79 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -1307,7 +1307,16 @@ dns_result(DNSHandler *h, DNSEntry *e, HostEnt *ent, bool retry, bool tcp_retry) DNS_SUM_DYN_STAT(dns_success_time_stat, Thread::get_hrtime() - e->submit_time); } } + + // Remove head node from DNSHandler::entries queue h->entries.remove(e); + // Release Query ID from DNSHandler + for (int i : e->id) { + if (i < 0) { + break; + } + h->release_query_id(i); + } if (is_debug_tag_set("dns")) { if (is_addr_query(e->qtype)) { @@ -1334,52 +1343,50 @@ dns_result(DNSHandler *h, DNSEntry *e, HostEnt *ent, bool retry, bool tcp_retry) DNS_INCREMENT_DYN_STAT(dns_lookup_fail_stat); } - DNSEntry *dup = nullptr; - while ((dup = e->dups.dequeue())) { - if (dup->post(h, ent)) { - e->dups.enqueue(dup); - goto Lretry; - } - } - - if (e->timeout) { - e->timeout->cancel(e); - e->timeout = nullptr; - } + // Save HostEnt to the head node e->result_ent = ent; + e->retries = 0; + SET_CONTINUATION_HANDLER(e, &DNSEntry::postAllEvent); + e->handleEvent(EVENT_NONE, nullptr); +} - if (h->mutex->thread_holding == e->submit_thread) { - MUTEX_TRY_LOCK(lock, e->action.mutex, h->mutex->thread_holding); - if (!lock.is_locked()) { - Debug("dns", "failed lock for result %s", e->qname); - goto Lretry; - } - for (int i : e->id) { - if (i < 0) { - break; - } - h->release_query_id(i); - } - e->postEvent(0, nullptr); - } else { - for (int i : e->id) { - if (i < 0) { - break; +int +DNSEntry::postAllEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) +{ + /* Traverse the DNSEntry queue and callback + * + * The first DNSEntry object is head node, + * - Pushed into DNSHandler::entries queue, + * - Initial a DNS request and send to named server, + * - Maintained a dups queue which holds the DNSEntry object for the same DNS request, + * - All the DNSEntry in the queue share the same HostEnt result + * + * The head node callback the HostEnt result to the Continuation of all nodes one by one, + * - If one of the callback fails, put the node back to the dups queue and try again later by reschedule the head node, + * - Always call back the head node until the dups queue is empty. + */ + DNSEntry *dup = nullptr; + while ((dup = dups.dequeue())) { + if (dup->post(dnsH, result_ent.get())) { + // If one of the callback fails, put the node back to the dups queue + dups.enqueue(dup); + // Try again by reschedule the head node + if (timeout) { + timeout->cancel(); } - h->release_query_id(i); + timeout = dnsH->mutex->thread_holding->schedule_in(this, MUTEX_RETRY_DELAY); + return EVENT_DONE; } - e->mutex = e->action.mutex; - SET_CONTINUATION_HANDLER(e, &DNSEntry::postEvent); - e->submit_thread->schedule_imm_signal(e); } - return; -Lretry: - e->result_ent = ent; - e->retries = 0; - if (e->timeout) { - e->timeout->cancel(); + + // Process the head node at last + if (post(dnsH, result_ent.get())) { + // If the callback fails, switch the handler to DNSEntry::postOneEvent and reschedule it. + mutex = action.mutex; + SET_HANDLER(&DNSEntry::postOneEvent); + submit_thread->schedule_imm(this); } - e->timeout = h->mutex->thread_holding->schedule_in(e, MUTEX_RETRY_DELAY); + return EVENT_DONE; } int @@ -1396,17 +1403,17 @@ DNSEntry::post(DNSHandler *h, HostEnt *ent) Debug("dns", "failed lock for result %s", qname); return 1; } - postEvent(0, nullptr); + postOneEvent(0, nullptr); } else { mutex = action.mutex; - SET_HANDLER(&DNSEntry::postEvent); + SET_HANDLER(&DNSEntry::postOneEvent); submit_thread->schedule_imm_signal(this); } return 0; } int -DNSEntry::postEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) +DNSEntry::postOneEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) { if (!action.cancelled) { Debug("dns", "called back continuation for %s", qname); diff --git a/iocore/dns/P_DNSProcessor.h b/iocore/dns/P_DNSProcessor.h index 86e6bdb6a46..391f53463de 100644 --- a/iocore/dns/P_DNSProcessor.h +++ b/iocore/dns/P_DNSProcessor.h @@ -159,8 +159,9 @@ struct DNSEntry : public Continuation { int mainEvent(int event, Event *e); int delayEvent(int event, Event *e); + int postAllEvent(int event, Event *e); int post(DNSHandler *h, HostEnt *ent); - int postEvent(int event, Event *e); + int postOneEvent(int event, Event *e); void init(const char *x, int len, int qtype_arg, Continuation *acont, DNSProcessor::Options const &opt); DNSEntry() From 0dab4f3f365e49aeef8381017dabbbf09d96c404 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 18 Jan 2019 14:05:51 -0600 Subject: [PATCH 170/526] TLS Bridge: Fix possible race condition. This may happen if the inbound connection closes just before the outbound connection closes. --- plugins/experimental/tls_bridge/tls_bridge.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/experimental/tls_bridge/tls_bridge.cc b/plugins/experimental/tls_bridge/tls_bridge.cc index d295d08d346..417e984c957 100644 --- a/plugins/experimental/tls_bridge/tls_bridge.cc +++ b/plugins/experimental/tls_bridge/tls_bridge.cc @@ -472,12 +472,14 @@ Bridge::flow_to_outbound() void Bridge::eos(TSVIO vio) { - if (vio == _out._write._vio || vio == _out._read._vio) { + if (nullptr == vio) { + // Generic close for some non-EOS reason. + } else if (vio == _out._write._vio || vio == _out._read._vio) { TSDebug(PLUGIN_TAG, "EOS upstream"); } else if (vio == _ua._write._vio || vio == _ua._read._vio) { TSDebug(PLUGIN_TAG, "EOS user agent"); } else { - TSDebug(PLUGIN_TAG, "EOS from unknown VIO"); + TSDebug(PLUGIN_TAG, "EOS from unknown VIO [%p]", vio); } _out.do_close(); _ua.do_close(); @@ -602,7 +604,8 @@ int CB_Exec(TSCont contp, TSEvent ev_idx, void *data) { auto ctx = static_cast(TSContDataGet(contp)); - + // No point in checking @a ctx for @c nullptr because if it's not there, neither is the + // continuation so things would already be over the cliff. switch (ev_idx) { case TS_EVENT_NET_ACCEPT: ctx->net_accept(static_cast(data)); @@ -626,7 +629,10 @@ CB_Exec(TSCont contp, TSEvent ev_idx, void *data) break; case TS_EVENT_HTTP_TXN_CLOSE: TSDebug(PLUGIN_TAG, "TXN_CLOSE: cleanup"); + ctx->eos(nullptr); // no specific VIO, close up everything. delete ctx; + TSContDataSet(contp, nullptr); // Not sure if necessary, it's cheap, let's be safe. + TSContDestroy(contp); break; default: TSDebug(PLUGIN_TAG, "Event %d", ev_idx); From 45be6020b06874510ee5e02ec9e9c4b52ac5e853 Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Thu, 4 Oct 2018 13:54:55 -0700 Subject: [PATCH 171/526] new scheduling and thread affinity apis with tests --- .../api/functions/TSContSchedule.en.rst | 29 +- .../api/functions/TSContScheduleOnPool.en.rst | 62 ++++ .../functions/TSContScheduleOnThread.en.rst | 40 +++ .../TSContThreadAffinityClear.en.rst | 34 ++ .../functions/TSContThreadAffinityGet.en.rst | 34 ++ .../functions/TSContThreadAffinitySet.en.rst | 42 +++ .../api/types/TSThreadPool.en.rst | 2 - .../plugins/actions/index.en.rst | 6 +- .../writing-handler-functions.en.rst | 6 + example/blacklist_1/blacklist_1.c | 4 +- example/thread_pool/psi.c | 6 +- include/ts/apidefs.h.in | 2 +- include/ts/ts.h | 12 +- include/tscpp/api/AsyncTimer.h | 2 +- include/tscpp/api/Continuation.h | 8 +- iocore/eventsystem/I_Continuation.h | 25 ++ iocore/eventsystem/I_EThread.h | 2 + iocore/eventsystem/P_UnixEThread.h | 11 + iocore/eventsystem/P_UnixEventProcessor.h | 9 +- iocore/eventsystem/UnixEThread.cc | 1 + plugins/background_fetch/background_fetch.cc | 2 +- plugins/esi/esi.cc | 2 +- plugins/experimental/certifier/certifier.cc | 2 +- .../collapsed_forwarding.cc | 2 +- .../experimental/header_freq/header_freq.cc | 2 +- plugins/experimental/inliner/ts.cc | 2 +- plugins/experimental/prefetch/fetch.cc | 2 +- .../stale_while_revalidate.c | 4 +- .../experimental/system_stats/system_stats.c | 4 +- plugins/generator/generator.cc | 2 +- plugins/lua/ts_lua_http_intercept.c | 2 +- plugins/lua/ts_lua_misc.c | 4 +- plugins/regex_revalidate/regex_revalidate.c | 6 +- src/traffic_server/InkAPI.cc | 173 ++++++++- src/traffic_server/InkAPITest.cc | 76 ++-- src/traffic_server/InkIOCoreAPI.cc | 6 + src/tscpp/api/AsyncTimer.cc | 6 +- src/tscpp/api/InterceptPlugin.cc | 2 +- .../cont_schedule/gold/http_200.gold | 9 + .../cont_schedule/gold/schedule.gold | 4 + .../cont_schedule/gold/schedule_on_pool.gold | 9 + .../gold/schedule_on_thread.gold | 4 + .../cont_schedule/gold/thread_affinity.gold | 5 + .../gold_tests/cont_schedule/schedule.test.py | 72 ++++ .../cont_schedule/schedule_on_pool.test.py | 72 ++++ .../cont_schedule/schedule_on_thread.test.py | 72 ++++ .../cont_schedule/thread_affinity.test.py | 72 ++++ tests/tools/plugins/cont_schedule.cc | 327 ++++++++++++++++++ tests/tools/plugins/continuations_verify.cc | 2 +- tests/tools/plugins/ssl_hook_test.cc | 6 +- tests/tools/plugins/ssntxnorder_verify.cc | 2 +- 51 files changed, 1173 insertions(+), 119 deletions(-) create mode 100644 doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst create mode 100644 doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst create mode 100644 doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst create mode 100644 doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst create mode 100644 doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst create mode 100644 tests/gold_tests/cont_schedule/gold/http_200.gold create mode 100644 tests/gold_tests/cont_schedule/gold/schedule.gold create mode 100644 tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold create mode 100644 tests/gold_tests/cont_schedule/gold/schedule_on_thread.gold create mode 100644 tests/gold_tests/cont_schedule/gold/thread_affinity.gold create mode 100644 tests/gold_tests/cont_schedule/schedule.test.py create mode 100644 tests/gold_tests/cont_schedule/schedule_on_pool.test.py create mode 100644 tests/gold_tests/cont_schedule/schedule_on_thread.test.py create mode 100644 tests/gold_tests/cont_schedule/thread_affinity.test.py create mode 100644 tests/tools/plugins/cont_schedule.cc diff --git a/doc/developer-guide/api/functions/TSContSchedule.en.rst b/doc/developer-guide/api/functions/TSContSchedule.en.rst index 944a481402b..dea45b01004 100644 --- a/doc/developer-guide/api/functions/TSContSchedule.en.rst +++ b/doc/developer-guide/api/functions/TSContSchedule.en.rst @@ -26,7 +26,7 @@ Synopsis `#include ` -.. function:: TSAction TSContSchedule(TSCont contp, ink_hrtime delay, TSThreadPool tp) +.. function:: TSAction TSContSchedule(TSCont contp, TSHRTime timeout) Description =========== @@ -38,27 +38,12 @@ not be effective. :arg:`contp` is required to have a mutex, which is provided to The return value can be used to cancel the scheduled event via :func:`TSActionCancel`. This is effective until the continuation :arg:`contp` is being dispatched. However, if it is scheduled on -another thread this can problematic to be correctly timed. The return value can be checked with +another thread this can be problematic to be correctly timed. The return value can be checked with :func:`TSActionDone` to see if the continuation ran before the return, which is possible if -:arg:`delay` is `0`. +:arg:`timeout` is `0`. Returns ``nullptr`` if thread affinity was cleared. -The continuation is scheduled for a particular thread selected from a group of similar threads, as indicated by :arg:`tp`. - -=========================== ======================================================================================= -Pool Properties -=========================== ======================================================================================= -``TS_THREAD_POOL_DEFAULT`` Use the default pool. Continuations using this must not block. -``TS_THREAD_POOL_NET`` Transaction processing threads. Continuations on these threads must not block. -``TS_THREAD_POOL_TASK`` Background threads. Continuations can perform blocking operations. -``TS_THREAD_POOL_SSL`` *DEPRECATED* - these are no longer used as of ATS 6. -``TS_THREAD_POOL_DNS`` DNS request processing. May not exist depending on configuration. Not recommended. -``TS_THREAD_POOL_REMAP`` *DEPRECATED* - these are not longer used. -``TS_THREAD_POOL_CLUSTER`` *DEPRECATED* - these are no longer used as of ATS 7. -``TS_THREAD_POOL_UDP`` *DEPRECATED* -=========================== ======================================================================================= +See Also +======== -In practice, any choice except ``TS_THREAD_POOL_NET`` or ``TS_THREAD_POOL_TASK`` is strong not -recommended. The ``TS_THREAD_POOL_NET`` threads are the same threads on which callback hooks are -called and continuations that use them have the same restrictions. ``TS_THREAD_POOL_TASK`` threads -are threads that exist to perform long or blocking actions, although sufficiently long operation can -impact system performance by blocking other continuations on the threads. +:doc:`TSContScheduleOnPool.en` +:doc:`TSContScheduleOnThread.en` diff --git a/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst b/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst new file mode 100644 index 00000000000..462f6a9aec9 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst @@ -0,0 +1,62 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContScheduleOnPool +******************** + +Synopsis +======== + +`#include ` + +.. function:: TSAction TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) + +Description +=========== + +Mostly the same as :func:`TSContSchedule`. Schedules :arg:`contp` on a random thread that belongs to :arg:`tp`. +If thread type of the thread specified by thread affinity is the same as :arg:`tp`, the :arg:`contp` will +be scheduled on the thread specified by thread affinity. + +The continuation is scheduled for a particular thread selected from a group of similar threads, as indicated by :arg:`tp`. + +=========================== ======================================================================================= +Pool Properties +=========================== ======================================================================================= +``TS_THREAD_POOL_NET`` Transaction processing threads. Continuations on these threads must not block. +``TS_THREAD_POOL_TASK`` Background threads. Continuations can perform blocking operations. +``TS_THREAD_POOL_SSL`` *DEPRECATED* - these are no longer used as of ATS 6. +``TS_THREAD_POOL_DNS`` DNS request processing. May not exist depending on configuration. Not recommended. +``TS_THREAD_POOL_REMAP`` *DEPRECATED* - these are no longer used. +``TS_THREAD_POOL_CLUSTER`` *DEPRECATED* - these are no longer used as of ATS 7. +``TS_THREAD_POOL_UDP`` *DEPRECATED* +=========================== ======================================================================================= + +In practice, any choice except ``TS_THREAD_POOL_NET`` or ``TS_THREAD_POOL_TASK`` is strongly not +recommended. The ``TS_THREAD_POOL_NET`` threads are the same threads on which callback hooks are +called and continuations that use them have the same restrictions. ``TS_THREAD_POOL_TASK`` threads +are threads that exist to perform long or blocking actions, although sufficiently long operation can +impact system performance by blocking other continuations on the threads. + +See Also +======== + +:doc:`TSContSchedule.en` +:doc:`TSContScheduleOnThread.en` diff --git a/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst b/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst new file mode 100644 index 00000000000..fed6313c4e8 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst @@ -0,0 +1,40 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContScheduleOnThread +********************** + +Synopsis +======== + +`#include ` + +.. function:: TSAction TSContScheduleOnThread(TSCont contp, TSHRTime timeout, TSEventThread ethread) + +Description +=========== + +Mostly the same as :func:`TSContSchedule`. Schedules :arg:`contp` on :arg:`ethread`. + +See Also +======== + +:doc:`TSContSchedule.en` +:doc:`TSContScheduleOnPool.en` diff --git a/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst new file mode 100644 index 00000000000..984653511d6 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst @@ -0,0 +1,34 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContThreadAffinityClear +************** + +Synopsis +======== + +`#include ` + +.. function:: void TSContThreadAffinityClear(TSCont contp) + +Description +=========== + +Clear the thread affinity of continuation :arg:`contp`. diff --git a/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst new file mode 100644 index 00000000000..c31363c67e1 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst @@ -0,0 +1,34 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContThreadAffinityGet +************** + +Synopsis +======== + +`#include ` + +.. function:: TSEventThread TSContThreadAffinityGet(TSCont contp) + +Description +=========== + +Get the thread affinity of continuation :arg:`contp`. diff --git a/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst new file mode 100644 index 00000000000..cae29032bce --- /dev/null +++ b/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst @@ -0,0 +1,42 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContThreadAffinitySet +************** + +Synopsis +======== + +`#include ` + +.. function:: TSReturnCode TSContThreadAffinitySet(TSCont contp, TSEventThread ethread) + +Description +=========== + +Set the thread affinity of continuation :arg:`contp` to :arg:`ethread`. Future calls to +:func:`TSContSchedule`, and :func:`TSContScheduleOnPool` that has the same type as :arg:`ethread` +will schedule the continuation on :arg:`ethread`, rather than an arbitrary thread of that type. + +Return Values +============= + +:data:`TS_SUCCESS` if thread affinity of continuation :arg:`contp` was set to :arg:`ethread`, +:data:`TS_ERROR` if not. diff --git a/doc/developer-guide/api/types/TSThreadPool.en.rst b/doc/developer-guide/api/types/TSThreadPool.en.rst index e7140a99fef..298a8505b15 100644 --- a/doc/developer-guide/api/types/TSThreadPool.en.rst +++ b/doc/developer-guide/api/types/TSThreadPool.en.rst @@ -31,8 +31,6 @@ Enum typedef. Enumeration Members =================== -.. c:member:: TSThreadPool TS_THREAD_POOL_DEFAULT - .. c:member:: TSThreadPool TS_THREAD_POOL_NET .. c:member:: TSThreadPool TS_THREAD_POOL_TASK diff --git a/doc/developer-guide/plugins/actions/index.en.rst b/doc/developer-guide/plugins/actions/index.en.rst index afe6ee1cfae..7ea25614de0 100644 --- a/doc/developer-guide/plugins/actions/index.en.rst +++ b/doc/developer-guide/plugins/actions/index.en.rst @@ -97,7 +97,7 @@ Below is an example of typical usage for an action: system is initialized. We'll simply schedule an event on the continuation to occur as soon as the rest of the system is started up. */ - TSContSchedule (contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool (contp, 0, TS_THREAD_POOL_NET); } The example above shows a simple plugin that creates a continuation and @@ -127,7 +127,7 @@ cancel the action. The following sample code implements this: { switch (event) { case (TS_EVENT_IMMEDIATE): - TSContSchedule (contp, 30000, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool (contp, 30000, TS_THREAD_POOL_NET); TSAction actionp = TSNetConnect(contp, 127.0.0.1, 9999); if (!TSActionDone (actionp)) { TSContDataSet (contp, actionp); @@ -168,7 +168,7 @@ cancel the action. The following sample code implements this: system is initialized. We'll simply schedule an event on the continuation to occur as soon as the rest of the system is started up. */ - TSContSchedule (contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool (contp, 0, TS_THREAD_POOL_NET); } The action functions are: diff --git a/doc/developer-guide/plugins/continuations/writing-handler-functions.en.rst b/doc/developer-guide/plugins/continuations/writing-handler-functions.en.rst index b4cde81f68a..7250f25c826 100644 --- a/doc/developer-guide/plugins/continuations/writing-handler-functions.en.rst +++ b/doc/developer-guide/plugins/continuations/writing-handler-functions.en.rst @@ -96,6 +96,8 @@ Event Event Sender :data:`TS_EVENT_IMMEDIATE` :func:`TSVConnClose` :func:`TSVIOReenable` :func:`TSContSchedule` + :func:`TSContScheduleOnPool` + :func:`TSContScheduleOnThread` :data:`TS_EVENT_IMMEDIATE` :data:`TS_HTTP_REQUEST_TRANSFORM_HOOK` :data:`TS_EVENT_IMMEDIATE` :data:`TS_HTTP_RESPONSE_TRANSFORM_HOOK` :data:`TS_EVENT_CACHE_OPEN_READ` :func:`TSCacheRead` Cache VC @@ -112,6 +114,8 @@ Event Event Sender :func:`TSHttpTxnIntercept` :data:`TS_EVENT_HOST_LOOKUP` :func:`TSHostLookup` :type:`TSHostLookupResult` :data:`TS_EVENT_TIMEOUT` :func:`TSContSchedule` + :func:`TSContScheduleOnPool` + :func:`TSContScheduleOnThread` :data:`TS_EVENT_ERROR` :data:`TS_EVENT_VCONN_READ_READY` :func:`TSVConnRead` :type:`TSVIO` :data:`TS_EVENT_VCONN_WRITE_READY` :func:`TSVConnWrite` :type:`TSVIO` @@ -134,3 +138,5 @@ The continuation functions are listed below: - :func:`TSContDestroy` - :func:`TSContMutexGet` - :func:`TSContSchedule` +- :func:`TSContScheduleOnPool` +- :func:`TSContScheduleOnThread` diff --git a/example/blacklist_1/blacklist_1.c b/example/blacklist_1/blacklist_1.c index 34703a2e8ec..7b08186cb7d 100644 --- a/example/blacklist_1/blacklist_1.c +++ b/example/blacklist_1/blacklist_1.c @@ -100,7 +100,7 @@ handle_dns(TSHttpTxn txnp, TSCont contp) TSDebug(PLUGIN_NAME, "Unable to get lock. Will retry after some time"); TSHandleMLocRelease(bufp, hdr_loc, url_loc); TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); - TSContSchedule(contp, RETRY_TIME, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, RETRY_TIME, TS_THREAD_POOL_NET); return; } @@ -188,7 +188,7 @@ read_blacklist(TSCont contp) if (file != NULL) { TSfclose(file); } - TSContSchedule(contp, RETRY_TIME, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, RETRY_TIME, TS_THREAD_POOL_NET); return; } diff --git a/example/thread_pool/psi.c b/example/thread_pool/psi.c index fa465b2c1f2..29db012b1e6 100644 --- a/example/thread_pool/psi.c +++ b/example/thread_pool/psi.c @@ -511,7 +511,7 @@ psi_include(TSCont contp, void *edata ATS_UNUSED) TS_HTTP_READ_REQUEST_HDR, TS_HTTP_OS_DNS and so on...) we could use TSHttpTxnReenable to wake up the transaction instead of sending an event. */ - TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); data->psi_success = 0; data->state = STATE_READ_DATA; TSMutexUnlock(TSContMutexGet(contp)); @@ -749,7 +749,7 @@ transform_handler(TSCont contp, TSEvent event, void *edata ATS_UNUSED) d->contp = contp; d->event = event; TSContDataSet(c, d); - TSContSchedule(c, 10, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(c, 10, TS_THREAD_POOL_NET); return 1; } @@ -765,7 +765,7 @@ transform_handler(TSCont contp, TSEvent event, void *edata ATS_UNUSED) the continuation right away as the thread will call us back on this continuation. */ if (state == STATE_READ_PSI) { - TSContSchedule(contp, 10, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 10, TS_THREAD_POOL_NET); } else { TSMutexUnlock(TSContMutexGet(contp)); cont_data_destroy(TSContDataGet(contp)); diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 5e7bf7c3cac..7adf619de29 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -823,7 +823,6 @@ typedef enum { /* The TASK pool of threads is the primary method of off-loading continuations from the net-threads. Configure this with proxy.config.task_threads in records.config. */ typedef enum { - TS_THREAD_POOL_DEFAULT = -1, TS_THREAD_POOL_NET, TS_THREAD_POOL_TASK, /* unlikely you should use these */ @@ -901,6 +900,7 @@ typedef struct tsapi_cachetxn *TSCacheTxn; typedef struct tsapi_port *TSPortDescriptor; typedef struct tsapi_vio *TSVIO; typedef struct tsapi_thread *TSThread; +typedef struct tsapi_event_thread *TSEventThread; typedef struct tsapi_x509 *TSSslX509; typedef struct tsapi_mutex *TSMutex; typedef struct tsapi_config *TSConfig; diff --git a/include/ts/ts.h b/include/ts/ts.h index a3ba7b3d7bf..d5ee8b0dc54 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -1108,6 +1108,7 @@ tsapi TSThread TSThreadInit(void); tsapi void TSThreadDestroy(TSThread thread); tsapi void TSThreadWait(TSThread thread); tsapi TSThread TSThreadSelf(void); +tsapi TSEventThread TSEventThreadSelf(void); /* -------------------------------------------------------------------------- Mutexes */ @@ -1188,8 +1189,15 @@ tsapi TSCont TSContCreate(TSEventFunc funcp, TSMutex mutexp); tsapi void TSContDestroy(TSCont contp); tsapi void TSContDataSet(TSCont contp, void *data); tsapi void *TSContDataGet(TSCont contp); -tsapi TSAction TSContSchedule(TSCont contp, TSHRTime timeout, TSThreadPool tp); -tsapi TSAction TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */, TSThreadPool tp); +tsapi TSAction TSContSchedule(TSCont contp, TSHRTime timeout); +tsapi TSAction TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp); +tsapi TSAction TSContScheduleOnThread(TSCont contp, TSHRTime timeout, TSEventThread ethread); +tsapi TSAction TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */); +tsapi TSAction TSContScheduleEveryOnPool(TSCont contp, TSHRTime every /* millisecs */, TSThreadPool tp); +tsapi TSAction TSContScheduleEveryOnThread(TSCont contp, TSHRTime every /* millisecs */, TSEventThread ethread); +tsapi TSReturnCode TSContThreadAffinitySet(TSCont contp, TSEventThread ethread); +tsapi TSEventThread TSContThreadAffinityGet(TSCont contp); +tsapi void TSContThreadAffinityClear(TSCont contp); tsapi TSAction TSHttpSchedule(TSCont contp, TSHttpTxn txnp, TSHRTime timeout); tsapi int TSContCall(TSCont contp, TSEvent event, void *edata); tsapi TSMutex TSContMutexGet(TSCont contp); diff --git a/include/tscpp/api/AsyncTimer.h b/include/tscpp/api/AsyncTimer.h index 85df2b58b7b..3e7658a976d 100644 --- a/include/tscpp/api/AsyncTimer.h +++ b/include/tscpp/api/AsyncTimer.h @@ -68,7 +68,7 @@ class AsyncTimer : public AsyncProvider // For convenience, additional constructor prototypes. AsyncTimer(Type type, int period_in_ms, int initial_period_in_ms = 0) - : AsyncTimer(type, period_in_ms, initial_period_in_ms, TS_THREAD_POOL_DEFAULT) + : AsyncTimer(type, period_in_ms, initial_period_in_ms, TS_THREAD_POOL_NET) { } diff --git a/include/tscpp/api/Continuation.h b/include/tscpp/api/Continuation.h index edb6345e0ff..62e40a13af4 100644 --- a/include/tscpp/api/Continuation.h +++ b/include/tscpp/api/Continuation.h @@ -111,9 +111,9 @@ class Continuation // Timeout of zero means no timeout. // Action - schedule(TSHRTime timeout = 0, TSThreadPool tp = TS_THREAD_POOL_DEFAULT) + schedule(TSHRTime timeout = 0, TSThreadPool tp = TS_THREAD_POOL_NET) { - return TSContSchedule(_cont, timeout, tp); + return TSContScheduleOnPool(_cont, timeout, tp); } // Timeout of zero means no timeout. @@ -125,9 +125,9 @@ class Continuation } Action - scheduleEvery(TSHRTime interval /* milliseconds */, TSThreadPool tp = TS_THREAD_POOL_DEFAULT) + scheduleEvery(TSHRTime interval /* milliseconds */, TSThreadPool tp = TS_THREAD_POOL_NET) { - return TSContScheduleEvery(_cont, interval, tp); + return TSContScheduleEveryOnPool(_cont, interval, tp); } protected: diff --git a/iocore/eventsystem/I_Continuation.h b/iocore/eventsystem/I_Continuation.h index 140a11da841..abb1d7dc269 100644 --- a/iocore/eventsystem/I_Continuation.h +++ b/iocore/eventsystem/I_Continuation.h @@ -48,6 +48,7 @@ class EThread; class Event; extern EThread *this_ethread(); +extern EThread *this_event_thread(); ////////////////////////////////////////////////////////////////////////////// // @@ -144,6 +145,30 @@ class Continuation : private force_VFPT_to_top */ ContFlags control_flags; + EThread *thread_affinity = this_event_thread(); + + bool + setThreadAffinity(EThread *ethread) + { + if (ethread != nullptr) { + thread_affinity = ethread; + return true; + } + return false; + } + + EThread * + getThreadAffinity() + { + return thread_affinity; + } + + void + clearThreadAffinity() + { + thread_affinity = nullptr; + } + /** Receives the event code and data for an Event. diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index 8ce926ab1c8..f80456e6981 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -325,6 +325,8 @@ class EThread : public Thread bool is_event_type(EventType et); void set_event_type(EventType et); + bool has_event_loop = false; + // Private Interface void execute() override; diff --git a/iocore/eventsystem/P_UnixEThread.h b/iocore/eventsystem/P_UnixEThread.h index 9328c3b455c..042757bfb33 100644 --- a/iocore/eventsystem/P_UnixEThread.h +++ b/iocore/eventsystem/P_UnixEThread.h @@ -179,6 +179,17 @@ this_ethread() return (EThread *)this_thread(); } +TS_INLINE EThread * +this_event_thread() +{ + EThread *ethread = this_ethread(); + if (ethread != nullptr && ethread->has_event_loop) { + return ethread; + } else { + return nullptr; + } +} + TS_INLINE void EThread::free_event(Event *e) { diff --git a/iocore/eventsystem/P_UnixEventProcessor.h b/iocore/eventsystem/P_UnixEventProcessor.h index ed9772995f1..90f08b95165 100644 --- a/iocore/eventsystem/P_UnixEventProcessor.h +++ b/iocore/eventsystem/P_UnixEventProcessor.h @@ -65,7 +65,14 @@ TS_INLINE Event * EventProcessor::schedule(Event *e, EventType etype, bool fast_signal) { ink_assert(etype < MAX_EVENT_TYPES); - e->ethread = assign_thread(etype); + + EThread *ethread = e->continuation->getThreadAffinity(); + if (ethread != nullptr && ethread->is_event_type(etype)) { + e->ethread = ethread; + } else { + e->ethread = assign_thread(etype); + } + if (e->continuation->mutex) { e->mutex = e->continuation->mutex; } else { diff --git a/iocore/eventsystem/UnixEThread.cc b/iocore/eventsystem/UnixEThread.cc index 06920dbe5b7..e8586fa73b3 100644 --- a/iocore/eventsystem/UnixEThread.cc +++ b/iocore/eventsystem/UnixEThread.cc @@ -208,6 +208,7 @@ EThread::execute_regular() static EventMetrics METRIC_INIT; // give priority to immediate events + has_event_loop = true; for (;;) { if (unlikely(shutdown_event_system == true)) { return; diff --git a/plugins/background_fetch/background_fetch.cc b/plugins/background_fetch/background_fetch.cc index db1c875fd0c..394a3dca406 100644 --- a/plugins/background_fetch/background_fetch.cc +++ b/plugins/background_fetch/background_fetch.cc @@ -296,7 +296,7 @@ BgFetchData::schedule() resp_io_buf_reader = TSIOBufferReaderAlloc(resp_io_buf); // Schedule - TSContSchedule(_cont, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(_cont, 0, TS_THREAD_POOL_NET); } // Log format is: diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc index 9aa8d61b561..db00073057a 100644 --- a/plugins/esi/esi.cc +++ b/plugins/esi/esi.cc @@ -1012,7 +1012,7 @@ transformHandler(TSCont contp, TSEvent event, void *edata) // lock on our continuation which will fail if we destroy // ourselves right now TSDebug(cont_debug_tag, "[%s] Deferring shutdown as data event was just processed", __FUNCTION__); - TSContSchedule(contp, 10, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(contp, 10, TS_THREAD_POOL_TASK); } else { goto lShutdown; } diff --git a/plugins/experimental/certifier/certifier.cc b/plugins/experimental/certifier/certifier.cc index 9415d75f7b0..d2788f84cda 100644 --- a/plugins/experimental/certifier/certifier.cc +++ b/plugins/experimental/certifier/certifier.cc @@ -568,7 +568,7 @@ cert_retriever(TSCont contp, TSEvent event, void *edata) TSDebug(PLUGIN_NAME, "cert_retriever(): schedule thread to generate/retrieve cert for %s", servername); TSCont schedule_cont = TSContCreate(shadow_cert_generator, TSMutexCreate()); TSContDataSet(schedule_cont, (void *)servername); - TSContSchedule(schedule_cont, 0, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(schedule_cont, 0, TS_THREAD_POOL_TASK); } else { // Use existing context TSDebug(PLUGIN_NAME, "cert_retriever(): Reuse existing cert and context for %s", servername); diff --git a/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc b/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc index d9165bed6ff..28561e5242f 100644 --- a/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc +++ b/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc @@ -210,7 +210,7 @@ on_send_response_header(RequestData *req, TSHttpTxn &txnp, TSCont &contp) if (delay_request) { req->wl_retry++; TSDebug(DEBUG_TAG, "delaying request, url@%p: {{%s}} on retry: %d time", txnp, req->req_url.c_str(), req->wl_retry); - TSContSchedule(contp, OPEN_WRITE_FAIL_REQ_DELAY_TIMEOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(contp, OPEN_WRITE_FAIL_REQ_DELAY_TIMEOUT, TS_THREAD_POOL_TASK); TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); return TS_SUCCESS; } diff --git a/plugins/experimental/header_freq/header_freq.cc b/plugins/experimental/header_freq/header_freq.cc index 6de66200e30..d696a1d0c7c 100644 --- a/plugins/experimental/header_freq/header_freq.cc +++ b/plugins/experimental/header_freq/header_freq.cc @@ -194,7 +194,7 @@ handle_hook(TSCont contp, TSEvent event, void *edata) TSDebug(DEBUG_TAG_HOOK, "Scheduled execution of '%s' command", CONTROL_MSG_LOG); TSCont c = TSContCreate(CB_Command_Log, TSMutexCreate()); TSContDataSet(c, new std::string(static_cast(msgp->data), msgp->data_size)); - TSContSchedule(c, 0, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(c, 0, TS_THREAD_POOL_TASK); } else { TSError("[%s] Unknown command '%.*s'", PLUGIN_NAME, static_cast(msgp->data_size), static_cast(msgp->data)); diff --git a/plugins/experimental/inliner/ts.cc b/plugins/experimental/inliner/ts.cc index 3c332cc99bf..be9684e494c 100644 --- a/plugins/experimental/inliner/ts.cc +++ b/plugins/experimental/inliner/ts.cc @@ -123,7 +123,7 @@ namespace io assert(vio_ != nullptr); if (timeout_ > 0) { - action_ = TSContSchedule(continuation_, timeout_, TS_THREAD_POOL_DEFAULT); + action_ = TSContScheduleOnPool(continuation_, timeout_, TS_THREAD_POOL_NET); assert(action_ != nullptr); } } diff --git a/plugins/experimental/prefetch/fetch.cc b/plugins/experimental/prefetch/fetch.cc index ca7daf0faa7..d63e3af704f 100644 --- a/plugins/experimental/prefetch/fetch.cc +++ b/plugins/experimental/prefetch/fetch.cc @@ -596,7 +596,7 @@ BgFetch::schedule() /* Schedule */ PrefetchDebug("schedule fetch: %s", _url.c_str()); _startTime = TShrtime(); - TSContSchedule(_cont, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(_cont, 0, TS_THREAD_POOL_NET); } /* Log format is: name-space bytes status url */ diff --git a/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c b/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c index c49eb4043d0..fdcc4669d92 100644 --- a/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c +++ b/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c @@ -597,7 +597,7 @@ main_plugin(TSCont cont, TSEvent event, void *edata) fetch_cont = TSContCreate(fetch_resource, TSMutexCreate()); TSContDataSet(fetch_cont, (void *)state); - TSContSchedule(fetch_cont, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(fetch_cont, 0, TS_THREAD_POOL_NET); TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); } else if ((state->txn_start - chi->date) < (chi->max_age + chi->stale_on_error)) { TSDebug(PLUGIN_NAME, "Looks like we can return fresh data on 500 error"); @@ -609,7 +609,7 @@ main_plugin(TSCont cont, TSEvent event, void *edata) state->main_cont = cont; // we need this for the warning header callback. not sure i like it, but it works. fetch_cont = TSContCreate(fetch_resource, TSMutexCreate()); TSContDataSet(fetch_cont, (void *)state); - TSContSchedule(fetch_cont, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(fetch_cont, 0, TS_THREAD_POOL_NET); } else { TSDebug(PLUGIN_NAME, "No love? now: %d date: %d max-age: %d swr: %d soe: %d", (int)state->txn_start, (int)chi->date, (int)chi->max_age, (int)chi->stale_while_revalidate, (int)chi->stale_on_error); diff --git a/plugins/experimental/system_stats/system_stats.c b/plugins/experimental/system_stats/system_stats.c index d25dada7d6f..44c91b87982 100644 --- a/plugins/experimental/system_stats/system_stats.c +++ b/plugins/experimental/system_stats/system_stats.c @@ -224,7 +224,7 @@ systemStatsContCB(TSCont cont, TSEvent event ATS_UNUSED, void *edata) stat_creation_mutex = TSContMutexGet(cont); getStats(stat_creation_mutex); - TSContSchedule(cont, SYSTEM_STATS_TIMEOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(cont, SYSTEM_STATS_TIMEOUT, TS_THREAD_POOL_TASK); TSDebug(DEBUG_TAG, "finished %s", __FUNCTION__); return 0; @@ -253,7 +253,7 @@ TSPluginInit(int argc, const char *argv[]) // We want our first hit immediate to populate the stats, // Subsequent schedules done within the function will be for // 5 seconds. - TSContSchedule(stats_cont, 0, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(stats_cont, 0, TS_THREAD_POOL_TASK); TSDebug(DEBUG_TAG, "Init complete"); } diff --git a/plugins/generator/generator.cc b/plugins/generator/generator.cc index 5f282cd6891..8eee22f5cda 100644 --- a/plugins/generator/generator.cc +++ b/plugins/generator/generator.cc @@ -500,7 +500,7 @@ GeneratorInterceptionHook(TSCont contp, TSEvent event, void *edata) if (cdata.grq->delay > 0) { VDEBUG("delaying response by %ums", cdata.grq->delay); - TSContSchedule(contp, cdata.grq->delay, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp, cdata.grq->delay, TS_THREAD_POOL_NET); return TS_EVENT_NONE; } diff --git a/plugins/lua/ts_lua_http_intercept.c b/plugins/lua/ts_lua_http_intercept.c index 048f52d9182..d794231c6c5 100644 --- a/plugins/lua/ts_lua_http_intercept.c +++ b/plugins/lua/ts_lua_http_intercept.c @@ -406,7 +406,7 @@ ts_lua_flush_wakeup(ts_lua_http_intercept_ctx *ictx) ci = &ictx->cinfo; contp = TSContCreate(ts_lua_flush_wakeup_handler, ci->mutex); - action = TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + action = TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); ai = ts_lua_async_create_item(contp, ts_lua_flush_cleanup, (void *)action, ci); TSContDataSet(contp, ai); diff --git a/plugins/lua/ts_lua_misc.c b/plugins/lua/ts_lua_misc.c index 8da3a160aa0..e12e5a1fb1b 100644 --- a/plugins/lua/ts_lua_misc.c +++ b/plugins/lua/ts_lua_misc.c @@ -211,7 +211,7 @@ ts_lua_schedule(lua_State *L) nci->contp = contp; nci->mutex = ci->mutex; - TSContSchedule(contp, sec * 1000, entry); + TSContScheduleOnPool(contp, sec * 1000, entry); return 0; } @@ -287,7 +287,7 @@ ts_lua_sleep(lua_State *L) } contp = TSContCreate(ts_lua_sleep_handler, ci->mutex); - action = TSContSchedule(contp, sec * 1000, TS_THREAD_POOL_DEFAULT); + action = TSContScheduleOnPool(contp, sec * 1000, TS_THREAD_POOL_NET); ai = ts_lua_async_create_item(contp, ts_lua_sleep_cleanup, (void *)action, ci); TSContDataSet(contp, ai); diff --git a/plugins/regex_revalidate/regex_revalidate.c b/plugins/regex_revalidate/regex_revalidate.c index 0a697d73588..d7ff7065639 100644 --- a/plugins/regex_revalidate/regex_revalidate.c +++ b/plugins/regex_revalidate/regex_revalidate.c @@ -366,7 +366,7 @@ config_handler(TSCont cont, TSEvent event, void *edata) if (iptr) { free_cont = TSContCreate(free_handler, TSMutexCreate()); TSContDataSet(free_cont, (void *)iptr); - TSContSchedule(free_cont, FREE_TMOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(free_cont, FREE_TMOUT, TS_THREAD_POOL_TASK); } } else { TSDebug(LOG_PREFIX, "No Changes"); @@ -379,7 +379,7 @@ config_handler(TSCont cont, TSEvent event, void *edata) // Don't reschedule for TS_EVENT_MGMT_UPDATE if (event == TS_EVENT_TIMEOUT) { - TSContSchedule(cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); } return 0; } @@ -562,7 +562,7 @@ TSPluginInit(int argc, const char *argv[]) TSMgmtUpdateRegister(config_cont, LOG_PREFIX); if (!disable_timed_reload) { - TSContSchedule(config_cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(config_cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); } TSDebug(LOG_PREFIX, "Plugin Init Complete"); diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index b92e42c19d3..55a8c132185 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -4399,16 +4399,45 @@ TSContDataGet(TSCont contp) } TSAction -TSContSchedule(TSCont contp, ink_hrtime timeout, TSThreadPool tp) +TSContSchedule(TSCont contp, TSHRTime timeout) { sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); FORCE_PLUGIN_SCOPED_MUTEX(contp); - INKContInternal *i = (INKContInternal *)contp; + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { + ink_assert(!"not reached"); + } + + EThread *eth = i->getThreadAffinity(); + if (eth == nullptr) { + return nullptr; + } + TSAction action; + if (timeout == 0) { + action = reinterpret_cast(eth->schedule_imm(i)); + } else { + action = reinterpret_cast(eth->schedule_in(i, HRTIME_MSECONDS(timeout))); + } - if (ink_atomic_increment((int *)&i->m_event_count, 1) < 0) { + /* This is a hack. Should be handled in ink_types */ + action = (TSAction)((uintptr_t)action | 0x1); + return action; +} + +TSAction +TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { ink_assert(!"not reached"); } @@ -4416,7 +4445,6 @@ TSContSchedule(TSCont contp, ink_hrtime timeout, TSThreadPool tp) switch (tp) { case TS_THREAD_POOL_NET: - case TS_THREAD_POOL_DEFAULT: etype = ET_NET; break; case TS_THREAD_POOL_TASK: @@ -4439,28 +4467,82 @@ TSContSchedule(TSCont contp, ink_hrtime timeout, TSThreadPool tp) break; } + TSAction action; if (timeout == 0) { action = reinterpret_cast(eventProcessor.schedule_imm(i, etype)); } else { action = reinterpret_cast(eventProcessor.schedule_in(i, HRTIME_MSECONDS(timeout), etype)); } - /* This is a hack. SHould be handled in ink_types */ + /* This is a hack. Should be handled in ink_types */ action = (TSAction)((uintptr_t)action | 0x1); return action; } TSAction -TSContScheduleEvery(TSCont contp, ink_hrtime every, TSThreadPool tp) +TSContScheduleOnThread(TSCont contp, TSHRTime timeout, TSEventThread ethread) { + ink_release_assert(ethread != nullptr); + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); FORCE_PLUGIN_SCOPED_MUTEX(contp); - INKContInternal *i = (INKContInternal *)contp; + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { + ink_assert(!"not reached"); + } + + EThread *eth = reinterpret_cast(ethread); + TSAction action; + if (timeout == 0) { + action = reinterpret_cast(eth->schedule_imm(i)); + } else { + action = reinterpret_cast(eth->schedule_in(i, HRTIME_MSECONDS(timeout))); + } - if (ink_atomic_increment((int *)&i->m_event_count, 1) < 0) { + /* This is a hack. Should be handled in ink_types */ + action = (TSAction)((uintptr_t)action | 0x1); + return action; +} + +TSAction +TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { + ink_assert(!"not reached"); + } + + EThread *eth = i->getThreadAffinity(); + if (eth == nullptr) { + return nullptr; + } + + TSAction action = reinterpret_cast(eth->schedule_every(i, HRTIME_MSECONDS(every))); + + /* This is a hack. Should be handled in ink_types */ + action = (TSAction)((uintptr_t)action | 0x1); + return action; +} + +TSAction +TSContScheduleEveryOnPool(TSCont contp, TSHRTime every, TSThreadPool tp) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { ink_assert(!"not reached"); } @@ -4468,7 +4550,6 @@ TSContScheduleEvery(TSCont contp, ink_hrtime every, TSThreadPool tp) switch (tp) { case TS_THREAD_POOL_NET: - case TS_THREAD_POOL_DEFAULT: etype = ET_NET; break; case TS_THREAD_POOL_TASK: @@ -4479,15 +4560,81 @@ TSContScheduleEvery(TSCont contp, ink_hrtime every, TSThreadPool tp) break; } - action = reinterpret_cast(eventProcessor.schedule_every(i, HRTIME_MSECONDS(every), etype)); + TSAction action = reinterpret_cast(eventProcessor.schedule_every(i, HRTIME_MSECONDS(every), etype)); + + /* This is a hack. Should be handled in ink_types */ + action = (TSAction)((uintptr_t)action | 0x1); + return action; +} + +TSAction +TSContScheduleEveryOnThread(TSCont contp, TSHRTime every /* millisecs */, TSEventThread ethread) +{ + ink_release_assert(ethread != nullptr); + + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { + ink_assert(!"not reached"); + } + + EThread *eth = reinterpret_cast(ethread); + + TSAction action = reinterpret_cast(eth->schedule_every(i, HRTIME_MSECONDS(every))); - /* This is a hack. SHould be handled in ink_types */ + /* This is a hack. Should be handled in ink_types */ action = (TSAction)((uintptr_t)action | 0x1); return action; } +TSReturnCode +TSContThreadAffinitySet(TSCont contp, TSEventThread ethread) +{ + ink_release_assert(ethread != nullptr); + + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + EThread *thread_affinity = reinterpret_cast(ethread); + + if (i->setThreadAffinity(thread_affinity)) { + return TS_SUCCESS; + } + return TS_ERROR; +} + +TSEventThread +TSContThreadAffinityGet(TSCont contp) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + return reinterpret_cast(i->getThreadAffinity()); +} + +void +TSContThreadAffinityClear(TSCont contp) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + i->clearThreadAffinity(); +} + TSAction -TSHttpSchedule(TSCont contp, TSHttpTxn txnp, ink_hrtime timeout) +TSHttpSchedule(TSCont contp, TSHttpTxn txnp, TSHRTime timeout) { sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); @@ -6412,7 +6559,7 @@ TSActionCancel(TSAction actionp) Action *a; INKContInternal *i; - /* This is a hack. SHould be handled in ink_types */ + /* This is a hack. Should be handled in ink_types */ if ((uintptr_t)actionp & 0x1) { a = (Action *)((uintptr_t)actionp - 1); i = (INKContInternal *)a->continuation; diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 811f1beefe2..dcf7a84b6b7 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -1882,7 +1882,7 @@ cache_handler(TSCont contp, TSEvent event, void *data) // now waiting for 100ms to make sure the key is // written in directory remove the content - TSContSchedule(contp, 100, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 100, TS_THREAD_POOL_NET); } return 1; @@ -2463,7 +2463,7 @@ REGRESSION_TEST(SDK_API_TSActionCancel)(RegressionTest *test, int /* atype ATS_U TSMutex cont_mutex = TSMutexCreate(); TSCont contp = TSContCreate(action_cancel_handler, cont_mutex); - TSAction actionp = TSContSchedule(contp, 10000, TS_THREAD_POOL_DEFAULT); + TSAction actionp = TSContScheduleOnPool(contp, 10000, TS_THREAD_POOL_NET); TSMutexLock(cont_mutex); if (TSActionDone(actionp)) { @@ -2475,7 +2475,7 @@ REGRESSION_TEST(SDK_API_TSActionCancel)(RegressionTest *test, int /* atype ATS_U } TSMutexUnlock(cont_mutex); - TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); } ////////////////////////////////////////////// @@ -2594,7 +2594,7 @@ REGRESSION_TEST(SDK_API_TSContDataGet)(RegressionTest *test, int /* atype ATS_UN TSContDataSet(contp, (void *)my_data); - TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); } ////////////////////////////////////////////// @@ -2633,7 +2633,7 @@ REGRESSION_TEST(SDK_API_TSContMutexGet)(RegressionTest *test, int /* atype ATS_U ////////////////////////////////////////////// // SDK_API_TSCont // -// Unit Test for API: TSContSchedule +// Unit Test for API: TSContScheduleOnPool ////////////////////////////////////////////// // this is needed for asynchronous APIs @@ -2649,15 +2649,15 @@ cont_schedule_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */ { if (event == TS_EVENT_IMMEDIATE) { // Test Case 1 - SDK_RPRINT(SDK_ContSchedule_test, "TSContSchedule", "TestCase1", TC_PASS, "ok"); + SDK_RPRINT(SDK_ContSchedule_test, "TSContScheduleOnPool", "TestCase1", TC_PASS, "ok"); tc1_count++; } else if (event == TS_EVENT_TIMEOUT) { // Test Case 2 - SDK_RPRINT(SDK_ContSchedule_test, "TSContSchedule", "TestCase2", TC_PASS, "ok"); + SDK_RPRINT(SDK_ContSchedule_test, "TSContScheduleOnPool", "TestCase2", TC_PASS, "ok"); tc2_count++; } else { // If we receive a bad event, it's a failure - SDK_RPRINT(SDK_ContSchedule_test, "TSContSchedule", "TestCase1|2", TC_FAIL, "received unexpected event number %d", event); + SDK_RPRINT(SDK_ContSchedule_test, "TSContScheduleOnPool", "TestCase1|2", TC_FAIL, "received unexpected event number %d", event); *SDK_ContSchedule_pstatus = REGRESSION_TEST_FAILED; return 0; } @@ -3035,10 +3035,10 @@ REGRESSION_TEST(SDK_API_TSContSchedule)(RegressionTest *test, int /* atype ATS_U TSCont contp2 = TSContCreate(cont_schedule_handler, TSMutexCreate()); // Test Case 1: schedule immediate - TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); // Test Case 2: schedule in 10ms - TSContSchedule(contp2, 10, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp2, 10, TS_THREAD_POOL_NET); } ////////////////////////////////////////////////////////////////////////////// @@ -3507,7 +3507,7 @@ mytest_handler(TSCont contp, TSEvent event, void *data) case TS_EVENT_TIMEOUT: /* Browser still waiting the response ? */ if (test->browser->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); } /* Browser got the response. test is over. clean up */ else { @@ -3600,7 +3600,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpHookAdd)(RegressionTest *test, int /* atyp /* Wait until transaction is done */ if (socktest->browser->status == REQUEST_INPROGRESS) { - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); } return; @@ -6443,7 +6443,7 @@ REGRESSION_TEST(SDK_API_TSTextLog)(RegressionTest *test, int /* atype ATS_UNUSED data->log = log; TSContDataSet(log_test_cont, data); - TSContSchedule(log_test_cont, 6000, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(log_test_cont, 6000, TS_THREAD_POOL_NET); return; } @@ -7000,7 +7000,7 @@ ssn_handler(TSCont contp, TSEvent event, void *edata) case TS_EVENT_TIMEOUT: /* Browser still waiting the response ? */ if (data->browser->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); } /* Browser got the response. test is over. clean up */ else { @@ -7085,7 +7085,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpSsn)(RegressionTest *test, int /* atype AT /* Wait until transaction is done */ if (socktest->browser->status == REQUEST_INPROGRESS) { - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); } return; @@ -7244,13 +7244,13 @@ parent_proxy_handler(TSCont contp, TSEvent event, void *edata) if (ptest->configured) { // If we are still in progress, reschedule. rprintf(ptest->regtest, "waiting for response\n"); - TSContSchedule(contp, 100, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 100, TS_THREAD_POOL_NET); break; } if (!ptest->parent_routing_enabled()) { rprintf(ptest->regtest, "waiting for configuration\n"); - TSContSchedule(contp, 100, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 100, TS_THREAD_POOL_NET); break; } @@ -7330,7 +7330,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpParentProxySet_Fail)(RegressionTest *test, ptest->os = synserver_create(SYNSERVER_LISTEN_PORT, TSContCreate(synserver_vc_refuse, TSMutexCreate())); synserver_start(ptest->os); - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); } EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpParentProxySet_Success)(RegressionTest *test, int level, int *pstatus) @@ -7362,7 +7362,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpParentProxySet_Success)(RegressionTest *te ptest->os = synserver_create(SYNSERVER_LISTEN_PORT, TSContCreate(synserver_vc_accept, TSMutexCreate())); synserver_start(ptest->os); - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); } ///////////////////////////////////////////////////// @@ -7485,12 +7485,12 @@ cache_hook_handler(TSCont contp, TSEvent event, void *edata) /* Browser still waiting the response ? */ if (data->first_time == true) { if (data->browser1->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } } else { if (data->browser2->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } } @@ -7508,7 +7508,7 @@ cache_hook_handler(TSCont contp, TSEvent event, void *edata) /* Send another similar client request */ synclient_txn_send_request(data->browser2, data->request); ink_assert(REQUEST_INPROGRESS == data->browser2->status); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } @@ -7577,7 +7577,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpTxnCache)(RegressionTest *test, int /* aty synclient_txn_send_request(socktest->browser1, socktest->request); /* Wait until transaction is done */ - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); return; } @@ -7951,37 +7951,37 @@ transform_hook_handler(TSCont contp, TSEvent event, void *edata) switch (data->req_no) { case 1: if (data->browser1->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } data->req_no++; Debug(UTDBG_TAG "_transform", "Running Browser 2"); synclient_txn_send_request(data->browser2, data->request2); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; case 2: if (data->browser2->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } data->req_no++; Debug(UTDBG_TAG "_transform", "Running Browser 3"); synclient_txn_send_request(data->browser3, data->request1); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; case 3: if (data->browser3->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } data->req_no++; Debug(UTDBG_TAG "_transform", "Running Browser 4"); synclient_txn_send_request(data->browser4, data->request2); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; case 4: if (data->browser4->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } synserver_delete(data->os); @@ -8120,7 +8120,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpTxnTransform)(RegressionTest *test, int /* // synclient_txn_send_request(socktest->browser2, socktest->request2); /* Wait until transaction is done */ - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); return; } @@ -8232,12 +8232,12 @@ altinfo_hook_handler(TSCont contp, TSEvent event, void *edata) /* Browser still waiting the response ? */ if (data->first_time == true) { if ((data->browser1->status == REQUEST_INPROGRESS) || (data->browser2->status == REQUEST_INPROGRESS)) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } } else { if (data->browser3->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } } @@ -8257,7 +8257,7 @@ altinfo_hook_handler(TSCont contp, TSEvent event, void *edata) /* Register to HTTP hooks that are called in case of alternate selection */ TSHttpHookAdd(TS_HTTP_SELECT_ALT_HOOK, contp); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } @@ -8336,7 +8336,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpAltInfo)(RegressionTest *test, int /* atyp synclient_txn_send_request(socktest->browser2, socktest->request2); /* Wait until transaction is done */ - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); return; } @@ -8425,7 +8425,7 @@ cont_test_handler(TSCont contp, TSEvent event, void *edata) /* Browser still waiting the response ? */ if (data->browser->status == REQUEST_INPROGRESS) { TSDebug(UTDBG_TAG, "Browser still waiting response..."); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); } /* Browser got the response */ else { @@ -8517,7 +8517,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_TSHttpConnectIntercept)(RegressionTest *test, synclient_txn_send_request_to_vc(data->browser, data->request, data->vc); /* Wait until transaction is done */ - TSContSchedule(cont_test, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont_test, 25, TS_THREAD_POOL_NET); return; } @@ -8556,7 +8556,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_TSHttpConnectServerIntercept)(RegressionTest * synclient_txn_send_request_to_vc(data->browser, data->request, data->vc); /* Wait until transaction is done */ - TSContSchedule(cont_test, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont_test, 25, TS_THREAD_POOL_NET); return; } diff --git a/src/traffic_server/InkIOCoreAPI.cc b/src/traffic_server/InkIOCoreAPI.cc index d3dc3bf35b9..221aa368ee5 100644 --- a/src/traffic_server/InkIOCoreAPI.cc +++ b/src/traffic_server/InkIOCoreAPI.cc @@ -227,6 +227,12 @@ TSThreadSelf(void) return ithread; } +TSEventThread +TSEventThreadSelf(void) +{ + return reinterpret_cast(this_event_thread()); +} + //////////////////////////////////////////////////////////////////// // // Mutexes diff --git a/src/tscpp/api/AsyncTimer.cc b/src/tscpp/api/AsyncTimer.cc index 9fab335b870..9387409cb6c 100644 --- a/src/tscpp/api/AsyncTimer.cc +++ b/src/tscpp/api/AsyncTimer.cc @@ -56,7 +56,7 @@ handleTimerEvent(TSCont cont, TSEvent event, void *edata) state->initial_timer_action_ = nullptr; // mark it so that it won't be canceled later on if (state->type_ == AsyncTimer::TYPE_PERIODIC) { LOG_DEBUG("Scheduling periodic event now"); - state->periodic_timer_action_ = TSContScheduleEvery(state->cont_, state->period_in_ms_, state->thread_pool_); + state->periodic_timer_action_ = TSContScheduleEveryOnPool(state->cont_, state->period_in_ms_, state->thread_pool_); } } if (!state->dispatch_controller_->dispatch()) { @@ -88,10 +88,10 @@ AsyncTimer::run() } if (one_off_timeout_in_ms) { LOG_DEBUG("Scheduling initial/one-off event"); - state_->initial_timer_action_ = TSContSchedule(state_->cont_, one_off_timeout_in_ms, state_->thread_pool_); + state_->initial_timer_action_ = TSContScheduleOnPool(state_->cont_, one_off_timeout_in_ms, state_->thread_pool_); } else if (regular_timeout_in_ms) { LOG_DEBUG("Scheduling regular timer events"); - state_->periodic_timer_action_ = TSContScheduleEvery(state_->cont_, regular_timeout_in_ms, state_->thread_pool_); + state_->periodic_timer_action_ = TSContScheduleEveryOnPool(state_->cont_, regular_timeout_in_ms, state_->thread_pool_); } } diff --git a/src/tscpp/api/InterceptPlugin.cc b/src/tscpp/api/InterceptPlugin.cc index db72a59363d..8ccc5a2ac4c 100644 --- a/src/tscpp/api/InterceptPlugin.cc +++ b/src/tscpp/api/InterceptPlugin.cc @@ -359,7 +359,7 @@ handleEvents(TSCont cont, TSEvent pristine_event, void *pristine_edata) state->saved_event_ = event; state->saved_edata_ = edata; } - state->timeout_action_ = TSContSchedule(cont, 1, TS_THREAD_POOL_DEFAULT); + state->timeout_action_ = TSContScheduleOnPool(cont, 1, TS_THREAD_POOL_NET); return 0; } if (event == TS_EVENT_TIMEOUT) { // we have a saved event to restore diff --git a/tests/gold_tests/cont_schedule/gold/http_200.gold b/tests/gold_tests/cont_schedule/gold/http_200.gold new file mode 100644 index 00000000000..f3752f16420 --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/http_200.gold @@ -0,0 +1,9 @@ +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< +`` diff --git a/tests/gold_tests/cont_schedule/gold/schedule.gold b/tests/gold_tests/cont_schedule/gold/schedule.gold new file mode 100644 index 00000000000..7b5d1f6a2b2 --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/schedule.gold @@ -0,0 +1,4 @@ +`` +``(TSContSchedule_test.check) pass [should be the same thread] +``(TSContSchedule_test.check) pass [should not be the same thread] +`` diff --git a/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold b/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold new file mode 100644 index 00000000000..42328e0dcd2 --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold @@ -0,0 +1,9 @@ +`` +``ET_NET``TSContScheduleOnPool handler 1`` +``ET_NET``TSContScheduleOnPool handler 1`` +``(TSContSchedule_test.check) pass [should not be the same thread] +``ET_TASK``TSContScheduleOnPool handler 2`` +``(TSContSchedule_test.check) pass [should not be the same thread] +``ET_TASK``TSContScheduleOnPool handler 2`` +``(TSContSchedule_test.check) pass [should not be the same thread] +`` diff --git a/tests/gold_tests/cont_schedule/gold/schedule_on_thread.gold b/tests/gold_tests/cont_schedule/gold/schedule_on_thread.gold new file mode 100644 index 00000000000..7b5d1f6a2b2 --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/schedule_on_thread.gold @@ -0,0 +1,4 @@ +`` +``(TSContSchedule_test.check) pass [should be the same thread] +``(TSContSchedule_test.check) pass [should not be the same thread] +`` diff --git a/tests/gold_tests/cont_schedule/gold/thread_affinity.gold b/tests/gold_tests/cont_schedule/gold/thread_affinity.gold new file mode 100644 index 00000000000..b81b4f5b76d --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/thread_affinity.gold @@ -0,0 +1,5 @@ +`` +``pass [affinity thread is null] +``pass [affinity thread is set] +``pass [affinity thread is cleared] +`` diff --git a/tests/gold_tests/cont_schedule/schedule.test.py b/tests/gold_tests/cont_schedule/schedule.test.py new file mode 100644 index 00000000000..e2867ef1fd3 --- /dev/null +++ b/tests/gold_tests/cont_schedule/schedule.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +Test.Summary = 'Test TSContSchedule API' +Test.SkipUnless(Condition.HasProgram('curl', 'Curl need to be installed on system for this test to work')) + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess('ts') +server = Test.MakeOriginServer('server') + +Test.testName = '' +request_header = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.5, + 'proxy.config.exec_thread.limit': 32, + 'proxy.config.accept_threads': 1, + 'proxy.config.task_threads': 2, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'TSContSchedule_test' +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Load plugin +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts) + +# www.example.com Host +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Streams.stderr = 'gold/http_200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Check Plugin Results +ts.Streams.All = "gold/schedule.gold" +ts.Streams.All += Testers.ExcludesExpression('fail', 'should not contain "fail"') diff --git a/tests/gold_tests/cont_schedule/schedule_on_pool.test.py b/tests/gold_tests/cont_schedule/schedule_on_pool.test.py new file mode 100644 index 00000000000..9fb54197b36 --- /dev/null +++ b/tests/gold_tests/cont_schedule/schedule_on_pool.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +Test.Summary = 'Test TSContScheduleOnPool API' +Test.SkipUnless(Condition.HasProgram('curl', 'Curl need to be installed on system for this test to work')) + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess('ts') +server = Test.MakeOriginServer('server') + +Test.testName = '' +request_header = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.5, + 'proxy.config.exec_thread.limit': 32, + 'proxy.config.accept_threads': 1, + 'proxy.config.task_threads': 2, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'TSContSchedule_test' +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Load plugin +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'pool') + +# www.example.com Host +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Streams.stderr = 'gold/http_200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Check Plugin Results +ts.Streams.All = "gold/schedule_on_pool.gold" +ts.Streams.All += Testers.ExcludesExpression('fail', 'should not contain "fail"') diff --git a/tests/gold_tests/cont_schedule/schedule_on_thread.test.py b/tests/gold_tests/cont_schedule/schedule_on_thread.test.py new file mode 100644 index 00000000000..5b5ab5f69d7 --- /dev/null +++ b/tests/gold_tests/cont_schedule/schedule_on_thread.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +Test.Summary = 'Test TSContScheduleOnThread API' +Test.SkipUnless(Condition.HasProgram('curl', 'Curl need to be installed on system for this test to work')) + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess('ts') +server = Test.MakeOriginServer('server') + +Test.testName = '' +request_header = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.5, + 'proxy.config.exec_thread.limit': 32, + 'proxy.config.accept_threads': 1, + 'proxy.config.task_threads': 2, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'TSContSchedule_test' +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Load plugin +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'thread') + +# www.example.com Host +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Streams.stderr = 'gold/http_200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Check Plugin Results +ts.Streams.All = "gold/schedule_on_thread.gold" +ts.Streams.All += Testers.ExcludesExpression('fail', 'should not contain "fail"') diff --git a/tests/gold_tests/cont_schedule/thread_affinity.test.py b/tests/gold_tests/cont_schedule/thread_affinity.test.py new file mode 100644 index 00000000000..8b91d5ac231 --- /dev/null +++ b/tests/gold_tests/cont_schedule/thread_affinity.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +Test.Summary = 'Test TSContThreadAffinity APIs' +Test.SkipUnless(Condition.HasProgram('curl', 'Curl need to be installed on system for this test to work')) + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess('ts') +server = Test.MakeOriginServer('server') + +Test.testName = '' +request_header = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.5, + 'proxy.config.exec_thread.limit': 32, + 'proxy.config.accept_threads': 1, + 'proxy.config.task_threads': 2, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'TSContSchedule_test' +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Load plugin +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'affinity') + +# www.example.com Host +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Streams.stderr = 'gold/http_200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Check Plugin Results +ts.Streams.All = "gold/thread_affinity.gold" +ts.Streams.All += Testers.ExcludesExpression('fail', 'should not contain "fail"') diff --git a/tests/tools/plugins/cont_schedule.cc b/tests/tools/plugins/cont_schedule.cc new file mode 100644 index 00000000000..accf3db958a --- /dev/null +++ b/tests/tools/plugins/cont_schedule.cc @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include // for abort +#include // for debug + +// debug messages viewable by setting 'proxy.config.diags.debug.tags' +// in 'records.config' + +// debug messages during one-time initialization +static const char DEBUG_TAG_INIT[] = "TSContSchedule_test.init"; +static const char DEBUG_TAG_SCHD[] = "TSContSchedule_test.schedule"; +static const char DEBUG_TAG_HDL[] = "TSContSchedule_test.handler"; +static const char DEBUG_TAG_CHK[] = "TSContSchedule_test.check"; + +// plugin registration info +static char plugin_name[] = "TSContSchedule_test"; +static char vendor_name[] = "apache"; +static char support_email[] = "duke8253@apache.org"; + +static int test_flag = 0; + +static TSEventThread test_thread = nullptr; +static TSThread check_thread = nullptr; + +static int TSContSchedule_handler_1(TSCont contp, TSEvent event, void *edata); +static int TSContSchedule_handler_2(TSCont contp, TSEvent event, void *edata); +static int TSContScheduleOnPool_handler_1(TSCont contp, TSEvent event, void *edata); +static int TSContScheduleOnPool_handler_2(TSCont contp, TSEvent event, void *edata); +static int TSContScheduleOnThread_handler_1(TSCont contp, TSEvent event, void *edata); +static int TSContScheduleOnThread_handler_2(TSCont contp, TSEvent event, void *edata); +static int TSContThreadAffinity_handler(TSCont contp, TSEvent event, void *edata); + +static int +TSContSchedule_handler_1(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContSchedule handler 1 thread [%p]", TSThreadSelf()); + if (test_thread == nullptr) { + test_thread = TSEventThreadSelf(); + + TSCont contp_new = TSContCreate(TSContSchedule_handler_2, TSMutexCreate()); + + if (contp_new == nullptr) { + TSDebug(DEBUG_TAG_HDL, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_HDL, "[%s] scheduling continuation", plugin_name); + TSContThreadAffinitySet(contp_new, test_thread); + TSContSchedule(contp_new, 0); + TSContSchedule(contp_new, 100); + } + } else if (check_thread == nullptr) { + TSDebug(DEBUG_TAG_CHK, "fail [schedule delay not applied]"); + } else { + if (check_thread != TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should not be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); + } + } + return 0; +} + +static int +TSContSchedule_handler_2(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContSchedule handler 2 thread [%p]", TSThreadSelf()); + if (check_thread == nullptr) { + check_thread = TSThreadSelf(); + } else if (check_thread == TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [not the same thread]"); + } + return 0; +} + +static int +TSContScheduleOnPool_handler_1(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContScheduleOnPool handler 1 thread [%p]", TSThreadSelf()); + if (check_thread == nullptr) { + check_thread = TSThreadSelf(); + } else { + if (check_thread != TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should not be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); + } + check_thread = TSThreadSelf(); + } + return 0; +} + +static int +TSContScheduleOnPool_handler_2(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContScheduleOnPool handler 2 thread [%p]", TSThreadSelf()); + if (check_thread == nullptr) { + check_thread = TSThreadSelf(); + } else { + if (check_thread != TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should not be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); + } + check_thread = TSThreadSelf(); + } + return 0; +} + +static int +TSContScheduleOnThread_handler_1(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContScheduleOnThread handler 1 thread [%p]", TSThreadSelf()); + if (test_thread == nullptr) { + test_thread = TSEventThreadSelf(); + + TSCont contp_new = TSContCreate(TSContScheduleOnThread_handler_2, TSMutexCreate()); + + if (contp_new == nullptr) { + TSDebug(DEBUG_TAG_HDL, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_HDL, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnThread(contp_new, 0, test_thread); + TSContScheduleOnThread(contp_new, 100, test_thread); + } + } else if (check_thread == nullptr) { + TSDebug(DEBUG_TAG_CHK, "fail [schedule delay not applied]"); + } else { + if (check_thread != TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should not be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); + } + } + return 0; +} + +static int +TSContScheduleOnThread_handler_2(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContScheduleOnThread handler 2 thread [%p]", TSThreadSelf()); + if (check_thread == nullptr) { + check_thread = TSThreadSelf(); + } else if (check_thread == TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [not the same thread]"); + } + return 0; +} + +static int +TSContThreadAffinity_handler(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContThreadAffinity handler thread [%p]", TSThreadSelf()); + + test_thread = TSEventThreadSelf(); + + if (TSContThreadAffinityGet(contp) == nullptr) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is null]"); + TSContThreadAffinitySet(contp, TSEventThreadSelf()); + if (TSContThreadAffinityGet(contp) == test_thread) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is set]"); + TSContThreadAffinityClear(contp); + if (TSContThreadAffinityGet(contp) == nullptr) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is cleared]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not cleared]"); + } + } else { + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not set]"); + } + } else { + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not null]"); + } + + return 0; +} + +void +TSContSchedule_test() +{ + TSCont contp = TSContCreate(TSContSchedule_handler_1, TSMutexCreate()); + + if (contp == nullptr) { + TSDebug(DEBUG_TAG_SCHD, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp, 200, TS_THREAD_POOL_NET); + } +} + +void +TSContScheduleOnPool_test() +{ + TSCont contp_1 = TSContCreate(TSContScheduleOnPool_handler_1, TSMutexCreate()); + TSCont contp_2 = TSContCreate(TSContScheduleOnPool_handler_2, TSMutexCreate()); + + if (contp_1 == nullptr || contp_2 == nullptr) { + TSDebug(DEBUG_TAG_SCHD, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnPool(contp_1, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp_1, 100, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp_2, 200, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(contp_2, 300, TS_THREAD_POOL_TASK); + } +} + +void +TSContScheduleOnThread_test() +{ + TSCont contp = TSContCreate(TSContScheduleOnThread_handler_1, TSMutexCreate()); + + if (contp == nullptr) { + TSDebug(DEBUG_TAG_SCHD, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp, 200, TS_THREAD_POOL_NET); + } +} + +void +TSContThreadAffinity_test() +{ + TSCont contp = TSContCreate(TSContThreadAffinity_handler, TSMutexCreate()); + + if (contp == nullptr) { + TSDebug(DEBUG_TAG_SCHD, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + } +} + +static int +LifecycleHookTracer(TSCont contp, TSEvent event, void *edata) +{ + if (event == TS_EVENT_LIFECYCLE_TASK_THREADS_READY) { + switch (test_flag) { + case 1: + TSContSchedule_test(); + break; + case 2: + TSContScheduleOnPool_test(); + break; + case 3: + TSContScheduleOnThread_test(); + break; + case 4: + TSContThreadAffinity_test(); + break; + default: + break; + } + } + return 0; +} + +void +TSPluginInit(int argc, const char *argv[]) +{ + if (argc == 1) { + TSDebug(DEBUG_TAG_INIT, "initializing plugin for testing TSContSchedule"); + test_flag = 1; + } else if (argc == 2) { + int len = strlen(argv[1]); + if (len == 4 && strncmp(argv[1], "pool", 4) == 0) { + TSDebug(DEBUG_TAG_INIT, "initializing plugin for testing TSContScheduleOnPool"); + test_flag = 2; + } else if (len == 6 && strncmp(argv[1], "thread", 6) == 0) { + TSDebug(DEBUG_TAG_INIT, "initializing plugin for testing TSContScheduleOnThread"); + test_flag = 3; + } else if (len == 8 && strncmp(argv[1], "affinity", 8) == 0) { + TSDebug(DEBUG_TAG_INIT, "initializing plugin for testing TSContThreadAffinity"); + test_flag = 4; + } else { + goto Lerror; + } + } else { + goto Lerror; + } + + TSPluginRegistrationInfo info; + + info.plugin_name = plugin_name; + info.vendor_name = vendor_name; + info.support_email = support_email; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSDebug(DEBUG_TAG_INIT, "[%s] plugin registration failed", plugin_name); + abort(); + } + + TSLifecycleHookAdd(TS_LIFECYCLE_TASK_THREADS_READY_HOOK, TSContCreate(LifecycleHookTracer, TSMutexCreate())); + + return; + +Lerror: + TSDebug(DEBUG_TAG_INIT, "[%s] plugin invalid argument", plugin_name); + abort(); +} diff --git a/tests/tools/plugins/continuations_verify.cc b/tests/tools/plugins/continuations_verify.cc index d4529684610..079c0495b28 100644 --- a/tests/tools/plugins/continuations_verify.cc +++ b/tests/tools/plugins/continuations_verify.cc @@ -56,7 +56,7 @@ handle_msg(TSCont contp, TSEvent event, void *edata) // threads do not get their thread local copy of the stats // merged in. So externally, test.done was stuck at 0 without // the Schedule to a NET thread - TSContSchedule(contp, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); } else { TSDebug(DEBUG_TAG_MSG, "event %d", event); TSStatIntIncrement(stat_test_done, 1); diff --git a/tests/tools/plugins/ssl_hook_test.cc b/tests/tools/plugins/ssl_hook_test.cc index f7fde1c3915..a863c8ce0d9 100644 --- a/tests/tools/plugins/ssl_hook_test.cc +++ b/tests/tools/plugins/ssl_hook_test.cc @@ -73,7 +73,7 @@ CB_Pre_Accept_Delay(TSCont cont, TSEvent event, void *edata) TSContDataSet(cb, ssl_vc); // Schedule to reenable in a bit - TSContSchedule(cb, 2000, TS_THREAD_POOL_NET); + TSContScheduleOnPool(cb, 2000, TS_THREAD_POOL_NET); return TS_SUCCESS; } @@ -108,7 +108,7 @@ CB_out_start_delay(TSCont cont, TSEvent event, void *edata) TSContDataSet(cb, ssl_vc); // Schedule to reenable in a bit - TSContSchedule(cb, 2000, TS_THREAD_POOL_NET); + TSContScheduleOnPool(cb, 2000, TS_THREAD_POOL_NET); return TS_SUCCESS; } @@ -183,7 +183,7 @@ CB_Cert(TSCont cont, TSEvent event, void *edata) TSContDataSet(cb, ssl_vc); // Schedule to reenable in a bit - TSContSchedule(cb, 2000, TS_THREAD_POOL_NET); + TSContScheduleOnPool(cb, 2000, TS_THREAD_POOL_NET); return TS_SUCCESS; } diff --git a/tests/tools/plugins/ssntxnorder_verify.cc b/tests/tools/plugins/ssntxnorder_verify.cc index 9ff16aedf14..fd0a2ce9749 100644 --- a/tests/tools/plugins/ssntxnorder_verify.cc +++ b/tests/tools/plugins/ssntxnorder_verify.cc @@ -249,7 +249,7 @@ handle_order(TSCont contp, TSEvent event, void *edata) if (!strcmp(ctl_tag, msgp->tag) && strncmp(ctl_dump, reinterpret_cast(msgp->data), strlen(ctl_dump)) == 0) { dump_tables(); } else { - TSContSchedule(contp, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); } break; From 7e8a181989ab9e27ddb86104f20065e877230210 Mon Sep 17 00:00:00 2001 From: Walt Karas Date: Thu, 14 Sep 2017 23:28:00 +0000 Subject: [PATCH 172/526] Add Log fields that dump all MIME headers in a message. --- .gitignore | 1 + doc/admin-guide/logging/formatting.en.rst | 32 ++++ proxy/logging/Log.cc | 25 +++ proxy/logging/LogAccess.cc | 52 ++++++ proxy/logging/LogAccess.h | 81 +++++---- proxy/logging/LogUtils.cc | 171 ++++++++++++++++++ proxy/logging/LogUtils.h | 14 ++ proxy/logging/Makefile.am | 27 ++- proxy/logging/test_LogUtils.cc | 9 + .../logging/unit-tests/BufferWriterFormat.cc | 24 +++ proxy/logging/unit-tests/test_LogUtils.h | 73 ++++++++ proxy/logging/unit-tests/test_LogUtils2.cc | 132 ++++++++++++++ src/traffic_server/Makefile.inc | 2 +- tests/gold_tests/logging/all_headers.test.py | 112 ++++++++++++ .../logging/all_headers_sanitizer.py | 43 +++++ .../logging/gold/test_all_headers.gold | 4 + 16 files changed, 759 insertions(+), 43 deletions(-) create mode 100644 proxy/logging/unit-tests/BufferWriterFormat.cc create mode 100644 proxy/logging/unit-tests/test_LogUtils.h create mode 100644 proxy/logging/unit-tests/test_LogUtils2.cc create mode 100644 tests/gold_tests/logging/all_headers.test.py create mode 100644 tests/gold_tests/logging/all_headers_sanitizer.py create mode 100644 tests/gold_tests/logging/gold/test_all_headers.gold diff --git a/.gitignore b/.gitignore index e56e2fef90a..361ce6d9158 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ proxy/http2/test_Http2DependencyTree proxy/http2/test_HPACK proxy/http2/hpack-tests/results proxy/logging/test_LogUtils +proxy/logging/test_LogUtils2 plugins/header_rewrite/header_rewrite_test plugins/experimental/esi/*_test diff --git a/doc/admin-guide/logging/formatting.en.rst b/doc/admin-guide/logging/formatting.en.rst index fce1edd8fba..5295d6bd308 100644 --- a/doc/admin-guide/logging/formatting.en.rst +++ b/doc/admin-guide/logging/formatting.en.rst @@ -274,6 +274,11 @@ HTTP Headers .. _epsh: .. _essh: .. _ecssh: +.. _cqah: +.. _pqah: +.. _psah: +.. _ssah: +.. _cssah: The following log tables provide access to the values of specified HTTP headers from each phase of the transaction lifecycle. Unlike many of the other log @@ -320,6 +325,33 @@ ssh essh cssh ecssh ============== =================== +It is also possible to log all of the headers in a transaction message with a +single field. For each original original field, there is a variant which ends in +``ah`` rather than ``h``, as shown here: + +============== =================== +Original Field All Headers Variant +============== =================== +cqh cqah +pqh pqah +psh psah +ssh ssah +cssh cssah +============== =================== + +No particular header is specified when using these variants, for example:: + + Format = '%' + +The output generated by these fields has the pattern:: + + {{{tag1}:{value1}}{{tag2}:{value2}}...} + +(The size of some messages may exceed internal buffer capacity. This may +result in the value of the last header being truncated, in which case, the +value will end with ``...}``. This may also result in the ommission of +entire tag/value pairs.) + .. _admin-logging-fields-methods: HTTP Methods diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc index 3d11a750e10..2e6d8fa5b1f 100644 --- a/proxy/logging/Log.cc +++ b/proxy/logging/Log.cc @@ -535,6 +535,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("ctec", field); + field = new LogField("client_request_all_header_fields", "cqah", LogField::STRING, + &LogAccess::marshal_client_req_all_header_fields, &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("cqah", field); + // proxy -> client fields field = new LogField("proxy_resp_content_type", "psct", LogField::STRING, &LogAccess::marshal_proxy_resp_content_type, (LogField::UnmarshalFunc)&LogAccess::unmarshal_str); @@ -632,6 +637,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("chm", field); + field = new LogField("proxy_response_all_header_fields", "psah", LogField::STRING, + &LogAccess::marshal_proxy_resp_all_header_fields, &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("psah", field); + // proxy -> server fields field = new LogField("proxy_req_header_len", "pqhl", LogField::sINT, &LogAccess::marshal_proxy_req_header_len, &LogAccess::unmarshal_int_to_str); @@ -710,6 +720,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("pqssl", field); + field = new LogField("proxy_request_all_header_fields", "pqah", LogField::STRING, &LogAccess::marshal_proxy_req_all_header_fields, + &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("pqah", field); + // server -> proxy fields field = new LogField("server_host_ip", "shi", LogField::IP, &LogAccess::marshal_server_host_ip, &LogAccess::unmarshal_ip_to_str); @@ -776,6 +791,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("sca", field); + field = new LogField("origin_response_all_header_fields", "ssah", LogField::STRING, + &LogAccess::marshal_server_resp_all_header_fields, &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("ssah", field); + field = new LogField("cached_resp_status_code", "csssc", LogField::sINT, &LogAccess::marshal_cache_resp_status_code, &LogAccess::unmarshal_http_status); global_field_list.add(field, false); @@ -801,6 +821,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("csshv", field); + field = new LogField("cache_origin_response_all_header_fields", "cssah", LogField::STRING, + &LogAccess::marshal_cache_resp_all_header_fields, &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("cssah", field); + field = new LogField("client_retry_after_time", "crat", LogField::sINT, &LogAccess::marshal_client_retry_after_time, &LogAccess::unmarshal_int_to_str); global_field_list.add(field, false); diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index 586676b09b8..a9a6fe7d82b 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -359,6 +359,15 @@ LogAccess::marshal_str(char *dest, const char *source, int padded_len) #endif } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_client_req_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_client_request, buf); +} + /*------------------------------------------------------------------------- LogAccess::marshal_mem @@ -476,6 +485,18 @@ LogAccess::unmarshal_int(char **buf) return val; } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_proxy_resp_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_proxy_response, buf); +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + /*------------------------------------------------------------------------- unmarshal_itoa @@ -586,6 +607,18 @@ LogAccess::unmarshal_int_to_str_hex(char **buf, char *dest, int len) return -1; } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_proxy_req_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_proxy_request, buf); +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + /*------------------------------------------------------------------------- LogAccess::unmarshal_str @@ -673,6 +706,18 @@ LogAccess::unmarshal_int_to_time_str(char **buf, char *dest, int len) return strlen; } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_server_resp_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_server_response, buf); +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + int LogAccess::unmarshal_int_to_netscape_str(char **buf, char *dest, int len) { @@ -702,6 +747,13 @@ LogAccess::unmarshal_http_method (char **buf, char *dest, int len) return unmarshal_str (buf, dest, len); } */ + +int +LogAccess::marshal_cache_resp_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_cache_response, buf); +} + /*------------------------------------------------------------------------- LogAccess::unmarshal_http_version diff --git a/proxy/logging/LogAccess.h b/proxy/logging/LogAccess.h index f402ad97186..fa221cacb4e 100644 --- a/proxy/logging/LogAccess.h +++ b/proxy/logging/LogAccess.h @@ -156,60 +156,65 @@ class LogAccess inkcoreapi int marshal_client_req_uuid(char *); // STR inkcoreapi int marshal_client_rx_error_code(char *); // STR inkcoreapi int marshal_client_tx_error_code(char *); // STR + inkcoreapi int marshal_client_req_all_header_fields(char *); // STR // // proxy -> client fields // - inkcoreapi int marshal_proxy_resp_content_type(char *); // STR - inkcoreapi int marshal_proxy_resp_reason_phrase(char *); // STR - inkcoreapi int marshal_proxy_resp_squid_len(char *); // INT - inkcoreapi int marshal_proxy_resp_content_len(char *); // INT - inkcoreapi int marshal_proxy_resp_status_code(char *); // INT - inkcoreapi int marshal_proxy_resp_header_len(char *); // INT - inkcoreapi int marshal_proxy_finish_status_code(char *); // INT - inkcoreapi int marshal_cache_result_code(char *); // INT - inkcoreapi int marshal_cache_result_subcode(char *); // INT - inkcoreapi int marshal_proxy_host_port(char *); // INT - inkcoreapi int marshal_cache_hit_miss(char *); // INT + inkcoreapi int marshal_proxy_resp_content_type(char *); // STR + inkcoreapi int marshal_proxy_resp_reason_phrase(char *); // STR + inkcoreapi int marshal_proxy_resp_squid_len(char *); // INT + inkcoreapi int marshal_proxy_resp_content_len(char *); // INT + inkcoreapi int marshal_proxy_resp_status_code(char *); // INT + inkcoreapi int marshal_proxy_resp_header_len(char *); // INT + inkcoreapi int marshal_proxy_finish_status_code(char *); // INT + inkcoreapi int marshal_cache_result_code(char *); // INT + inkcoreapi int marshal_cache_result_subcode(char *); // INT + inkcoreapi int marshal_proxy_host_port(char *); // INT + inkcoreapi int marshal_cache_hit_miss(char *); // INT + inkcoreapi int marshal_proxy_resp_all_header_fields(char *); // STR // // proxy -> server fields // - inkcoreapi int marshal_proxy_req_header_len(char *); // INT - inkcoreapi int marshal_proxy_req_squid_len(char *); // INT - inkcoreapi int marshal_proxy_req_content_len(char *); // INT - inkcoreapi int marshal_proxy_req_server_ip(char *); // INT - inkcoreapi int marshal_proxy_req_server_port(char *); // INT - inkcoreapi int marshal_proxy_hierarchy_route(char *); // INT - inkcoreapi int marshal_next_hop_ip(char *); // STR - inkcoreapi int marshal_next_hop_port(char *); // INT - inkcoreapi int marshal_proxy_host_name(char *); // STR - inkcoreapi int marshal_proxy_host_ip(char *); // STR - inkcoreapi int marshal_proxy_req_is_ssl(char *); // INT + inkcoreapi int marshal_proxy_req_header_len(char *); // INT + inkcoreapi int marshal_proxy_req_squid_len(char *); // INT + inkcoreapi int marshal_proxy_req_content_len(char *); // INT + inkcoreapi int marshal_proxy_req_server_ip(char *); // INT + inkcoreapi int marshal_proxy_req_server_port(char *); // INT + inkcoreapi int marshal_proxy_hierarchy_route(char *); // INT + inkcoreapi int marshal_next_hop_ip(char *); // STR + inkcoreapi int marshal_next_hop_port(char *); // INT + inkcoreapi int marshal_proxy_host_name(char *); // STR + inkcoreapi int marshal_proxy_host_ip(char *); // STR + inkcoreapi int marshal_proxy_req_is_ssl(char *); // INT + inkcoreapi int marshal_proxy_req_all_header_fields(char *); // STR // // server -> proxy fields // - inkcoreapi int marshal_server_host_ip(char *); // INT - inkcoreapi int marshal_server_host_name(char *); // STR - inkcoreapi int marshal_server_resp_status_code(char *); // INT - inkcoreapi int marshal_server_resp_squid_len(char *); // INT - inkcoreapi int marshal_server_resp_content_len(char *); // INT - inkcoreapi int marshal_server_resp_header_len(char *); // INT - inkcoreapi int marshal_server_resp_http_version(char *); // INT - inkcoreapi int marshal_server_resp_time_ms(char *); // INT - inkcoreapi int marshal_server_resp_time_s(char *); // INT - inkcoreapi int marshal_server_transact_count(char *); // INT - inkcoreapi int marshal_server_connect_attempts(char *); // INT + inkcoreapi int marshal_server_host_ip(char *); // INT + inkcoreapi int marshal_server_host_name(char *); // STR + inkcoreapi int marshal_server_resp_status_code(char *); // INT + inkcoreapi int marshal_server_resp_squid_len(char *); // INT + inkcoreapi int marshal_server_resp_content_len(char *); // INT + inkcoreapi int marshal_server_resp_header_len(char *); // INT + inkcoreapi int marshal_server_resp_http_version(char *); // INT + inkcoreapi int marshal_server_resp_time_ms(char *); // INT + inkcoreapi int marshal_server_resp_time_s(char *); // INT + inkcoreapi int marshal_server_transact_count(char *); // INT + inkcoreapi int marshal_server_connect_attempts(char *); // INT + inkcoreapi int marshal_server_resp_all_header_fields(char *); // STR // // cache -> client fields // - inkcoreapi int marshal_cache_resp_status_code(char *); // INT - inkcoreapi int marshal_cache_resp_squid_len(char *); // INT - inkcoreapi int marshal_cache_resp_content_len(char *); // INT - inkcoreapi int marshal_cache_resp_header_len(char *); // INT - inkcoreapi int marshal_cache_resp_http_version(char *); // INT + inkcoreapi int marshal_cache_resp_status_code(char *); // INT + inkcoreapi int marshal_cache_resp_squid_len(char *); // INT + inkcoreapi int marshal_cache_resp_content_len(char *); // INT + inkcoreapi int marshal_cache_resp_header_len(char *); // INT + inkcoreapi int marshal_cache_resp_http_version(char *); // INT + inkcoreapi int marshal_cache_resp_all_header_fields(char *); // STR inkcoreapi void set_client_req_url(char *, int); // STR inkcoreapi void set_client_req_url_canon(char *, int); // STR diff --git a/proxy/logging/LogUtils.cc b/proxy/logging/LogUtils.cc index 7b4146362c7..22359ebd24a 100644 --- a/proxy/logging/LogUtils.cc +++ b/proxy/logging/LogUtils.cc @@ -21,8 +21,22 @@ See the License for the specific language governing permissions and limitations under the License. */ + +#include #include "tscore/ink_config.h" #include "tscore/ink_string.h" +#include +#include + +#ifdef TEST_LOG_UTILS + +#include "unit-tests/test_LogUtils.h" + +#else + +#include + +#endif #include #include @@ -30,6 +44,8 @@ #include #include #include +#include +#include #include #include @@ -571,3 +587,158 @@ LogUtils::file_is_writeable(const char *full_filename, off_t *size_bytes, bool * return ret_val; } + +namespace +{ +// Get a string out of a MIMEField using one of its member funcitions, and put it into a buffer writer, terminated with a nul. +// +void +marshalStr(ts::FixedBufferWriter &bw, const MIMEField &mf, const char *(MIMEField::*get_func)(int *length) const) +{ + int length; + const char *data = (mf.*get_func)(&length); + + if (!data or (*data == '\0')) { + // Empty string. This is a problem, since it would result in two successive nul characters, which indicates the end of the + // marshaled hearer. Change the string to a single blank character. + + static const char Blank[] = " "; + data = Blank; + length = 1; + } + + bw << std::string_view(data, length) << '\0'; +} + +void +unmarshalStr(ts::FixedBufferWriter &bw, const char *&data) +{ + bw << '{'; + + while (*data) { + bw << *(data++); + } + + // Skip over terminal nul. + ++data; + + bw << '}'; +} + +} // end anonymous namespace + +namespace LogUtils +{ +// Marshals header tags and values together, with a single terminating nul character. Returns buffer space required. 'buf' points +// to where to put the marshaled data. If 'buf' is null, no data is marshaled, but the function returns the amount of space that +// would have been used. +// +int +marshalMimeHdr(MIMEHdr *hdr, char *buf) +{ + std::size_t bwSize = buf ? SIZE_MAX : 0; + + ts::FixedBufferWriter bw(buf, bwSize); + + if (hdr) { + MIMEFieldIter mfIter; + const MIMEField *mfp = hdr->iter_get_first(&mfIter); + + while (mfp) { + marshalStr(bw, *mfp, &MIMEField::name_get); + marshalStr(bw, *mfp, &MIMEField::value_get); + + mfp = hdr->iter_get_next(&mfIter); + } + } + + bw << '\0'; + + return int(INK_ALIGN_DEFAULT(bw.extent())); +} + +// Unmarshaled/printable format is {{{tag1}:{value1}}{{tag2}:{value2}} ... } +// +int +unmarshalMimeHdr(char **buf, char *dest, int destLength) +{ + ink_assert(*buf != nullptr); + + const char *data{*buf}; + + ink_assert(data != nullptr); + + ts::FixedBufferWriter bw(dest, destLength); + + bw << '{'; + + int pairEndFallback{0}, pairEndFallback2{0}, pairSeparatorFallback{0}; + + while (*data) { + if (!bw.error()) { + pairEndFallback2 = pairEndFallback; + pairEndFallback = bw.size(); + } + + // Add open bracket of pair. + // + bw << '{'; + + // Unmarshal field name. + unmarshalStr(bw, data); + + bw << ':'; + + if (!bw.error()) { + pairSeparatorFallback = bw.size(); + } + + // Unmarshal field value. + unmarshalStr(bw, data); + + // Add close bracket of pair. + bw << '}'; + + } // end for loop + + bw << '}'; + + if (bw.error()) { + // The output buffer wasn't big enough. + + static std::string_view FULL_ELLIPSES("...}}}"); + + if ((pairSeparatorFallback > pairEndFallback) and ((pairSeparatorFallback + 7) <= destLength)) { + // In the report, we can show the existence of the last partial tag/value pair, and maybe part of the value. If we only + // show part of the value, we want to end it with an elipsis, to make it clear it's not complete. + + bw.reduce(destLength - FULL_ELLIPSES.size()); + bw << FULL_ELLIPSES; + + } else if (pairEndFallback and (pairEndFallback < destLength)) { + bw.reduce(pairEndFallback); + bw << '}'; + + } else if ((pairSeparatorFallback > pairEndFallback2) and ((pairSeparatorFallback + 7) <= destLength)) { + bw.reduce(destLength - FULL_ELLIPSES.size()); + bw << FULL_ELLIPSES; + + } else if (pairEndFallback2 and (pairEndFallback2 < destLength)) { + bw.reduce(pairEndFallback2); + bw << '}'; + + } else if (destLength > 1) { + bw.reduce(1); + bw << '}'; + + } else { + bw.reduce(0); + } + } + + *buf += INK_ALIGN_DEFAULT(data - *buf + 1); + + return bw.size(); +} + +} // end namespace LogUtils diff --git a/proxy/logging/LogUtils.h b/proxy/logging/LogUtils.h index 96cd05148b0..2860db1b63f 100644 --- a/proxy/logging/LogUtils.h +++ b/proxy/logging/LogUtils.h @@ -27,6 +27,8 @@ #include "tscore/ink_platform.h" #include "tscore/Arena.h" +class MIMEHdr; + namespace LogUtils { enum AlarmType { @@ -59,4 +61,16 @@ int timestamp_to_hex_str(unsigned timestamp, char *str, size_t len, size_t *n_ch int seconds_to_next_roll(time_t time_now, int rolling_offset, int rolling_interval); int file_is_writeable(const char *full_filename, off_t *size_bytes = nullptr, bool *has_size_limit = nullptr, uint64_t *current_size_limit_bytes = nullptr); + +// Marshals header tags and values together, with a single terminating nul character. Returns buffer space required. 'buf' points +// to where to put the marshaled data. If 'buf' is null, no data is marshaled, but the function returns the amount of space that +// would have been used. +int marshalMimeHdr(MIMEHdr *hdr, char *buf); + +// Unmarshelled/printable format is {{{tag1}:{value1}}{{tag2}:{value2}} ... } +// +// Returns -1 if data corruption is detected, otherwise the actual amount of data put into the 'dest' buffer. '*buf' is advanced +// to byte after the last byte consumed. +int unmarshalMimeHdr(char **buf, char *dest, int destLength); + }; // namespace LogUtils diff --git a/proxy/logging/Makefile.am b/proxy/logging/Makefile.am index cac0eb4891b..9f438edb1a4 100644 --- a/proxy/logging/Makefile.am +++ b/proxy/logging/Makefile.am @@ -79,18 +79,37 @@ liblogcollation_a_SOURCES = \ LogCollationHostSM.h check_PROGRAMS = \ - test_LogUtils + test_LogUtils \ + test_LogUtils2 TESTS = \ - test_LogUtils + test_LogUtils \ + test_LogUtils2 + +test_LogUtils_CPPFLAGS = $(AM_CPPFLAGS)\ + -DTEST_LOG_UTILS + +test_LogUtils_SOURCES = \ + test_LogUtils.cc test_LogUtils_LDADD = \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/iocore/eventsystem/libinkevent.a -test_LogUtils_SOURCES = \ - test_LogUtils.cc +test_LogUtils2_CPPFLAGS = $(AM_CPPFLAGS)\ + -DTEST_LOG_UTILS \ + -I$(abs_top_srcdir)/tests/include + +test_LogUtils2_SOURCES = \ + LogUtils.cc \ + unit-tests/BufferWriterFormat.cc \ + unit-tests/test_LogUtils2.cc + +test_LogUtils2_LDADD = \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a clang-tidy-local: $(liblogging_a_SOURCES) $(liblogcollation_a_SOURCES) $(EXTRA_DIST) $(CXX_Clang_Tidy) diff --git a/proxy/logging/test_LogUtils.cc b/proxy/logging/test_LogUtils.cc index 0f39524ceaa..02b8cfee6b7 100644 --- a/proxy/logging/test_LogUtils.cc +++ b/proxy/logging/test_LogUtils.cc @@ -25,6 +25,15 @@ #include "LogUtils.cc" #include +#if 0 +// Stub +EThread * +this_ethread() +{ + return nullptr; +} +#endif + REGRESSION_TEST(LogUtils_pure_escapify_url)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) { TestBox box(t, pstatus); diff --git a/proxy/logging/unit-tests/BufferWriterFormat.cc b/proxy/logging/unit-tests/BufferWriterFormat.cc new file mode 100644 index 00000000000..d2c3e39015b --- /dev/null +++ b/proxy/logging/unit-tests/BufferWriterFormat.cc @@ -0,0 +1,24 @@ +/** @file + + includes BufferWriterFormat.cc from TS utilities. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "../../../src/tscore/BufferWriterFormat.cc" diff --git a/proxy/logging/unit-tests/test_LogUtils.h b/proxy/logging/unit-tests/test_LogUtils.h new file mode 100644 index 00000000000..3c9ed225932 --- /dev/null +++ b/proxy/logging/unit-tests/test_LogUtils.h @@ -0,0 +1,73 @@ +/** @file + + Header file for shared declarations/definitions for test_LogUtils2.cc and LogUtils.h for unit testing. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +struct MIMEField { + const char *tag, *value; + + const char * + name_get(int *length) const + { + *length = strlen(tag); + return tag; + } + const char * + value_get(int *length) const + { + *length = strlen(value); + return value; + } +}; + +struct MIMEFieldIter { +}; + +class MIMEHdr +{ +public: + MIMEHdr(const MIMEField *first, int count) : _first(first), _count(count), _idx(0) {} + + const MIMEField * + iter_get_first(MIMEFieldIter *) + { + return _idx < _count ? _first + _idx : nullptr; + } + const MIMEField * + iter_get_next(MIMEFieldIter *) + { + ++_idx; + return iter_get_first(nullptr); + } + + void + reset() + { + _idx = 0; + } + +private: + const MIMEField *const _first; + const int _count; + int _idx; +}; diff --git a/proxy/logging/unit-tests/test_LogUtils2.cc b/proxy/logging/unit-tests/test_LogUtils2.cc new file mode 100644 index 00000000000..31376b3f4d0 --- /dev/null +++ b/proxy/logging/unit-tests/test_LogUtils2.cc @@ -0,0 +1,132 @@ +/** @file + + Catch-based tests for LogUtils.h. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include + +#include + +#include "test_LogUtils.h" + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include + +using namespace LogUtils; + +namespace +{ +void +test(const MIMEField *pairs, int numPairs, const char *asciiResult, int extraUnmarshalSpace = 0) +{ + char binBuf[1500], asciiBuf[1500]; + + MIMEHdr hdr{pairs, numPairs}; + + int binAlignSize = marshalMimeHdr(numPairs ? &hdr : nullptr, nullptr); + + REQUIRE(binAlignSize < sizeof(binBuf)); + + hdr.reset(); + + REQUIRE(marshalMimeHdr(numPairs ? &hdr : nullptr, binBuf) == binAlignSize); + + int binSize{1}; + + if (binBuf[0]) { + for (; binBuf[binSize] or binBuf[binSize + 1]; ++binSize) { + } + + binSize += 2; + + } else { + binSize = 1; + } + + REQUIRE(INK_ALIGN_DEFAULT(binSize) == binAlignSize); + + char *bp = binBuf; + + int asciiSize = unmarshalMimeHdr(&bp, asciiBuf, std::strlen(asciiResult) + extraUnmarshalSpace); + + REQUIRE(asciiSize == std::strlen(asciiResult)); + + REQUIRE((bp - binBuf) == binAlignSize); + + REQUIRE(std::memcmp(asciiBuf, asciiResult, asciiSize) == 0); +} + +} // namespace + +TEST_CASE("LogUtilsHttp", "[LUHP]") +{ +#define X "12345678" +#define X2 X X +#define X3 X2 X2 +#define X4 X3 X3 +#define X5 X4 X4 +#define X6 X5 X5 +#define X7 X6 X6 +#define X8 X7 X7 + + const MIMEField pairs[] = {{"Argh", "Ugh"}, {"Argh2", "UghUgh"}, {"alltogethernow", X8}}; + + test(pairs, 1, "{{{Argh}:{Ugh}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}{{Argh2}:{UghUgh}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}{{Argh2}:{Ug...}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}{{Argh2}:{U...}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}{{Argh2}:{...}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}}", 1); + test(pairs, 2, "{{{Argh}:{Ugh}}}", sizeof("{{Argh2}:{...}}") - 2); + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{UghUgh}}{{alltogethernow}:{" X8 "}}}"); + + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{UghUgh}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{Ug...}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{U...}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{...}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}}", 1); + test(pairs, 3, "{{{Argh}:{Ugh}}}", sizeof("{{Argh2}:{...}}") - 2); + + test(nullptr, 0, "{}"); + test(nullptr, 0, ""); + test(nullptr, 0, "", 1); +} + +#include +#include + +void +_ink_assert(const char *a, const char *f, int line) +{ + std::cout << a << '\n' << f << '\n' << line << '\n'; + + std::exit(1); +} + +void +RecSignalManager(int, char const *, std::size_t) +{ +} diff --git a/src/traffic_server/Makefile.inc b/src/traffic_server/Makefile.inc index 4657128de0f..e828109b508 100644 --- a/src/traffic_server/Makefile.inc +++ b/src/traffic_server/Makefile.inc @@ -64,8 +64,8 @@ traffic_server_traffic_server_LDADD = \ $(top_builddir)/proxy/http/remap/libhttp_remap.a \ $(top_builddir)/proxy/http2/libhttp2.a \ $(top_builddir)/proxy/logging/liblogging.a \ - $(top_builddir)/proxy/logging/liblogcollation.a \ $(top_builddir)/proxy/hdrs/libhdrs.a \ + $(top_builddir)/proxy/logging/liblogcollation.a \ $(top_builddir)/proxy/shared/libdiagsconfig.a \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/iocore/utils/libinkutils.a \ diff --git a/tests/gold_tests/logging/all_headers.test.py b/tests/gold_tests/logging/all_headers.test.py new file mode 100644 index 00000000000..8279ae36b78 --- /dev/null +++ b/tests/gold_tests/logging/all_headers.test.py @@ -0,0 +1,112 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +Test.Summary = ''' +Test new "all headers" log fields +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram( + "curl", "Curl need to be installed on system for this test to work"), + # Condition.IsPlatform("linux"), Don't see the need for this. +) + +# Define ATS. +# +ts = Test.MakeATSProcess("ts") + +# Define MicroServer. +# +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: does.not.matter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-control: max-age=85000\r\n\r\n", + "timestamp": "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|dns', +}) + +ts.Disk.remap_config.AddLine( + 'map http://127.0.0.1:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) +) + +# Mix in a numeric log field. Hopefull this will detect any binary alignment problems. +# +ts.Disk.logging_yaml.AddLines( + ''' +formats: + - name: custom + format: " % % % % % % " +logs: + - filename: test_all_headers + format: custom +'''.split("\n") +) + +# Configure comparison of "sanitized" log file with gold file at end of test. +# +Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'test_all_headers.log.san'), + exists=True, content='gold/test_all_headers.gold') + +# Ask the OS if the port is ready for connect() +# +def CheckPort(Port): + return lambda: 0 == subprocess.call('netstat --listen --tcp -n | grep -q :{}'.format(Port), shell=True) + +def reallyLong(): + value = 'abcdefghijklmnop' + value = value + value + value = value + value + value = value + value + retval = "" + for i in range(3): + retval += ' -H "x-header{}: {}"'.format(i, value) + return retval + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server, ready=CheckPort(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=CheckPort(ts.Variables.port)) +tr.Processes.Default.Command = ( +'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() +) +tr.Processes.Default.ReturnCode = 0 + +# Repeat same curl, will be answered from the ATS cache. +# +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( +'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() +) +tr.Processes.Default.ReturnCode = 0 + +# Delay to allow TS to flush report to disk, then "sanitize" generated log. +# +tr = Test.AddTestRun() +tr.DelayStart = 10 +tr.Processes.Default.Command = 'python {0} {3} < {1} > {2}'.format( + os.path.join(Test.TestDirectory, 'all_headers_sanitizer.py'), + os.path.join(ts.Variables.LOGDIR, 'test_all_headers.log'), + os.path.join(ts.Variables.LOGDIR, 'test_all_headers.log.san'), + server.Variables.Port) +tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/logging/all_headers_sanitizer.py b/tests/gold_tests/logging/all_headers_sanitizer.py new file mode 100644 index 00000000000..0e798df96da --- /dev/null +++ b/tests/gold_tests/logging/all_headers_sanitizer.py @@ -0,0 +1,43 @@ +''' +Sanitize the ATS-generated custom log file from the all_headers test. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import re + +rexl = [] +rexl.append((re.compile(r"\{\{Date\}\:\{[^}]*\}\}"), "__DATE__")) +rexl.append((re.compile(r"\{\{Expires\}\:\{[^}]*\}\}"), "__EXPIRES__")) +rexl.append((re.compile(r"\{\{Last-Modified\}\:\{[^}]*\}\}"), "__LAST_MODIFIED__")) +rexl.append((re.compile(r"\{\{Server\}\:\{ATS/[0-9.]*\}\}"), "__ATS_SERVER__")) +rexl.append((re.compile(r"\{\{Server\}\:\{ECS [^}]*\}\}"), "__ECS_SERVER__")) +rexl.append((re.compile(r"\{\{Via\}\:\{[^}]*\}\}"), "__VIA__")) +rexl.append((re.compile(r"\{\{Server\}\:\{ApacheTrafficServer/[0-9.]*\}\}"), "__ATS2_SERVER__")) +rexl.append((re.compile(r"\{\{Age\}\:\{[0-9]*\}\}"), "__AGE__")) +rexl.append((re.compile(r"\:" + sys.argv[1]), "__TS_PORT__")) # 1st and only argument is TS client port + +# Handle inconsistencies which I think are caused by different revisions of the standard Python http.server.HTTPServer class. + +rexl.append((re.compile(r'\{"359670651[^"]*"\}'), '{"359670651__WEIRD__"}')) +rexl.append((re.compile(r'\{\{Accept-Ranges\}:\{bytes\}\}'), '')) + +for line in sys.stdin: + for rex, subStr in rexl: + line = rex.sub(subStr, line) + + print(line) diff --git a/tests/gold_tests/logging/gold/test_all_headers.gold b/tests/gold_tests/logging/gold/test_all_headers.gold new file mode 100644 index 00000000000..3cb54e5b81e --- /dev/null +++ b/tests/gold_tests/logging/gold/test_all_headers.gold @@ -0,0 +1,4 @@ + {{{Host}:{127.0.0.1__TS_PORT__}}{{User-Agent}:{007}}{{Accept}:{*/*}}{{x-header0}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header1}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header2}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}} 200 {{{Cache-Control}:{max-age=85000}}{{Content-Length}:{3}}__DATE____AGE__{{Connection}:{keep-alive}}__ATS_SERVER__} {{{Connection}:{close}}{{Cache-Control}:{max-age=85000}}{{Content-Length}:{3}}__DATE__} {{{Host}:{127.0.0.1__TS_PORT__}}{{User-Agent}:{007}}{{Accept}:{*/*}}{{x-header0}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header1}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header2}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{Client-ip}:{127.0.0.1}}{{X-Forwarded-For}:{127.0.0.1}}__VIA__} {} + + {{{Host}:{127.0.0.1__TS_PORT__}}{{User-Agent}:{007}}{{Accept}:{*/*}}{{x-header0}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header1}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header2}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}} 200 {{{Cache-Control}:{max-age=85000}}{{Content-Length}:{3}}__DATE____AGE__{{Connection}:{keep-alive}}__ATS_SERVER__} {} {} {{{Cache-Control}:{max-age=85000}}{{Content-Length}:{3}}__DATE____AGE__{{Connection}:{keep-alive}}__ATS_SERVER__} + From 34d9c4eaac536a2f27debd15326c58da91d711af Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 16 Jan 2019 20:50:27 +0000 Subject: [PATCH 173/526] Add valid_tls_protocols_in to allow for per-domain protocols. --- build/crypto.m4 | 41 +++++++ configure.ac | 3 + .../files/ssl_server_name.yaml.en.rst | 11 +- include/tscore/ink_config.h.in | 1 + iocore/net/P_SNIActionPerformer.h | 26 ++++ iocore/net/P_SSLNetVConnection.h | 3 + iocore/net/P_SSLUtils.h | 1 + iocore/net/SSLSNIConfig.cc | 3 + iocore/net/SSLUtils.cc | 64 +++++++++- iocore/net/YamlSNIConfig.cc | 50 +++++++- iocore/net/YamlSNIConfig.h | 6 + .../tls/tls_client_versions.test.py | 112 ++++++++++++++++++ 12 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 tests/gold_tests/tls/tls_client_versions.test.py diff --git a/build/crypto.m4 b/build/crypto.m4 index e3ac31bae02..248be9eefbe 100644 --- a/build/crypto.m4 +++ b/build/crypto.m4 @@ -144,6 +144,47 @@ AC_DEFUN([TS_CHECK_CRYPTO_CERT_CB], [ AC_SUBST(use_cert_cb) ]) +AC_DEFUN([TS_CHECK_CRYPTO_HELLO_CB], [ + _hello_saved_LIBS=$LIBS + enable_hello_cb=yes + + TS_ADDTO(LIBS, [$OPENSSL_LIBS]) + AC_CHECK_HEADERS(openssl/ssl.h openssl/ts.h) + AC_CHECK_HEADERS(openssl/tls1.h, [], [], +[ #if HAVE_OPENSSL_SSL_H +#include +#include +#endif ]) + + AC_MSG_CHECKING([for SSL_CTX_set_client_hello_cb]) + AC_LINK_IFELSE( + [ + AC_LANG_PROGRAM([[ +#if HAVE_OPENSSL_SSL_H +#include +#endif +#if HAVE_OPENSSL_TLS1_H +#include +#endif + ]], + [[SSL_CTX_set_client_hello_cb(NULL, NULL, NULL);]]) + ], + [ + AC_MSG_RESULT([yes]) + ], + [ + AC_MSG_RESULT([no]) + enable_hello_cb=no + ]) + + LIBS=$_hello_saved_LIBS + + AC_MSG_CHECKING(whether to enable TLS client hello callback support) + AC_MSG_RESULT([$enable_hello_cb]) + TS_ARG_ENABLE_VAR([use], [hello-cb]) + AC_SUBST(use_hello_cb) +]) + AC_DEFUN([TS_CHECK_CRYPTO_SET_RBIO], [ _rbio_saved_LIBS=$LIBS enable_set_rbio=yes diff --git a/configure.ac b/configure.ac index 9600c4f04b1..bec8ef593bd 100644 --- a/configure.ac +++ b/configure.ac @@ -1195,6 +1195,9 @@ TS_CHECK_CRYPTO_EC_KEYS # Check for the presense of the certificate callback in the ssl library TS_CHECK_CRYPTO_CERT_CB +# Check for the client hello callback +TS_CHECK_CRYPTO_HELLO_CB + # # Check for SSL_set0_rbio call TS_CHECK_CRYPTO_SET_RBIO diff --git a/doc/admin-guide/files/ssl_server_name.yaml.en.rst b/doc/admin-guide/files/ssl_server_name.yaml.en.rst index e95c3dd2d33..ac8068b49dc 100644 --- a/doc/admin-guide/files/ssl_server_name.yaml.en.rst +++ b/doc/admin-guide/files/ssl_server_name.yaml.en.rst @@ -71,9 +71,18 @@ verify_client One of the values :code:`NONE`, :code:`MODERATE`, or : fail the TLS handshake if new certificate is presented. If ``STRICT`` is specified the client must resent a certificate during the TLS handshake. - By default this is :ts:cv:`proxy.config.ssl.client.certification_level`. +valid_tls_versions_in This specifies the list of TLS protocols that will be offered to user agents during + the TLS negotiaton. This replaces the global settings in :ts:cv:`proxy.config.ssl.TSLv1`, + :ts:cv:`proxy.config.ssl.TLSv1_1`, :ts:cv:`proxy.config.ssl.TLSv1_2`, + and :ts:cv:`proxy.config.ssl.TLSv1_3`. The potential values are TLSv1, TLSv1_1, TLSv1_2, and + TLSv1_3. You must list all protocols that |TS| should offer to the client when using + this key. This key is only valid for openssl 1.1.0 and later. Older versions of openssl do not + provide a hook early enough to update the SSL object. It is a syntax error for |TS| built + against earlier versions. + + client_cert The file containing the client certificate to use for the outbound connection. If this is relative, it is relative to the path in diff --git a/include/tscore/ink_config.h.in b/include/tscore/ink_config.h.in index d6c1a54ec85..770bd7653ba 100644 --- a/include/tscore/ink_config.h.in +++ b/include/tscore/ink_config.h.in @@ -72,6 +72,7 @@ #define TS_USE_TLS_ALPN @use_tls_alpn@ #define TS_USE_TLS_ASYNC @use_tls_async@ #define TS_USE_CERT_CB @use_cert_cb@ +#define TS_USE_HELLO_CB @use_hello_cb@ #define TS_USE_SET_RBIO @use_set_rbio@ #define TS_USE_GET_DH_2048_256 @use_dh_get_2048_256@ #define TS_USE_TLS_ECKEY @use_tls_eckey@ diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index d515fdcfdce..f940490e302 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -115,6 +115,32 @@ class VerifyClient : public ActionItem } }; +class TLSValidProtocols : public ActionItem +{ + bool unset; + unsigned long protocol_mask; + +public: +#ifdef SSL_OP_NO_TLSv1_3 + static const unsigned long max_mask = SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3; +#else + static const unsigned long max_mask = SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2; +#endif + TLSValidProtocols() : unset(true), protocol_mask(max_mask) {} + TLSValidProtocols(unsigned long protocols) : unset(false), protocol_mask(protocols) {} + int + SNIAction(Continuation *cont) const override + { + if (!unset) { + auto ssl_vc = dynamic_cast(cont); + Debug("ssl_sni", "TLSValidProtocol param 0%x", static_cast(this->protocol_mask)); + ssl_vc->protocol_mask_set = true; + ssl_vc->protocol_mask = protocol_mask; + } + return SSL_TLSEXT_ERR_OK; + } +}; + class SNI_IpAllow : public ActionItem { IpMap ip_map; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 19976be2a08..4de0038f751 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -370,6 +370,9 @@ class SSLNetVConnection : public UnixNetVConnection SSLNetVConnection(const SSLNetVConnection &) = delete; SSLNetVConnection &operator=(const SSLNetVConnection &) = delete; + bool protocol_mask_set = false; + unsigned long protocol_mask; + private: std::string_view map_tls_protocol_to_tag(const char *proto_string) const; bool update_rbio(bool move_to_socket); diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index 3d2b01a1baf..e3ec9f6d082 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -177,6 +177,7 @@ void SSLNetVCDetach(SSL *ssl); SSLNetVConnection *SSLNetVCAccess(const SSL *ssl); void setClientCertLevel(SSL *ssl, uint8_t certLevel); +void setTLSValidProtocols(SSL *ssl, unsigned long proto_mask, unsigned long max_mask); namespace ssl { diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 470a54419ce..06f15cd0927 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -71,6 +71,9 @@ SNIConfigParams::loadSNIConfig() if (item.verify_client_level != 255) { ai->actions.push_back(std::make_unique(item.verify_client_level)); } + if (!item.protocol_unset) { + ai->actions.push_back(std::make_unique(item.protocol_mask)); + } if (item.tunnel_destination.length() > 0) { ai->actions.push_back(std::make_unique(item.tunnel_destination, item.tunnel_decrypt)); } diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 7c9fbaf1ff2..e7a64447677 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -418,6 +418,57 @@ PerformAction(Continuation *cont, const char *servername) return SSL_TLSEXT_ERR_OK; } +#if TS_USE_HELLO_CB +// Pausable callback +static int +ssl_client_hello_callback(SSL *s, int *al, void *arg) +{ + const char *servername = nullptr; + const unsigned char *p; + size_t remaining, len; + if (SSL_client_hello_get0_ext(s, TLSEXT_TYPE_server_name, &p, &remaining) || remaining <= 2) { + // Parse to get to the name, originally from test/handshake_helper.c in openssl tree + /* Extract the length of the supplied list of names. */ + len = *(p++) << 8; + len += *(p++); + if (len + 2 == remaining) { + remaining = len; + /* + * The list in practice only has a single element, so we only consider + * the first one. + */ + if (remaining != 0 && *p++ == TLSEXT_NAMETYPE_host_name) { + remaining--; + /* Now we can finally pull out the byte array with the actual hostname. */ + if (remaining > 2) { + len = *(p++) << 8; + len += *(p++); + if (len + 2 <= remaining) { + remaining = len; + servername = reinterpret_cast(p); + } + } + } + } + } + + SSLNetVConnection *netvc = SSLNetVCAccess(s); + + netvc->serverName = servername ? servername : ""; + int ret = PerformAction(netvc, netvc->serverName); + if (ret != SSL_TLSEXT_ERR_OK) { + return SSL_TLSEXT_ERR_ALERT_FATAL; + } + if (netvc->has_tunnel_destination() && !netvc->decrypt_tunnel()) { + netvc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; + } + if (netvc->protocol_mask_set) { + setTLSValidProtocols(s, netvc->protocol_mask, TLSValidProtocols::max_mask); + } + return 1; +} +#endif + // Use the certificate callback for openssl 1.0.2 and greater // otherwise use the SNI callback #if TS_USE_CERT_CB @@ -471,6 +522,8 @@ ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) if (nullptr == netvc->serverName) { netvc->serverName = ""; } + + // Rerun the actions in case a plugin changed the server name int ret = PerformAction(netvc, netvc->serverName); if (ret != SSL_TLSEXT_ERR_OK) { return SSL_TLSEXT_ERR_ALERT_FATAL; @@ -478,7 +531,6 @@ ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) if (netvc->has_tunnel_destination() && !netvc->decrypt_tunnel()) { netvc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; } - return SSL_TLSEXT_ERR_OK; } @@ -1592,6 +1644,16 @@ ssl_set_handshake_callbacks(SSL_CTX *ctx) #else SSL_CTX_set_tlsext_servername_callback(ctx, ssl_servername_and_cert_callback); #endif +#if TS_USE_HELLO_CB + SSL_CTX_set_client_hello_cb(ctx, ssl_client_hello_callback, nullptr); +#endif +} + +void +setTLSValidProtocols(SSL *ssl, unsigned long proto_mask, unsigned long max_mask) +{ + SSL_set_options(ssl, proto_mask); + SSL_clear_options(ssl, max_mask & ~proto_mask); } void diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc index 8a542cd8706..84e118ea006 100644 --- a/iocore/net/YamlSNIConfig.cc +++ b/iocore/net/YamlSNIConfig.cc @@ -26,10 +26,12 @@ #include #include +#include #include "tscore/Diags.h" #include "tscore/EnumDescriptor.h" #include "tsconfig/Errata.h" +#include "P_SNIActionPerformer.h" ts::Errata YamlSNIConfig::loader(const char *cfgFilename) @@ -55,9 +57,37 @@ YamlSNIConfig::loader(const char *cfgFilename) return ts::Errata(); } -TsEnumDescriptor LEVEL_DESCRIPTOR = {{{"NONE", 0}, {"MODERATE", 1}, {"STRICT", 2}}}; -TsEnumDescriptor POLICY_DESCRIPTOR = {{{"DISABLED", 0}, {"PERMISSIVE", 1}, {"ENFORCED", 2}}}; -TsEnumDescriptor PROPERTIES_DESCRIPTOR = {{{"NONE", 0}, {"SIGNATURE", 0x1}, {"NAME", 0x2}, {"ALL", 0x3}}}; +void +YamlSNIConfig::Item::EnableProtocol(YamlSNIConfig::TLSProtocol proto) +{ + if (proto <= YamlSNIConfig::TLSProtocol::TLS_MAX) { + if (protocol_unset) { + protocol_mask = TLSValidProtocols::max_mask; + protocol_unset = false; + } + switch (proto) { + case YamlSNIConfig::TLSProtocol::TLSv1: + protocol_mask &= ~SSL_OP_NO_TLSv1; + break; + case YamlSNIConfig::TLSProtocol::TLSv1_1: + protocol_mask &= ~SSL_OP_NO_TLSv1_1; + break; + case YamlSNIConfig::TLSProtocol::TLSv1_2: + protocol_mask &= ~SSL_OP_NO_TLSv1_2; + break; + case YamlSNIConfig::TLSProtocol::TLSv1_3: +#ifdef SSL_OP_NO_TLSv1_3 + protocol_mask &= ~SSL_OP_NO_TLSv1_3; +#endif + break; + } + } +} + +TsEnumDescriptor LEVEL_DESCRIPTOR = {{{"NONE", 0}, {"MODERATE", 1}, {"STRICT", 2}}}; +TsEnumDescriptor POLICY_DESCRIPTOR = {{{"DISABLED", 0}, {"PERMISSIVE", 1}, {"ENFORCED", 2}}}; +TsEnumDescriptor PROPERTIES_DESCRIPTOR = {{{"NONE", 0}, {"SIGNATURE", 0x1}, {"NAME", 0x2}, {"ALL", 0x3}}}; +TsEnumDescriptor TLS_PROTOCOLS_DESCRIPTOR = {{{"TLSv1", 0}, {"TLSv1_1", 1}, {"TLSv1_2", 2}, {"TLSv1_3", 3}}}; std::set valid_sni_config_keys = {TS_fqdn, TS_disable_h2, @@ -69,7 +99,12 @@ std::set valid_sni_config_keys = {TS_fqdn, TS_verify_server_properties, TS_client_cert, TS_client_key, - TS_ip_allow}; + TS_ip_allow +#if TS_USE_HELLO_CB + , + TS_valid_tls_versions_in +#endif +}; namespace YAML { @@ -158,6 +193,13 @@ template <> struct convert { if (node[TS_ip_allow]) { item.ip_allow = node[TS_ip_allow].as(); } + if (node[TS_valid_tls_versions_in]) { + for (unsigned int i = 0; i < node[TS_valid_tls_versions_in].size(); i++) { + auto value = node[TS_valid_tls_versions_in][i].as(); + int protocol = TLS_PROTOCOLS_DESCRIPTOR.get(value); + item.EnableProtocol(static_cast(protocol)); + } + } return true; } }; diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h index 0acf4f21f21..babd0166376 100644 --- a/iocore/net/YamlSNIConfig.h +++ b/iocore/net/YamlSNIConfig.h @@ -38,6 +38,7 @@ TSDECL(verify_origin_server); TSDECL(client_cert); TSDECL(client_key); TSDECL(ip_allow); +TSDECL(valid_tls_versions_in); #undef TSDECL const int start = 0; @@ -54,6 +55,7 @@ struct YamlSNIConfig { enum class Level { NONE = 0, MODERATE, STRICT }; enum class Policy : uint8_t { DISABLED = 0, PERMISSIVE, ENFORCED, UNSET }; enum class Property : uint8_t { NONE = 0, SIGNATURE_MASK = 0x1, NAME_MASK = 0x2, ALL_MASK = 0x3, UNSET }; + enum class TLSProtocol : uint8_t { TLSv1 = 0, TLSv1_1, TLSv1_2, TLSv1_3, TLS_MAX = TLSv1_3 }; YamlSNIConfig() {} @@ -68,6 +70,10 @@ struct YamlSNIConfig { std::string client_cert; std::string client_key; std::string ip_allow; + bool protocol_unset = true; + unsigned long protocol_mask; + + void EnableProtocol(YamlSNIConfig::TLSProtocol proto); }; ts::Errata loader(const char *cfgFilename); diff --git a/tests/gold_tests/tls/tls_client_versions.test.py b/tests/gold_tests/tls/tls_client_versions.test.py new file mode 100644 index 00000000000..18557998978 --- /dev/null +++ b/tests/gold_tests/tls/tls_client_versions.test.py @@ -0,0 +1,112 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test TLS protocol offering based on SNI +''' + +# By default only offer TLSv1_2 +# for special doman foo.com only offer TLSv1 and TLSv1_1 + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work"), + Condition.HasOpenSSLVersion("1.1.1") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server", ssl=True) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "foo ok"} +server.addResponse("sessionlog.json", request_foo_header, response_foo_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.ssl.TLSv1': 0, + 'proxy.config.ssl.TLSv1_1': 0, + 'proxy.config.ssl.TLSv1_2': 1 +}) + +# foo.com should only offer the older TLS protocols +# bar.com should terminate. +# empty SNI should tunnel to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: foo.com', + ' valid_tls_versions_in: [ TLSv1, TLSv1_1 ]' +]) + +# Target foo.com for TLSv1_2. Should fail +tr = Test.AddTestRun("foo.com TLSv1_2") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 35 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ssl_choose_client_version:unsupported protocol", "Should not allow TLSv1_2") + +# Target foo.com for TLSv1. Should succeed +tr = Test.AddTestRun("foo.com TLSv1") +tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# Target bar.com for TLSv1. Should fail +tr = Test.AddTestRun("bar.com TLSv1") +tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 35 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All += Testers.ContainsExpression("alert protocol version", "Should not allow TLSv1_0") +tr.TimeOut = 5 + +# Target bar.com for TLSv1_2. Should succeed +tr = Test.AddTestRun("bar.com TLSv1_2") +tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + From f681f9adbd9aa0f05138a0bfa7caaadbe0134927 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Fri, 18 Jan 2019 20:49:12 +0000 Subject: [PATCH 174/526] Correctly deal with the ssl.client.sni_policy if not set via conf_remap --- mgmt/RecordsConfig.cc | 2 +- proxy/http/HttpConfig.cc | 4 ++++ proxy/http/HttpConfig.h | 1 + .../tls/tls_verify_override.test.py | 24 +++++++++++++++---- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 896f171f3d3..6c24bd20641 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1156,7 +1156,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.client.sni_policy", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.client.sni_policy", RECD_STRING, "host", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.session_cache", RECD_INT, "2", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index 5521233bffc..ddb4bdc86a4 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -1222,6 +1222,8 @@ HttpConfig::startup() HttpEstablishStaticConfigLongLong(c.post_copy_size, "proxy.config.http.post_copy_size"); HttpEstablishStaticConfigStringAlloc(c.redirect_actions_string, "proxy.config.http.redirect.actions"); + HttpEstablishStaticConfigStringAlloc(c.oride.ssl_client_sni_policy, "proxy.config.ssl.client.sni_policy"); + OutboundConnTrack::config_init(&c.outbound_conntrack, &c.oride.outbound_conntrack); MUTEX_TRY_LOCK(lock, http_config_cont->mutex, this_ethread()); @@ -1495,6 +1497,8 @@ HttpConfig::reconfigure() params->redirect_actions_string = ats_strdup(m_master.redirect_actions_string); params->redirect_actions_map = parse_redirect_actions(params->redirect_actions_string, params->redirect_actions_self_action); + params->oride.ssl_client_sni_policy = ats_strdup(m_master.oride.ssl_client_sni_policy); + params->negative_caching_list = m_master.negative_caching_list; m_id = configProcessor.set(m_id, params); diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index 61092e9b645..ce4739edc77 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -981,6 +981,7 @@ inline HttpConfigParams::~HttpConfigParams() ats_free(connect_ports_string); ats_free(reverse_proxy_no_host_redirect); ats_free(redirect_actions_string); + ats_free(oride.ssl_client_sni_policy); delete connect_ports; delete redirect_actions_map; diff --git a/tests/gold_tests/tls/tls_verify_override.test.py b/tests/gold_tests/tls/tls_verify_override.test.py index fd568194ed9..59608635062 100644 --- a/tests/gold_tests/tls/tls_verify_override.test.py +++ b/tests/gold_tests/tls/tls_verify_override.test.py @@ -55,6 +55,8 @@ ts.addSSLfile("ssl/signer.key") ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map http://foo.com/basictobar https://bar.com:{0}'.format(server_bar.Variables.Port)) ts.Disk.remap_config.AddLine( 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.Port)) ts.Disk.remap_config.AddLine( @@ -68,7 +70,7 @@ ts.Disk.remap_config.AddLine( 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) ts.Disk.remap_config.AddLine( - 'map /basic https://127.0.0.1:{0}'.format(server.Variables.Port)) + 'map /basic https://random.com:{0}'.format(server.Variables.Port)) ts.Disk.remap_config.AddLine( 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.Port)) ts.Disk.remap_config.AddLine( @@ -103,11 +105,13 @@ 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', 'proxy.config.url_remap.pristine_host_hdr': 1, 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), - 'proxy.config.dns.resolv_conf': 'NULL' + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.ssl.client.sni_policy': 'remap' }) dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) +dns.addRecords(records={"random.com.": ["127.0.0.1"]}) # Should succeed without message tr = Test.AddTestRun("default-permissive-success") @@ -149,6 +153,15 @@ tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.TimeOut = 5 +tr3 = Test.AddTestRun("default-foo-to-bar") +tr3.Processes.Default.Command = "curl -k -v -H \"host: foo.com\" http://127.0.0.1:{0}/basictobar".format(ts.Variables.port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +# Should succeed. No error messages +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr3.Processes.Default.TimeOut = 5 + tr3 = Test.AddTestRun("override-foo") tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/override".format(ts.Variables.port) tr3.ReturnCode = 0 @@ -220,16 +233,17 @@ tr.Processes.Default.TimeOut = 5 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") - # Over riding the built in ERROR check since we expect some cases to fail # checks on random.com should fail with message only -ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=127.0.0.1\(127.0.0.1\) depth=0", "Warning for self signed certificate") +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=random.com\(127.0.0.1\) depth=0", "Warning for self signed certificate") # permissive failure for bar.com ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", "Warning on missing name for bar.com") # name check failure for random.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", "Warning on missing name for randome.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=random.com\(127.0.0.1\)", "Warning on missing name for randome.com") # name check failure for bar.com ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", "Failure on missing name for bar.com") +# See if the explicitly set default sni_policy of remap works. +ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI \(foo.com\) not in certificate. Action=Continue", "Warning on missing name for foo.com") From 600875f76079d35f5da9a6d307a09bfb4b526935 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 17 Jan 2019 21:40:09 +0000 Subject: [PATCH 175/526] Add test cases to exercise the verify.server* defaults. --- tests/gold_tests/tls/tls_verify3.test.py | 14 +- tests/gold_tests/tls/tls_verify_base.test.py | 144 +++++++++++ .../tls/tls_verify_override_base.test.py | 234 ++++++++++++++++++ 3 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 tests/gold_tests/tls/tls_verify_base.test.py create mode 100644 tests/gold_tests/tls/tls_verify_override_base.test.py diff --git a/tests/gold_tests/tls/tls_verify3.test.py b/tests/gold_tests/tls/tls_verify3.test.py index 222222d9bf2..c12c787544c 100644 --- a/tests/gold_tests/tls/tls_verify3.test.py +++ b/tests/gold_tests/tls/tls_verify3.test.py @@ -114,6 +114,16 @@ tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.TimeOut = 5 +tr = Test.AddTestRun("my.foo.com Permissive-Test log failure") +tr.Processes.Default.Command = "curl -v -k --resolve 'my.foo.com:{0}:127.0.0.1' https://my.foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.TimeOut = 5 + + tr2 = Test.AddTestRun("bob.bar.com Override-enforcing-Test") tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 @@ -143,5 +153,5 @@ # Over riding the built in ERROR check since we expect tr3 to fail -ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bob.bar.com\) not in certificate", "Make sure bob.bar name checked failed.") +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: SNI \(bob.bar.com\) not in certificate", "Make sure bob.bar name checked failed.") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(my.foo.com\). Action=Continue", "Make sure default permissive action takes") diff --git a/tests/gold_tests/tls/tls_verify_base.test.py b/tests/gold_tests/tls/tls_verify_base.test.py new file mode 100644 index 00000000000..16a2d303602 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_base.test.py @@ -0,0 +1,144 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: badfoo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: badbar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server': 2, + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.ssl.client.sni_policy': 'host' +}) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bad_bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL' +]) + +tr = Test.AddTestRun("Permissive-Test") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = "curl -v -k -H \"host: foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr = Test.AddTestRun("Permissive-Test with logged failure") +tr.Processes.Default.Command = "curl -v -k -H \"host: random.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + + +tr2 = Test.AddTestRun("Override-enforcing-Test") +tr2.Processes.Default.Command = "curl -v -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.TimeOut = 5 +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr3 = Test.AddTestRun("Override-enforcing-Test-fail-name-check") +tr3.Processes.Default.Command = "curl -v -k -H \"host: bad_bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +tr3.Processes.Default.TimeOut = 5 + +# Over riding the built in ERROR check since we expect tr3 to fail +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: SNI \(bad_bar.com\) not in certificate. Action=Terminate", "Make sure bad_bar name checked failed.") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue ", "Permissive failure for random") diff --git a/tests/gold_tests/tls/tls_verify_override_base.test.py b/tests/gold_tests/tls/tls_verify_override_base.test.py new file mode 100644 index 00000000000..404062e6660 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_override_base.test.py @@ -0,0 +1,234 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options. Exercise conf_remap +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) + +dns = Test.MakeDNServer("dns") + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /basic https://127.0.0.1:{0}'.format(server.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server' : 2, + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL' +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + +# Should succeed without message +tr = Test.AddTestRun("default-permissive-success") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = 'curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/basic'.format(ts.Variables.port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +# Should succed. No message +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.TimeOut = 5 + +tr2 = Test.AddTestRun("default-permissive-fail") +tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about name mismatch +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.TimeOut = 5 + +tr2 = Test.AddTestRun("default-permissive-fail2") +tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about signature +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.TimeOut = 5 + +tr3 = Test.AddTestRun("override-foo") +tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/override".format(ts.Variables.port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +# Should succeed. No error messages +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr3.Processes.Default.TimeOut = 5 + +tr4 = Test.AddTestRun("override-bar-disabled") +tr4.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" http://127.0.0.1:{0}/overridedisabled".format(ts.Variables.port) +tr4.ReturnCode = 0 +tr4.StillRunningAfter = server +tr4.StillRunningAfter = ts +# Succeed. No error messages +tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr4.Processes.Default.TimeOut = 5 + +tr5 = Test.AddTestRun("override-bar-signature-enforced") +tr5.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overridesignature".format(ts.Variables.port) +tr5.ReturnCode = 0 +tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr5.StillRunningAfter = server +tr5.StillRunningAfter = ts +tr5.Processes.Default.TimeOut = 5 + +tr6 = Test.AddTestRun("override-bar-enforced") +tr6.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overrideenforced".format(ts.Variables.port) +tr6.ReturnCode = 0 +# Should fail +tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") +tr6.StillRunningAfter = server +tr6.StillRunningAfter = ts +tr6.Processes.Default.TimeOut = 5 + +# Should succeed +tr = Test.AddTestRun("foo-to-bar-sni-policy-remap") +tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/snipolicybarremap".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") + +# Should fail +tr = Test.AddTestRun("foo-to-bar-sni-policy-host") +tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/snipolicybarhost".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") + +# Should fail +tr = Test.AddTestRun("bar-to-foo-sni-policy-remap") +tr.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/snipolicyfooremap".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") + +# Should succeed +tr = Test.AddTestRun("bar-to-foo-sni-policy-host") +tr.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/snipolicyfoohost".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") + + +# Over riding the built in ERROR check since we expect some cases to fail + +# checks on random.com should fail with message only +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=127.0.0.1\(127.0.0.1\) depth=0", "Warning for self signed certificate") +# permissive failure for bar.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", "Warning on missing name for bar.com") +# name check failure for random.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", "Warning on missing name for randome.com") +# name check failure for bar.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", "Failure on missing name for bar.com") + + From de350a112d6a311c4b05b5db464948d1a859c7aa Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Tue, 22 Jan 2019 11:13:56 -0600 Subject: [PATCH 176/526] doc for option -F --- doc/appendices/command-line/traffic_server.en.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/appendices/command-line/traffic_server.en.rst b/doc/appendices/command-line/traffic_server.en.rst index 247dfca2fe6..75e17e8f8f2 100644 --- a/doc/appendices/command-line/traffic_server.en.rst +++ b/doc/appendices/command-line/traffic_server.en.rst @@ -52,8 +52,7 @@ environments or where the working set is highly variable. .. option:: -F, --disable_pfreelist Disable free list in ProxyAllocator which were left out by the -f -option. Please note that this option is a temporary, testing -option, and will be removed in the future. +option. This option includes the functionality of :option:`-f`. .. option:: -R LEVEL, --regression LEVEL From 63c05aec6c7eb51a1e8c6c963af88ae10bb604c3 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Mon, 21 Jan 2019 17:06:09 -0800 Subject: [PATCH 177/526] Doc: minor records.config cleaning - Removes Fuzz options that left in ATS 7 - Fixes parsing of proxy.config.hostdb.round_robin_max_count --- doc/admin-guide/files/records.config.en.rst | 51 +-------------------- 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index ab814014f3c..b1a86b35915 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -2281,55 +2281,6 @@ Heuristic Expiration aging factor is applied, the final maximum age calculated will never be higher than the value in this variable. -.. ts:cv:: CONFIG proxy.config.http.cache.fuzz.time INT 0 - :deprecated: - :reloadable: - :overridable: - - How often |TS| checks for an early refresh, during the period before the - document stale time. The interval specified must be in seconds. - -.. note:: - - Previous versions of Apache |TS| defaulted this to 240s. This - feature is deprecated as of ATS v6.2.0. - -.. ts:cv:: CONFIG proxy.config.http.cache.fuzz.probability FLOAT 0.0 - :deprecated: - :reloadable: - :overridable: - - The probability that a refresh is made on a document during the fuzz time - specified in :ts:cv:`proxy.config.http.cache.fuzz.time`. - -.. note:: - - Previous versions of Apache |TS| defaulted this to 0.005 (0.5%). - This feature is deprecated as of ATS v6.2.0 - -.. ts:cv:: CONFIG proxy.config.http.cache.fuzz.min_time INT 0 - :deprecated: - :reloadable: - :overridable: - - Handles requests with a TTL less than :ts:cv:`proxy.config.http.cache.fuzz.time`. - It allows for different times to evaluate the probability of revalidation - for small TTLs and big TTLs. Objects with small TTLs will start "rolling the - revalidation dice" near the ``fuzz.min_time``, while objects with large TTLs - would start at ``fuzz.time``. A logarithmic-like function between determines - the revalidation evaluation start time (which will be between - ``fuzz.min_time`` and ``fuzz.time``). As the object gets closer to expiring, - the window start becomes more likely. By default this setting is not enabled, - but should be enabled any time you have objects with small TTLs. - -.. note:: - - These fuzzing options are marked as deprecated as of v6.2.0, and will be - removed for v7.0.0. Instead, we recommend looking at the new - :ts:cv:`proxy.config.http.cache.open_write_fail_action` configuration and - the features around thundering heard avoidance (see - :ref:`http-proxy-caching` for details). - Dynamic Content & Content Negotiation ===================================== @@ -2602,7 +2553,7 @@ HostDB For values above ``200000``, you must increase :ts:cv:`proxy.config.hostdb.max_size` by at least 44 bytes per entry. -.. ts:cv:: proxy.config.hostdb.round_robin_max_count INT 16 +.. ts:cv:: CONFIG proxy.config.hostdb.round_robin_max_count INT 16 The maximum count of DNS answers per round robin hostdb record. The default variable is 16. From aaa9aaabfb53167358081967914aa7b01745cc63 Mon Sep 17 00:00:00 2001 From: Jesse Zhang Date: Fri, 18 Jan 2019 15:19:03 -0600 Subject: [PATCH 178/526] Fix potential instability in cacheIMSRange Microserver may terminate early. --- tests/gold_tests/headers/cachedIMSRange.test.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/gold_tests/headers/cachedIMSRange.test.py b/tests/gold_tests/headers/cachedIMSRange.test.py index b17a386eb89..da93f95be3d 100644 --- a/tests/gold_tests/headers/cachedIMSRange.test.py +++ b/tests/gold_tests/headers/cachedIMSRange.test.py @@ -81,11 +81,12 @@ # Test 0 - Fill a 3 byte object with Last-Modified time into cache. tr = Test.AddTestRun() tr.Processes.Default.StartBefore(server) -tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.StartBefore(ts, ready=1) tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: Fill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-miss.gold" tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 1 - Once it goes stale, fetch it again. We expect Origin to get IMS request, and serve a 304. We expect ATS to refresh the object, and give a 200 to user tr = Test.AddTestRun() @@ -94,6 +95,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale.gold" tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 2 - Once it goes stale, fetch it via a range request. We expect Origin to get IMS request, and serve a 304. We expect ATS to refresh the object, and give a 206 to user tr = Test.AddTestRun() @@ -102,12 +104,14 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-206.gold" tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 3 - Fill a new object with an Etag. Not checking the output here. tr = Test.AddTestRun() tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagFill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 4 - Once the etag object goes stale, fetch it again. We expect Origin to get INM request, and serve a 304. We expect ATS to refresh the object, and give a 200 to user tr = Test.AddTestRun() @@ -116,6 +120,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-INM.gold" tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 5 - Once the etag object goes stale, fetch it via a range request. We expect Origin to get INM request, and serve a 304. We expect ATS to refresh the object, and give a 206 to user tr = Test.AddTestRun() @@ -124,6 +129,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-206-etag.gold" tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 6 - The origin changes the initial LMT object to 0 byte. We expect ATS to fetch and serve the new 0 byte object. tr = Test.AddTestRun() @@ -132,6 +138,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_nobody-hit-stale.gold" tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 7 - Fetch the new 0 byte object again when fresh in cache to ensure its still a 0 byte object. tr = Test.AddTestRun() @@ -140,6 +147,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_nobody-hit-stale.gold" tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 8 - The origin changes the etag object to 0 byte 404. We expect ATS to fetch and serve the 404 0 byte object. tr = Test.AddTestRun() @@ -148,6 +156,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_error_nobody.gold" tr.StillRunningAfter = ts +tr.StillRunningAfter = server # Test 9 - Fetch the 0 byte etag object again when fresh in cache to ensure its still a 0 byte object tr = Test.AddTestRun() @@ -155,4 +164,5 @@ tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_error_nobody.gold" -tr.StillRunningAfter = ts \ No newline at end of file +tr.StillRunningAfter = ts +tr.StillRunningAfter = server From fb3eb6913471c0915716c658335b1bdf3b155cba Mon Sep 17 00:00:00 2001 From: scw00 Date: Mon, 21 Jan 2019 20:15:32 +0800 Subject: [PATCH 179/526] Fixes heap-use-after-free in RangeTransform --- proxy/Transform.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/proxy/Transform.cc b/proxy/Transform.cc index 516f8e94b75..37129d023e8 100644 --- a/proxy/Transform.cc +++ b/proxy/Transform.cc @@ -782,7 +782,11 @@ RangeTransform::handle_event(int event, void *edata) if (m_closed) { if (m_deletable) { - Debug("http_trans", "RangeTransform destroy: %p ndone=%" PRId64, this, m_output_vio ? m_output_vio->ndone : 0); + if (m_output_vc != nullptr) { + Debug("http_trans", "RangeTransform destroy: %p ndone=%" PRId64, this, m_output_vio ? m_output_vio->ndone : 0); + } else { + Debug("http_trans", "RangeTransform destroy"); + } delete this; } } else { From 04517790aba9e6dcd1228e9e68a26c1fe28dfad7 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Wed, 16 Jan 2019 22:09:31 +0000 Subject: [PATCH 180/526] Create an autest for the regex_revalidate plugin --- .../gold/regex_reval-hit.gold | 10 + .../gold/regex_reval-miss.gold | 10 + .../gold/regex_reval-stale.gold | 10 + .../regex_revalidate/regex_revalidate.test.py | 273 ++++++++++++++++++ 4 files changed, 303 insertions(+) create mode 100644 tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold create mode 100644 tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold create mode 100644 tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold create mode 100644 tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold new file mode 100644 index 00000000000..f98f8ac02c3 --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: hit-fresh +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold new file mode 100644 index 00000000000..35ec6036eff --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: miss +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold new file mode 100644 index 00000000000..ea29316b964 --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: hit-stale +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py new file mode 100644 index 00000000000..bb2aa96fe6e --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py @@ -0,0 +1,273 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Test a basic regex_revalidate +''' + +## Test description: +# Load up cache, ensure fresh +# Create regex reval rule, config reload: +# ensure item is staled only once. +# Add a new rule, config reload: +# ensure item isn't restaled again, but rule still in effect. +# +# If the rule disappears from regex_revalidate.conf its still loaded!! +# A rule's expiry can't be changed after the fact! + +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work"), + Condition.PluginExists('regex_revalidate.so'), + Condition.PluginExists('xdebug.so') +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager") + +#**testname is required** +#testName = "regex_reval" + +# default root +request_header_0 = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_0 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Cache-Control: max-age=300\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "xxx", +} + +# cache item path1 +request_header_1 = {"headers": + "GET /path1 HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" +} +response_header_1 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path1"\r\n' + + "Cache-Control: max-age=600,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "abc" +} + +# cache item path1a +request_header_2 = {"headers": + "GET /path1a HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" +} +response_header_2 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path1a"\r\n' + + "Cache-Control: max-age=600,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "cde" +} + +# cache item path2a +request_header_3 = {"headers": + "GET /path2a HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" +} +response_header_3 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path2a"\r\n' + + "Cache-Control: max-age=900,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "efg" +} + +server.addResponse("sessionlog.json", request_header_0, response_header_0) +server.addResponse("sessionlog.json", request_header_1, response_header_1) +server.addResponse("sessionlog.json", request_header_2, response_header_2) +server.addResponse("sessionlog.json", request_header_3, response_header_3) + +# Configure ATS server +ts.Disk.plugin_config.AddLine('xdebug.so') +ts.Disk.plugin_config.AddLine( + 'regex_revalidate.so -d -c regex_revalidate.conf' +) + +regex_revalidate_conf_path = os.path.join(ts.Variables.CONFIGDIR, 'regex_revalidate.conf') +curl_and_args = 'curl -s -D - -v -H "x-debug: x-cache" -H "Host: www.example.com"' + +path1_rule = 'path1 {}\n'.format(int(time.time()) + 600) + +# Define first revistion for when trafficserver starts +ts.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLines([ + "# Empty\n" +]) + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) +) + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'regex_revalidate', +# 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.response_via_str': 3, + 'proxy.config.http.server_ports': '{}'.format(ts.Variables.port), +}) + +# 0 Test - Load cache (miss) (path1) +tr = Test.AddTestRun("Cache miss path1") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-miss.gold" +tr.StillRunningAfter = ts + +# 1 Test - Load cache (miss) for later test (path1a) +tr = Test.AddTestRun("Cache miss path1a") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1a'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-miss.gold" +tr.StillRunningAfter = ts + +# 2 Test - Load cache (miss) for later test (path2a) +tr = Test.AddTestRun("Cache miss path2a") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path2a'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-miss.gold" +tr.StillRunningAfter = ts + +# 3 Test - Cache hit path1 +tr = Test.AddTestRun("Cache hit fresh path1") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-hit.gold" +tr.StillRunningAfter = ts + +# 4 Stage - Reload new regex_revalidate +tr = Test.AddTestRun("Reload config add path1") +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLines([ + path1_rule +]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 5 Test - Revalidate path1 +tr = Test.AddTestRun("Revalidate stale path1") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-stale.gold" +tr.StillRunningAfter = ts + +# 6 Test - Cache hit (path1) +tr = Test.AddTestRun("Cache hit fresh path1") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-hit.gold" +tr.StillRunningAfter = ts + +# 7 Stage - Reload new regex_revalidate +tr = Test.AddTestRun("Reload config add path2") +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLines([ + path1_rule, + 'path2 {}\n'.format(int(time.time()) + 700) +]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 8 Test - Cache hit (path1) +tr = Test.AddTestRun("Cache hit fresh path1") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-hit.gold" +tr.StillRunningAfter = ts + +# 9 Test - Cache stale (check rule is still loaded) (path1a) +tr = Test.AddTestRun("Revalidate stale path1a") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1a'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-stale.gold" +tr.StillRunningAfter = ts + +# The C version of regex_revalidate doesn't allow an existing rule to +# be changed by a reload. + +# 10 Stage - regex_revalidate rewrite rule early expire +tr = Test.AddTestRun("Reload config change path2") +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLines([ + path1_rule, + 'path2 {}\n'.format(int(time.time()) - 100), +]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 11 Test - Cache hit (path2a) +tr = Test.AddTestRun("Cache hit stale path2a") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path2a'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-stale.gold" +tr.StillRunningAfter = ts From ad32dd243e6ffc5a01f4e1b6f6889dcd779b4d54 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 17 Jan 2019 12:35:27 -0700 Subject: [PATCH 181/526] Removes proxy.config.http.parse.allow_non_http In addition, it changes the default / code such that we do not allow parse failues on the HTTP/ version parsing. --- mgmt/RecordsConfig.cc | 2 -- proxy/hdrs/HTTP.cc | 19 +++---------------- proxy/hdrs/HTTP.h | 3 +-- proxy/http/HttpConfig.cc | 2 -- proxy/http/HttpConfig.h | 1 - proxy/http/HttpSM.cc | 1 - 6 files changed, 4 insertions(+), 24 deletions(-) diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 6c24bd20641..b8e5e53c77a 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -299,8 +299,6 @@ static const RecordElement RecordsConfig[] = // ########### {RECT_CONFIG, "proxy.config.header.parse.no_host_url_redirect", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_STR, ".*", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http.parse.allow_non_http", RECD_INT, "1", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , //############################################################################## //# diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 3858bd0cafa..dabbc4ec29e 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -834,15 +834,6 @@ http_hdr_reason_lookup(unsigned status) return nullptr; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -void -_http_parser_init(HTTPParser *parser) -{ - parser->m_parsing_http = true; -} - ////////////////////////////////////////////////////// // init first time structure setup // // clear resets an already-initialized structure // @@ -851,14 +842,14 @@ _http_parser_init(HTTPParser *parser) void http_parser_init(HTTPParser *parser) { - _http_parser_init(parser); + parser->m_parsing_http = true; mime_parser_init(&parser->m_mime_parser); } void http_parser_clear(HTTPParser *parser) { - _http_parser_init(parser); + parser->m_parsing_http = true; mime_parser_clear(&parser->m_mime_parser); } @@ -1384,11 +1375,7 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const eoh: *start = old_start; - if (parser->m_allow_non_http) { - return PARSE_RESULT_DONE; - } else { - return PARSE_RESULT_ERROR; - } + return PARSE_RESULT_ERROR; // This used to return PARSE_RESULT_DONE by default before done: if (!version_start || !version_end) { diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h index 682a65a3557..e9718c3e18b 100644 --- a/proxy/hdrs/HTTP.h +++ b/proxy/hdrs/HTTP.h @@ -322,8 +322,7 @@ struct HTTPValTE { }; struct HTTPParser { - bool m_parsing_http = false; - bool m_allow_non_http = false; + bool m_parsing_http = false; MIMEParser m_mime_parser; }; diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index ddb4bdc86a4..1e892ef5826 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -1152,7 +1152,6 @@ HttpConfig::startup() HttpEstablishStaticConfigByte(c.send_100_continue_response, "proxy.config.http.send_100_continue_response"); HttpEstablishStaticConfigByte(c.disallow_post_100_continue, "proxy.config.http.disallow_post_100_continue"); - HttpEstablishStaticConfigByte(c.parser_allow_non_http, "proxy.config.http.parse.allow_non_http"); HttpEstablishStaticConfigByte(c.keepalive_internal_vc, "proxy.config.http.keepalive_internal_vc"); @@ -1439,7 +1438,6 @@ HttpConfig::reconfigure() params->send_100_continue_response = INT_TO_BOOL(m_master.send_100_continue_response); params->disallow_post_100_continue = INT_TO_BOOL(m_master.disallow_post_100_continue); - params->parser_allow_non_http = INT_TO_BOOL(m_master.parser_allow_non_http); params->keepalive_internal_vc = INT_TO_BOOL(m_master.keepalive_internal_vc); params->oride.cache_open_write_fail_action = m_master.oride.cache_open_write_fail_action; diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index ce4739edc77..e7aae44d6a4 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -904,7 +904,6 @@ struct HttpConfigParams : public ConfigInfo { MgmtByte send_100_continue_response = 0; MgmtByte disallow_post_100_continue = 0; - MgmtByte parser_allow_non_http = 1; MgmtByte keepalive_internal_vc = 0; MgmtByte server_session_sharing_pool = TS_SERVER_SESSION_SHARING_POOL_THREAD; diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index f59a2c20548..0a0dea4f73b 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -341,7 +341,6 @@ HttpSM::init() t_state.force_dns = (ip_rule_in_CacheControlTable() || t_state.parent_params->parent_table->ipMatch || !(t_state.txn_conf->doc_in_cache_skip_dns) || !(t_state.txn_conf->cache_http)); - http_parser.m_allow_non_http = t_state.http_config_param->parser_allow_non_http; http_parser_init(&http_parser); SET_HANDLER(&HttpSM::main_handler); From 854324af36f615a71ef4265f4a7c6ee8f40f42cd Mon Sep 17 00:00:00 2001 From: Shinya Kawano Date: Thu, 17 Jan 2019 15:56:10 +0900 Subject: [PATCH 182/526] Fix ATS does not honor proxy.config.body_factory.response_max_size --- proxy/http/HttpTransact.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index d97a516f5a4..aba6edcad4f 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -7979,9 +7979,9 @@ HttpTransact::build_error_response(State *s, HTTPStatus status_code, const char int64_t len; char *new_msg; - new_msg = body_factory->fabricate_with_old_api(error_body_type, s, 8192, &len, body_language, sizeof(body_language), body_type, - sizeof(body_type), s->internal_msg_buffer_size, - s->internal_msg_buffer_size ? s->internal_msg_buffer : nullptr); + new_msg = body_factory->fabricate_with_old_api( + error_body_type, s, s->http_config_param->body_factory_response_max_size, &len, body_language, sizeof(body_language), body_type, + sizeof(body_type), s->internal_msg_buffer_size, s->internal_msg_buffer_size ? s->internal_msg_buffer : nullptr); // After the body factory is called, a new "body" is allocated, and we must replace it. It is // unfortunate that there's no way to avoid this fabrication even when there is no substitutions... From c7efd7cb07098685505ba882a2a2c8fc9189611c Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 22 Jan 2019 11:00:35 -0700 Subject: [PATCH 183/526] Initialize redir_len, compiler warnings It turns out, t_state.hdr_info.client_response.value_get() can actually fail, at which point, redir_len is undefined. I don't know if this happens in reality though, but has made wrong assumptions. --- proxy/http/HttpSM.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 0a0dea4f73b..8cdb22585ac 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -7695,7 +7695,7 @@ HttpSM::do_redirect() HTTP_INCREMENT_DYN_STAT(http_total_x_redirect_stat); } else { // get the location header and setup the redirect - int redir_len; + int redir_len = 0; char *redir_url = (char *)t_state.hdr_info.client_response.value_get(MIME_FIELD_LOCATION, MIME_LEN_LOCATION, &redir_len); redirect_request(redir_url, redir_len); } From f29d68f9b1fd8f43d3915c2ba00f7d695fb14a9f Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Wed, 2 Jan 2019 15:46:01 -0600 Subject: [PATCH 184/526] Add more information about event data to 'hook add' API function documentation. --- doc/developer-guide/api/functions/TSHttpHookAdd.en.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst index d21bb4b7ce8..3019be013be 100644 --- a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst @@ -77,9 +77,11 @@ function will be TS_EVENT_HTTP_READ_REQUEST_HDR. When a continuation is triggered by a hook, the actual type of the event data (the void pointer passed as the third parameter to the continuation function) is determined by which hook it is. For example, for the hook ID TS_HTTP_TXN_CLOSE_HOOK, -the event data is of type TSHttpTxn. This is the case regardless of whether the +the event data is of type :type:`TSHttpTxn`. This is the case regardless of whether the continuation was added to the hook using :func:`TSHttpTxnHookAdd`, :func:`TSHttpSsnHookAdd` -or :func:`TSHttpHookAdd`. +or :func:`TSHttpHookAdd`. If the event data is of type :type:`TSHttpTxn`, :type:`TSHttpSsn` or +:type:`TSVConn`, the continuation function can assume the mutex of the indicated +event data object is locked. (But the continuation function must not unlock it.) Return Values ============= From 54c38b4192647d838d2326be4fb98fc17fc4295c Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 22 Jan 2019 12:17:06 -0600 Subject: [PATCH 185/526] Fix initialization style. --- proxy/logging/LogUtils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/logging/LogUtils.cc b/proxy/logging/LogUtils.cc index 22359ebd24a..49587c48992 100644 --- a/proxy/logging/LogUtils.cc +++ b/proxy/logging/LogUtils.cc @@ -664,7 +664,7 @@ unmarshalMimeHdr(char **buf, char *dest, int destLength) { ink_assert(*buf != nullptr); - const char *data{*buf}; + const char *data = *buf; ink_assert(data != nullptr); From 55112bb9371e24719c5e704bf894e91bba6d1c02 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 22 Jan 2019 22:18:08 +0000 Subject: [PATCH 186/526] Inrease the per test case timeout from 5 to 10 seconds --- tests/gold_tests/tls/tls.test.py | 3 +- .../tls/tls_check_cert_selection.test.py | 10 ++--- tests/gold_tests/tls/tls_client_cert.test.py | 45 +++++++------------ tests/gold_tests/tls/tls_client_cert2.test.py | 18 +++----- .../tls/tls_client_cert_override.test.py | 12 ++--- .../gold_tests/tls/tls_client_verify.test.py | 36 +++++---------- .../gold_tests/tls/tls_client_verify2.test.py | 33 +++++--------- .../tls/tls_client_versions.test.py | 12 ++--- .../tls/tls_forward_nonhttp.test.py | 3 +- tests/gold_tests/tls/tls_keepalive.test.py | 3 +- tests/gold_tests/tls/tls_ticket.test.py | 5 +-- tests/gold_tests/tls/tls_tunnel.test.py | 24 ++++------ .../gold_tests/tls/tls_tunnel_forward.test.py | 9 ++-- .../tls/tls_tunnel_plugin_rename.test.py | 3 +- tests/gold_tests/tls/tls_verify.test.py | 8 ++-- tests/gold_tests/tls/tls_verify2.test.py | 14 +++--- tests/gold_tests/tls/tls_verify3.test.py | 15 +++---- tests/gold_tests/tls/tls_verify_base.test.py | 11 ++--- .../tls/tls_verify_ca_override.test.py | 11 ++--- .../tls/tls_verify_not_pristine.test.py | 6 +-- .../tls/tls_verify_override.test.py | 27 +++++------ .../tls/tls_verify_override_base.test.py | 25 +++++------ 22 files changed, 121 insertions(+), 212 deletions(-) diff --git a/tests/gold_tests/tls/tls.test.py b/tests/gold_tests/tls/tls.test.py index f23232a0398..58ffdaddabb 100644 --- a/tests/gold_tests/tls/tls.test.py +++ b/tests/gold_tests/tls/tls.test.py @@ -86,5 +86,4 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.Processes.Default.Streams.stdout = "gold/ssl-post.gold" tr.StillRunningAfter = server -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 diff --git a/tests/gold_tests/tls/tls_check_cert_selection.test.py b/tests/gold_tests/tls/tls_check_cert_selection.test.py index b029af5cde0..92e52c41252 100644 --- a/tests/gold_tests/tls/tls_check_cert_selection.test.py +++ b/tests/gold_tests/tls/tls_check_cert_selection.test.py @@ -84,32 +84,30 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=bar.com", "Cert should contain bar.com") tr.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=foo.com", "Cert should not contain foo.com") tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") -tr.TimeOut = 5 # Should receive a foo.com cert tr2 = Test.AddTestRun("foo.com cert") tr2.Processes.Default.Command = "curl -v --cacert signer.pem --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=foo.com", "Cert should contain foo.com") tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=bar.com", "Cert should not contain bar.com") tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") -tr2.TimeOut = 5 # Should receive random.server.com tr2 = Test.AddTestRun("random.server.com cert") tr2.Processes.Default.Command = "curl -v -k --resolve 'random.server.com:{0}:127.0.0.1' https://random.server.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=random.server.com", "Cert should contain random.server.com") @@ -123,7 +121,7 @@ tr2.Processes.Default.Command = "curl -v -k --cacert signer.pem --resolve 'bad.sni.com:{0}:127.0.0.1' https://bad.sni.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=foo.com", "Cert should contain foo.com") diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index e48bbe2621a..c03e4b882a4 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -107,9 +107,8 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:example.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr.TimeOut = 5 #Should fail trfail = Test.AddTestRun("Connect with first client cert to second server") @@ -118,9 +117,8 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trfail.TimeOut = 5 # Should succeed trbar = Test.AddTestRun("Connect with signed2 bar to second server") @@ -129,9 +127,8 @@ trbar.StillRunningAfter = server2 trbar.Processes.Default.Command = "curl -H host:bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) trbar.Processes.Default.ReturnCode = 0 -trbar.Processes.Default.TimeOut = 5 +trbar.Processes.Default.TimeOut = 10 trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -trbar.TimeOut = 5 #Should fail trbarfail = Test.AddTestRun("Connect with signed2 bar cert to first server") @@ -140,9 +137,8 @@ trbarfail.StillRunningAfter = server2 trbarfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) trbarfail.Processes.Default.ReturnCode = 0 -trbarfail.Processes.Default.TimeOut = 5 +trbarfail.Processes.Default.TimeOut = 10 trbarfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trbarfail.TimeOut = 5 tr2 = Test.AddTestRun("Update config files") # Update the SNI config @@ -178,8 +174,7 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket tr2.Processes.Default.Env = ts.Env tr2.Processes.Default.ReturnCode = 0 -tr2.Processes.Default.TimeOut = 5 -tr2.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2reload = Test.AddTestRun("Reload config") tr2reload.StillRunningAfter = ts @@ -189,8 +184,7 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket tr2reload.Processes.Default.Env = ts.Env tr2reload.Processes.Default.ReturnCode = 0 -tr2reload.Processes.Default.TimeOut = 5 -tr2reload.TimeOut = 5 +tr2reload.Processes.Default.TimeOut = 10 #Should succeed @@ -202,9 +196,8 @@ tr3bar.StillRunningAfter = server2 tr3bar.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr3bar.Processes.Default.ReturnCode = 0 -tr3bar.Processes.Default.TimeOut = 5 +tr3bar.Processes.Default.TimeOut = 10 tr3bar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr3bar.TimeOut = 5 #Should fail tr3barfail = Test.AddTestRun("Make request with other bar cert to second server") @@ -213,9 +206,8 @@ tr3barfail.StillRunningAfter = server2 tr3barfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr3barfail.Processes.Default.ReturnCode = 0 -tr3barfail.Processes.Default.TimeOut = 5 +tr3barfail.Processes.Default.TimeOut = 10 tr3barfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -tr3barfail.TimeOut = 5 #Should succeed tr3 = Test.AddTestRun("Make request with other cert to second server") @@ -225,9 +217,8 @@ tr3.StillRunningAfter = server2 tr3.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr3.Processes.Default.ReturnCode = 0 -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr3.TimeOut = 5 #Should fail tr3fail = Test.AddTestRun("Make request with other cert to first server") @@ -236,9 +227,8 @@ tr3fail.StillRunningAfter = server2 tr3fail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr3fail.Processes.Default.ReturnCode = 0 -tr3fail.Processes.Default.TimeOut = 5 +tr3fail.Processes.Default.TimeOut = 10 tr3fail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -tr3fail.TimeOut = 5 # Test the case of updating certificate contents without changing file name. @@ -254,7 +244,7 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket trupdate.Processes.Default.Env = ts.Env trupdate.Processes.Default.ReturnCode = 0 -trupdate.Processes.Default.TimeOut = 5 +trupdate.Processes.Default.TimeOut = 10 trreload = Test.AddTestRun("Reload config after renaming certs") trreload.StillRunningAfter = ts @@ -263,7 +253,7 @@ trreload.Processes.Default.Command = 'traffic_ctl config reload' trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 -trreload.Processes.Default.TimeOut = 5 +trreload.Processes.Default.TimeOut = 10 #Should succeed tr4bar = Test.AddTestRun("Make request with renamed bar cert to second server") @@ -274,9 +264,8 @@ tr4bar.StillRunningAfter = server2 tr4bar.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr4bar.Processes.Default.ReturnCode = 0 -tr4bar.Processes.Default.TimeOut = 5 +tr4bar.Processes.Default.TimeOut = 10 tr4bar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr4bar.TimeOut = 5 #Should fail tr4barfail = Test.AddTestRun("Make request with renamed bar cert to first server") @@ -285,7 +274,7 @@ tr4barfail.StillRunningAfter = server2 tr4barfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr4barfail.Processes.Default.ReturnCode = 0 -tr4barfail.Processes.Default.TimeOut = 5 +tr4barfail.Processes.Default.TimeOut = 10 tr4barfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") #Should succeed @@ -295,9 +284,8 @@ tr4.StillRunningAfter = server2 tr4.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr4.Processes.Default.ReturnCode = 0 -tr4.Processes.Default.TimeOut = 5 +tr4.Processes.Default.TimeOut = 10 tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr4.TimeOut = 5 #Should fail tr4fail = Test.AddTestRun("Make request with renamed foo cert to second server") @@ -306,7 +294,6 @@ tr4fail.StillRunningAfter = server2 tr4fail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr4fail.Processes.Default.ReturnCode = 0 -tr4fail.Processes.Default.TimeOut = 5 +tr4fail.Processes.Default.TimeOut = 10 tr4fail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -tr4fail.TimeOut = 5 diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py index 0c085fd042a..f50d33051b7 100644 --- a/tests/gold_tests/tls/tls_client_cert2.test.py +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -108,9 +108,8 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:bob.bar.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr.TimeOut = 5 #Should fail trfail = Test.AddTestRun("bob.bar.com to server 2") @@ -119,9 +118,8 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:bob.bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trfail.TimeOut = 5 # Should succeed tr = Test.AddTestRun("bob.foo.com to server 1") @@ -130,9 +128,8 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:bob.foo.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr.TimeOut = 5 #Should fail trfail = Test.AddTestRun("bob.foo.com to server 2") @@ -141,9 +138,8 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:bob.foo.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trfail.TimeOut = 5 # Should succeed tr = Test.AddTestRun("random.bar.com to server 2") @@ -152,9 +148,8 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:random.bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr.TimeOut = 5 #Should fail trfail = Test.AddTestRun("random.bar.com to server 1") @@ -163,7 +158,6 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:random.bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trfail.TimeOut = 5 diff --git a/tests/gold_tests/tls/tls_client_cert_override.test.py b/tests/gold_tests/tls/tls_client_cert_override.test.py index e5d21b8f442..70be6131b5a 100644 --- a/tests/gold_tests/tls/tls_client_cert_override.test.py +++ b/tests/gold_tests/tls/tls_client_cert_override.test.py @@ -105,9 +105,8 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:example.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr.TimeOut = 5 #Should fail trfail = Test.AddTestRun("Connect with bad client cert to first server") @@ -116,9 +115,8 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/badcase1'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 5 +trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trfail.TimeOut = 5 # Should succeed trbar = Test.AddTestRun("Connect with correct client cert to second server") @@ -127,9 +125,8 @@ trbar.StillRunningAfter = server2 trbar.Processes.Default.Command = "curl -H host:bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) trbar.Processes.Default.ReturnCode = 0 -trbar.Processes.Default.TimeOut = 5 +trbar.Processes.Default.TimeOut = 10 trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -trbar.TimeOut = 5 #Should fail trbarfail = Test.AddTestRun("Connect with bad client cert to second server") @@ -138,8 +135,7 @@ trbarfail.StillRunningAfter = server2 trbarfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/badcase2'.format(ts.Variables.port) trbarfail.Processes.Default.ReturnCode = 0 -trbarfail.Processes.Default.TimeOut = 5 +trbarfail.Processes.Default.TimeOut = 10 trbarfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trbarfail.TimeOut = 5 diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py index f0abd7ecc9f..2406d28cfad 100644 --- a/tests/gold_tests/tls/tls_client_verify.test.py +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -84,9 +84,8 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to foo.com with bad cert") tr.Setup.Copy("ssl/server.pem") @@ -96,9 +95,8 @@ tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) # Should fail with badly signed certs tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to foo.com with cert") tr.Setup.Copy("ssl/signed-foo.pem") @@ -107,18 +105,16 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-foo.pem --key signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.bar.com with cert") tr.Setup.Copy("ssl/signed-bob-bar.pem") @@ -127,9 +123,8 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-bar.pem --key signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.bar.com with bad cert") tr.Setup.Copy("ssl/server.pem") @@ -138,18 +133,16 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.foo.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.foo.com with cert") tr.Setup.Copy("ssl/signed-bob-foo.pem") @@ -158,9 +151,8 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-foo.pem --key signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.foo.com with bad cert") tr.Setup.Copy("ssl/server.pem") @@ -169,18 +161,16 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert handshake failure", "TLS handshake should fail") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bar.com with cert") tr.Setup.Copy("ssl/signed-bar.pem") @@ -189,9 +179,8 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bar.com with bad cert") tr.Setup.Copy("ssl/server.pem") @@ -200,8 +189,7 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") -tr.TimeOut = 5 diff --git a/tests/gold_tests/tls/tls_client_verify2.test.py b/tests/gold_tests/tls/tls_client_verify2.test.py index a435e944884..fd70d865c3d 100644 --- a/tests/gold_tests/tls/tls_client_verify2.test.py +++ b/tests/gold_tests/tls/tls_client_verify2.test.py @@ -84,9 +84,8 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to foo.com with cert") tr.Setup.Copy("ssl/signed-foo.pem") @@ -95,18 +94,16 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-foo.pem --key signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.bar.com with cert") tr.Setup.Copy("ssl/signed-bob-bar.pem") @@ -115,9 +112,8 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-bar.pem --key signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.bar.com with bad cert") tr.Setup.Copy("ssl/server.pem") @@ -126,18 +122,16 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("error", "TLS handshake should fail") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.foo.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.foo.com with cert") tr.Setup.Copy("ssl/signed-bob-foo.pem") @@ -146,9 +140,8 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-foo.pem --key signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bob.foo.com with bad cert") tr.Setup.Copy("ssl/server.pem") @@ -157,18 +150,16 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("error", "TLS handshake should fail") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bar.com with cert") tr.Setup.Copy("ssl/signed-bar.pem") @@ -177,9 +168,8 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") -tr.TimeOut = 5 tr = Test.AddTestRun("Connect to bar.com with bad cert") tr.Setup.Copy("ssl/server.pem") @@ -188,7 +178,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert unknown ca", "TLS handshake should succeed") -tr.TimeOut = 5 diff --git a/tests/gold_tests/tls/tls_client_versions.test.py b/tests/gold_tests/tls/tls_client_versions.test.py index 18557998978..77bad0d77c0 100644 --- a/tests/gold_tests/tls/tls_client_versions.test.py +++ b/tests/gold_tests/tls/tls_client_versions.test.py @@ -81,8 +81,7 @@ tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 35 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ContainsExpression("ssl_choose_client_version:unsupported protocol", "Should not allow TLSv1_2") # Target foo.com for TLSv1. Should succeed @@ -90,23 +89,20 @@ tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 # Target bar.com for TLSv1. Should fail tr = Test.AddTestRun("bar.com TLSv1") tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 35 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ContainsExpression("alert protocol version", "Should not allow TLSv1_0") -tr.TimeOut = 5 # Target bar.com for TLSv1_2. Should succeed tr = Test.AddTestRun("bar.com TLSv1_2") tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 diff --git a/tests/gold_tests/tls/tls_forward_nonhttp.test.py b/tests/gold_tests/tls/tls_forward_nonhttp.test.py index 2fb8b9052e3..8516dc1030c 100644 --- a/tests/gold_tests/tls/tls_forward_nonhttp.test.py +++ b/tests/gold_tests/tls/tls_forward_nonhttp.test.py @@ -69,8 +69,7 @@ tr.ReturnCode = 0 tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 testout_path = os.path.join(Test.RunDirectory, "test.out") tr.Disk.File(testout_path, id = "testout") tr.Processes.Default.Streams.All += Testers.IncludesExpression("This is a reply", "s_client should get response") diff --git a/tests/gold_tests/tls/tls_keepalive.test.py b/tests/gold_tests/tls/tls_keepalive.test.py index 518f2bfae24..92fa45edbc6 100644 --- a/tests/gold_tests/tls/tls_keepalive.test.py +++ b/tests/gold_tests/tls/tls_keepalive.test.py @@ -102,5 +102,4 @@ ts.Disk.squid_log.Content = "gold/accesslog.gold" -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 diff --git a/tests/gold_tests/tls/tls_ticket.test.py b/tests/gold_tests/tls/tls_ticket.test.py index c9953cc313e..355c5fed1ed 100644 --- a/tests/gold_tests/tls/tls_ticket.test.py +++ b/tests/gold_tests/tls/tls_ticket.test.py @@ -89,8 +89,7 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) path1 = tr.Processes.Default.Streams.stdout.AbsPath tr.StillRunningAfter = server -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 # Pull out session created in tr to test for session id in tr2 def checkSession(ev) : @@ -124,5 +123,5 @@ def checkSession(ev) : tr2.Processes.Default.StartBefore(Test.Processes.ts2, ready=When.PortOpen(ts2.Variables.ssl_port)) tr2.ReturnCode = 0 path2 = tr2.Processes.Default.Streams.stdout.AbsPath -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.Processes.Default.Streams.All.Content = Testers.Lambda(checkSession) diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index 30b976072e0..1db455ea05b 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -92,8 +92,7 @@ tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") @@ -105,8 +104,7 @@ tr.Processes.Default.Command = "curl -v --resolve 'bob.bar.com:{0}:127.0.0.1' -k https://bob.bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") @@ -118,8 +116,7 @@ tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") @@ -128,8 +125,7 @@ tr.Processes.Default.Command = "curl -v -k https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") @@ -152,8 +148,7 @@ tr.Processes.Default.Env = ts.Env tr.Processes.Default.Command = 'echo Updated configs' tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 trreload = Test.AddTestRun("Reload config") trreload.StillRunningAfter = ts @@ -163,8 +158,7 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 -trreload.Processes.Default.TimeOut = 5 -trreload.TimeOut = 5 +trreload.Processes.Default.TimeOut = 10 # Should termimate on traffic_server (not tunnel) tr = Test.AddTestRun("foo.com no Tunnel-test") @@ -172,8 +166,7 @@ # Wait for the reload to complete tr.DelayStart = 5 tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -183,8 +176,7 @@ tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Terminate on Traffic Server") diff --git a/tests/gold_tests/tls/tls_tunnel_forward.test.py b/tests/gold_tests/tls/tls_tunnel_forward.test.py index ce031516495..a9e2cec6f6e 100644 --- a/tests/gold_tests/tls/tls_tunnel_forward.test.py +++ b/tests/gold_tests/tls/tls_tunnel_forward.test.py @@ -96,8 +96,7 @@ tr.Processes.Default.StartBefore(server_random) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") @@ -108,9 +107,8 @@ tr2.Processes.Default.Command = "curl -v --http1.1 -H 'host:bar.com' --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server_bar -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts -tr2.TimeOut = 5 tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") @@ -121,9 +119,8 @@ tr3.Processes.Default.Command = "curl --http1.1 -v -k -H 'host:random.com' https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr3.ReturnCode = 0 tr3.StillRunningAfter = server_random -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 tr3.StillRunningAfter = ts -tr3.TimeOut = 5 tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr3.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") diff --git a/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py index 920ed9f9337..6341aeca0c6 100644 --- a/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py +++ b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py @@ -93,9 +93,8 @@ tr.Processes.Default.StartBefore(server_random) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server_random -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.StillRunningAfter = ts -tr.TimeOut = 5 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") diff --git a/tests/gold_tests/tls/tls_verify.test.py b/tests/gold_tests/tls/tls_verify.test.py index 81164e83e97..cc88c99163f 100644 --- a/tests/gold_tests/tls/tls_verify.test.py +++ b/tests/gold_tests/tls/tls_verify.test.py @@ -109,18 +109,16 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr2 = Test.AddTestRun("Override-enforcing-Test") tr2.Processes.Default.Command = "curl -v -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 tr2.StillRunningAfter = ts +tr2.Processes.Default.TimeOut = 10 tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr3 = Test.AddTestRun("Override-enforcing-Test-fail-name-check") tr3.Processes.Default.Command = "curl -v -k -H \"host: bad_bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) @@ -128,7 +126,7 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") diff --git a/tests/gold_tests/tls/tls_verify2.test.py b/tests/gold_tests/tls/tls_verify2.test.py index 5e4f2b5e21c..8a4742da659 100644 --- a/tests/gold_tests/tls/tls_verify2.test.py +++ b/tests/gold_tests/tls/tls_verify2.test.py @@ -116,25 +116,23 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr2 = Test.AddTestRun("override-disabled") tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr3 = Test.AddTestRun("override-permissive") tr3.Processes.Default.Command = "curl -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr4 = Test.AddTestRun("override-permissive-bad-name") @@ -142,7 +140,7 @@ tr4.ReturnCode = 0 tr4.StillRunningAfter = server tr4.StillRunningAfter = ts -tr4.Processes.Default.TimeOut = 5 +tr4.Processes.Default.TimeOut = 10 tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr5 = Test.AddTestRun("default-enforce-bad-sig") @@ -151,7 +149,7 @@ tr5.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr5.StillRunningAfter = server tr5.StillRunningAfter = ts -tr5.Processes.Default.TimeOut = 5 +tr5.Processes.Default.TimeOut = 10 tr6 = Test.AddTestRun("default-enforce-fail") tr6.Processes.Default.Command = "curl -k -H \"host: bad_foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) @@ -159,7 +157,7 @@ tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr6.StillRunningAfter = server tr6.StillRunningAfter = ts -tr6.Processes.Default.TimeOut = 5 +tr6.Processes.Default.TimeOut = 10 # No name checking for the sig-only permissive override for bad_bar diff --git a/tests/gold_tests/tls/tls_verify3.test.py b/tests/gold_tests/tls/tls_verify3.test.py index c12c787544c..81a7798e2bb 100644 --- a/tests/gold_tests/tls/tls_verify3.test.py +++ b/tests/gold_tests/tls/tls_verify3.test.py @@ -110,28 +110,25 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr = Test.AddTestRun("my.foo.com Permissive-Test log failure") tr.Processes.Default.Command = "curl -v -k --resolve 'my.foo.com:{0}:127.0.0.1' https://my.foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr2 = Test.AddTestRun("bob.bar.com Override-enforcing-Test") tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr3 = Test.AddTestRun("bob.foo.com override-enforcing-name-test") tr3.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/".format(ts.Variables.ssl_port) @@ -139,8 +136,7 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 5 -tr3.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 tr3 = Test.AddTestRun("random.bar.com override-no-test") tr3.Processes.Default.Command = "curl -v -k --resolve 'random.bar.com:{0}:127.0.0.1' https://random.bar.com:{0}".format(ts.Variables.ssl_port) @@ -148,8 +144,7 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 5 -tr3.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 # Over riding the built in ERROR check since we expect tr3 to fail diff --git a/tests/gold_tests/tls/tls_verify_base.test.py b/tests/gold_tests/tls/tls_verify_base.test.py index 16a2d303602..baa373fba0c 100644 --- a/tests/gold_tests/tls/tls_verify_base.test.py +++ b/tests/gold_tests/tls/tls_verify_base.test.py @@ -108,8 +108,7 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr = Test.AddTestRun("Permissive-Test with logged failure") @@ -117,8 +116,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -126,9 +124,8 @@ tr2.Processes.Default.Command = "curl -v -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts -tr2.TimeOut = 5 tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr3 = Test.AddTestRun("Override-enforcing-Test-fail-name-check") @@ -137,7 +134,7 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: SNI \(bad_bar.com\) not in certificate. Action=Terminate", "Make sure bad_bar name checked failed.") diff --git a/tests/gold_tests/tls/tls_verify_ca_override.test.py b/tests/gold_tests/tls/tls_verify_ca_override.test.py index 016ae9935be..749ebbf43bb 100644 --- a/tests/gold_tests/tls/tls_verify_ca_override.test.py +++ b/tests/gold_tests/tls/tls_verify_ca_override.test.py @@ -100,30 +100,27 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) tr.StillRunningAfter = server1 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 # Should succed. No message tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr2 = Test.AddTestRun("Use incorrect ca bundle for server 1") tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/badcase1".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server1 -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about name mismatch tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr2 = Test.AddTestRun("Use currect ca bundle for server 2") tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/case2".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server2 -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about signature tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr3 = Test.AddTestRun("User incorrect ca bundle for server 2") tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/badcase2".format(ts.Variables.port) @@ -132,6 +129,6 @@ tr3.StillRunningAfter = ts # Should succeed. No error messages tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 diff --git a/tests/gold_tests/tls/tls_verify_not_pristine.test.py b/tests/gold_tests/tls/tls_verify_not_pristine.test.py index 2418063827a..529ebfbc8da 100644 --- a/tests/gold_tests/tls/tls_verify_not_pristine.test.py +++ b/tests/gold_tests/tls/tls_verify_not_pristine.test.py @@ -95,19 +95,17 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 # foo.com in. bar.com out. Should not verify tr2 = Test.AddTestRun("Enforced-bad-test") tr2.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should not have succeeded") -tr2.TimeOut = 5 # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") diff --git a/tests/gold_tests/tls/tls_verify_override.test.py b/tests/gold_tests/tls/tls_verify_override.test.py index 59608635062..626e8587bff 100644 --- a/tests/gold_tests/tls/tls_verify_override.test.py +++ b/tests/gold_tests/tls/tls_verify_override.test.py @@ -128,30 +128,27 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 # Should succed. No message tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr2 = Test.AddTestRun("default-permissive-fail") tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about name mismatch tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr2 = Test.AddTestRun("default-permissive-fail2") tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about signature tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr3 = Test.AddTestRun("default-foo-to-bar") tr3.Processes.Default.Command = "curl -k -v -H \"host: foo.com\" http://127.0.0.1:{0}/basictobar".format(ts.Variables.port) @@ -160,7 +157,7 @@ tr3.StillRunningAfter = ts # Should succeed. No error messages tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 tr3 = Test.AddTestRun("override-foo") tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/override".format(ts.Variables.port) @@ -169,7 +166,7 @@ tr3.StillRunningAfter = ts # Should succeed. No error messages tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 tr4 = Test.AddTestRun("override-bar-disabled") tr4.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" http://127.0.0.1:{0}/overridedisabled".format(ts.Variables.port) @@ -178,7 +175,7 @@ tr4.StillRunningAfter = ts # Succeed. No error messages tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr4.Processes.Default.TimeOut = 5 +tr4.Processes.Default.TimeOut = 10 tr5 = Test.AddTestRun("override-bar-signature-enforced") tr5.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overridesignature".format(ts.Variables.port) @@ -186,7 +183,7 @@ tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr5.StillRunningAfter = server tr5.StillRunningAfter = ts -tr5.Processes.Default.TimeOut = 5 +tr5.Processes.Default.TimeOut = 10 tr6 = Test.AddTestRun("override-bar-enforced") tr6.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overrideenforced".format(ts.Variables.port) @@ -195,7 +192,7 @@ tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr6.StillRunningAfter = server tr6.StillRunningAfter = ts -tr6.Processes.Default.TimeOut = 5 +tr6.Processes.Default.TimeOut = 10 # Should succeed tr = Test.AddTestRun("foo-to-bar-sni-policy-remap") @@ -203,7 +200,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") # Should fail @@ -212,7 +209,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") # Should fail @@ -221,7 +218,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") # Should succeed @@ -230,7 +227,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") # Over riding the built in ERROR check since we expect some cases to fail diff --git a/tests/gold_tests/tls/tls_verify_override_base.test.py b/tests/gold_tests/tls/tls_verify_override_base.test.py index 404062e6660..8f35718154c 100644 --- a/tests/gold_tests/tls/tls_verify_override_base.test.py +++ b/tests/gold_tests/tls/tls_verify_override_base.test.py @@ -123,30 +123,27 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 # Should succed. No message tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr2 = Test.AddTestRun("default-permissive-fail") tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about name mismatch tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr2 = Test.AddTestRun("default-permissive-fail2") tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 +tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about signature tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr3 = Test.AddTestRun("override-foo") tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/override".format(ts.Variables.port) @@ -155,7 +152,7 @@ tr3.StillRunningAfter = ts # Should succeed. No error messages tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.TimeOut = 5 +tr3.Processes.Default.TimeOut = 10 tr4 = Test.AddTestRun("override-bar-disabled") tr4.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" http://127.0.0.1:{0}/overridedisabled".format(ts.Variables.port) @@ -164,7 +161,7 @@ tr4.StillRunningAfter = ts # Succeed. No error messages tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr4.Processes.Default.TimeOut = 5 +tr4.Processes.Default.TimeOut = 10 tr5 = Test.AddTestRun("override-bar-signature-enforced") tr5.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overridesignature".format(ts.Variables.port) @@ -172,7 +169,7 @@ tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr5.StillRunningAfter = server tr5.StillRunningAfter = ts -tr5.Processes.Default.TimeOut = 5 +tr5.Processes.Default.TimeOut = 10 tr6 = Test.AddTestRun("override-bar-enforced") tr6.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overrideenforced".format(ts.Variables.port) @@ -181,7 +178,7 @@ tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr6.StillRunningAfter = server tr6.StillRunningAfter = ts -tr6.Processes.Default.TimeOut = 5 +tr6.Processes.Default.TimeOut = 10 # Should succeed tr = Test.AddTestRun("foo-to-bar-sni-policy-remap") @@ -189,7 +186,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") # Should fail @@ -198,7 +195,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") # Should fail @@ -207,7 +204,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") # Should succeed @@ -216,7 +213,7 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") From 408bc6be7ba1470fc3a887a99c90adf861bef4ee Mon Sep 17 00:00:00 2001 From: Pushkar Pradhan Date: Wed, 2 Jan 2019 19:10:07 -0800 Subject: [PATCH 187/526] The cookie_remap plugin. --- doc/admin-guide/plugins/cookie_remap.en.rst | 462 +++++++ plugins/Makefile.am | 1 + .../experimental/cookie_remap/Makefile.inc | 39 + plugins/experimental/cookie_remap/README | 9 + .../experimental/cookie_remap/cookie_remap.cc | 1151 +++++++++++++++++ .../experimental/cookie_remap/cookiejar.cc | 255 ++++ plugins/experimental/cookie_remap/cookiejar.h | 62 + plugins/experimental/cookie_remap/hash.c | 110 ++ plugins/experimental/cookie_remap/hash.h | 54 + plugins/experimental/cookie_remap/strip.c | 282 ++++ plugins/experimental/cookie_remap/strip.h | 146 +++ .../unit_tests/cookiejar/unit_test.cc | 250 ++++ .../cookie_remap/bucketcookie.test.py | 104 ++ .../cookie_remap/collapseslashes.test.py | 71 + .../cookie_remap/configs/bucketconfig.txt | 9 + .../cookie_remap/configs/collapseconfig.txt | 6 + .../cookie_remap/configs/connectorconfig.txt | 16 + .../cookie_remap/configs/existsconfig.txt | 8 + .../cookie_remap/configs/matchconfig.txt | 8 + .../cookie_remap/configs/matchuriconfig.txt | 17 + .../cookie_remap/configs/matrixconfig.txt | 24 + .../cookie_remap/configs/notexistsconfig.txt | 8 + .../cookie_remap/configs/regexconfig.txt | 9 + .../cookie_remap/configs/statusconfig.txt | 17 + .../cookie_remap/configs/subcookie.txt | 17 + .../cookie_remap/configs/substituteconfig.txt | 21 + .../pluginTest/cookie_remap/connector.test.py | 104 ++ .../cookie_remap/existscookie.test.py | 104 ++ .../cookie_remap/gold/collapseslashes.gold | 3 + .../cookie_remap/gold/doesntexistcookie.gold | 3 + .../cookie_remap/gold/existscookie.gold | 3 + .../cookie_remap/gold/matchcookie.gold | 3 + .../cookie_remap/gold/matchcookie2.gold | 3 + .../cookie_remap/gold/matchelsestatus.gold | 3 + .../cookie_remap/gold/matchstatus.gold | 3 + .../pluginTest/cookie_remap/gold/matrix.gold | 15 + .../cookie_remap/gold/regexdoesntmatch.gold | 3 + .../cookie_remap/gold/regexmatches.gold | 3 + .../cookie_remap/gold/substitute.gold | 9 + .../cookie_remap/gold/wontmatchcookie.gold | 3 + .../cookie_remap/gold/wontmatchcookie2.gold | 3 + .../cookie_remap/matchcookie.test.py | 104 ++ .../pluginTest/cookie_remap/matchuri.test.py | 103 ++ .../cookie_remap/matrixparams.test.py | 148 +++ .../cookie_remap/notexistscookie.test.py | 103 ++ .../cookie_remap/regexcookie.test.py | 104 ++ .../pluginTest/cookie_remap/setstatus.test.py | 79 ++ .../pluginTest/cookie_remap/subcookie.test.py | 100 ++ .../cookie_remap/substitute.test.py | 116 ++ 49 files changed, 4278 insertions(+) create mode 100644 doc/admin-guide/plugins/cookie_remap.en.rst create mode 100644 plugins/experimental/cookie_remap/Makefile.inc create mode 100644 plugins/experimental/cookie_remap/README create mode 100644 plugins/experimental/cookie_remap/cookie_remap.cc create mode 100644 plugins/experimental/cookie_remap/cookiejar.cc create mode 100644 plugins/experimental/cookie_remap/cookiejar.h create mode 100644 plugins/experimental/cookie_remap/hash.c create mode 100644 plugins/experimental/cookie_remap/hash.h create mode 100644 plugins/experimental/cookie_remap/strip.c create mode 100644 plugins/experimental/cookie_remap/strip.h create mode 100644 plugins/experimental/cookie_remap/unit_tests/cookiejar/unit_test.cc create mode 100644 tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/bucketconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/collapseconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/connectorconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/existsconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/matchconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/matchuriconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/matrixconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/notexistsconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/regexconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/statusconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/subcookie.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/configs/substituteconfig.txt create mode 100644 tests/gold_tests/pluginTest/cookie_remap/connector.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/matchelsestatus.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/matchstatus.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold create mode 100644 tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py create mode 100644 tests/gold_tests/pluginTest/cookie_remap/substitute.test.py diff --git a/doc/admin-guide/plugins/cookie_remap.en.rst b/doc/admin-guide/plugins/cookie_remap.en.rst new file mode 100644 index 00000000000..f79074b416e --- /dev/null +++ b/doc/admin-guide/plugins/cookie_remap.en.rst @@ -0,0 +1,462 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _admin-plugins-cookie_remap: + +============================================================ +Cookie Based Routing Inside TrafficServer Using cookie_remap +============================================================ + +---- + + +* `Cookie Based Routing Inside TrafficServer Using cookie_remap <#cookie-based-routing-inside-trafficserver-using-cookie_remap>`_ + + * `Features <#features>`_ + * `Limitations <#limitations>`_ + * `Setup <#setup>`_ + * `Operations <#operations>`_ + + * `Comments <#comments>`_ + * `cookie: X|X.Y <#cookie-xxy>`_ + * `operation: exists|not exists|string|regex|bucket <#operation-existsnot-existsstringregexbucket>`_ + * `match: str <#match-str>`_ + * `regex: str <#regex-str>`_ + * `bucket|hash: X/Y <#buckethash-xy>`_ + * `sendto|url: url <#sendtourl-url>`_ + * `status: HTTP status-code <#status-http-status-code>`_ + * `else: url [optional] <#else-url-optional>`_ + * `connector: and <#connector-and>`_ + + * `Reserved path expressions <#reserved-path-expressions>`_ + + * `$cr_request_url <#cr_request_url-v-15>`_ + * `$cr_urlencode() <#cr_urlencode-v-15>`_ + * `$path <#path>`_ + * `$unmatched_path <#unmatched_path>`_ + + * `An example configuration file <#an-example-configuration-file>`_ + * `Debugging things <#debugging-things>`_ + + * `Initial output <#initial-output>`_ + +This remap plugin makes decisions about where to send your request based on properties present (or absent) within the HTTP Cookie header. It can also make decisions based on your uri (url path + query.) + +Features +-------- + +---- + + +* Also supports sub-level cookies + + * K indicates top-level cookie "K" + * K.l indicates top-level cookie "K", subfield "l" + +* Cookie exists / Cookie doesn't exist +* Cookie or uri matches string +* Cookie or uri matches regex (with match replacement in the sendto like `http://foo.com/$1/$2 `_\ ) +* Cookie falls into a hash/bucket range +* Can url encode dynamic data + +Limitations +----------- + +---- + + +* Does not support `plugin chaining `_. + +Setup +----- + +---- + +The plugin is specified in remap.config using a syntax similar to: + + +.. raw:: html + + 
+   map http://foo.com http://bar.com @plugin=/usr/bin/trafficserver/libexec64/cookie_remap.so @pparam=/home/trafficserver/conf/cookie_remap/cookie_remap.txt
+
+ + +Operations +---------- + +---- + +All operations are specified in a YAML configuration file you pass as @pparam on the plugin configuration line. YAML is very simple syntax. See the example configuration files below. + +Each operation results in a sendto action. That means that, if matched, cookie_remap forwards the request to the given ``sendto`` url. It can be proxied or redirected. An ``else`` sendto can be specified, in which case a failure to match will forward the request also. Once a sendto is invoked: + + +* cookie_remap will not process any more "operations" from the configuration +* the default destination specified in the remap.config file will not be used; it will be replaced by the ``sendto`` + +Each ``operation`` can have multiple "sub-operations", connected with an conjunction operator. Currently, only the ``and`` operator is supported. So, you can say, "if cookie exists, ``and`` uri is ``x``\ , then redirect." + +Comments +^^^^^^^^ + +---- + +Comments are allowed in the configuration file if they begin with '#' + +cookie: X|X.Y +^^^^^^^^^^^^^ + +---- + +This sub-operation is testing against the X cookie or X.Y cookie where X.Y denotes the X cookie, sub cookie Y +e.g + + +.. raw:: html + + 
+   A=ACOOKIE;B=data&f=fsub&z=zsub;
+   A will operate on ACOOKIE
+   B will operate on data&f=fsub&z=zsub
+   B.f will operate on fsub
+
+ + +operation: exists|not exists|string|regex|bucket +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +---- + +This keyword actually denotes a "suboperation" - it can be specified multiple times, connected with ``and``\ , and send the user to a given destination. + + +* exists: Test for the existence of a cookie +* not exists: Test for the non-existence of a cookie +* string: string match +* regex: regex matching with $1 - $9 replaced in ``sendto`` +* bucket: Hash the cookie and check if it falls in a bucket + +The ``cookie`` operator must be specified in the YAML file just before the suboperation that will apply to the cookie. + +The ``string``, ``regex`` and ``bucket`` operate on the specified cookie, or if no cookie is specified, they will operate on the uri of the request. + +match: str +^^^^^^^^^^ + +---- + +Match the cookie data to str. If no cookie is specified, match to the request uri. + +regex: str +^^^^^^^^^^ + +---- + +Sets the regex to str. Matching is enabled with $1-$9 (replaced in sendto). Note that only the final regex match in an operation will be used to populate the substitutions $1-$9. If no cookie is specified, match to the request uri. + +bucket|hash: X/Y +^^^^^^^^^^^^^^^^ + +---- + +Hashes (bucketizes) the data in Y buckets. If you fall into the first X of them, this suboperation passes. If no cookie is specified, match to the request uri. + + +.. raw:: html + + 
+   bucket: 1/100
+   will effectively bucketize 1% of your users
+
+ + +sendto|url: url +^^^^^^^^^^^^^^^ + +---- + +if the sub-operation(s) all match, send to url + +status: HTTP status-code +^^^^^^^^^^^^^^^^^^^^^^^^ + +---- + +if the sub-operation(s) all match, set the status code (e.g. set it to 302). In the case of a redirect, the sendto URL becomes the redirect URL. + +else: url [optional] +^^^^^^^^^^^^^^^^^^^^ + +---- + +If one of the sub-operations fails, send to url. This is optional. If there is no 'else', we will continue to process operations until either one succeeds, there is an else in one of the operations or we fall through to the default mapping from remap.config + +connector: and +^^^^^^^^^^^^^^ + +---- + +'and' is the only supported connector + +Reserved path expressions +------------------------- + +---- + +The following expressions can be used in either the sendto **or** ``else`` URLs, and will be expanded. + +$cr_request_url +^^^^^^^^^^^^^^^^^^^^^^^^^ + +---- + +Replaced with the full original url. + +Therefore, a rule like: + + +.. raw:: html + + 
+   op:
+     cookie: K
+     operation: exists
+     sendto: http://foo.com/?.done=$cr_request_url
+
+ + +and a request that matches, e.g. + + +.. raw:: html + + 
+   http://bar.com/hello?fruit=bananas
+
+ + +will become + + +.. raw:: html + + 
+   http://foo.com/?.done=http://bar.com/hello?fruit=bananas
+
+ + +$cr_urlencode() +^^^^^^^^^^^^^^^^^^^^^^^^^ + +---- + +Replaced with a urlencoded version of its argument. The url argument is aggressively encoded such that all non-alphanumeric characters are converted to % hex notation. The simple algorithm could probably be refined in the future to be less aggressive (encode fewer characters.) + +Therefore, a rule like: + + +.. raw:: html + + 
+   op:
+     cookie: B
+     operation: exists
+     sendto: http://foo.com/?.done=$cr_urlencode($cr_request_url)
+
+ + +and a request that matches, e.g. + + +.. raw:: html + + 
+   http://bar.com/hello?fruit=bananas
+
+ + +will become + + +.. raw:: html + + 
+   http://foo.com/?.done=http%3A%2F%2Fbar%2Ecom@2Fhello%3Ffruit%3Dbananas
+
+ + +$path +^^^^^ + +---- + +$path can be used to replace in either the sendto **or** else URLs the original request path. Therefore a rule like: + + +.. raw:: html + + 
+   op:
+     cookie: K
+     operation: exists
+     sendto: http://foo.com/$path/x/y/z
+
+ + +and a request like `http://finance.yahoo.com/photos/what/ever/ `_ that matches the rule + + +.. raw:: html + + 
+   map http://finance.yahoo.com/photos/ http://newfinance.yahoo.com/1k.html @plugin=cookie_remap.so @pparam=foo.txt
+
+ + +will become `http://foo.com/photos/what/ever/x/y/z `_ + +$unmatched_path +^^^^^^^^^^^^^^^ + +---- + +$unmatched_path can be used to gather the URL arguments **beyond** what was matched by the remap rule. Therefore a rule like: + + +.. raw:: html + + 
+   op:
+     cookie: K
+     operation: exists
+     sendto: http://foo.com/$unmatched_path/x/y/z
+
+ + +and a request like `http://finance.yahoo.com/photos/what/ever/ `_ that matches the rule + + +.. raw:: html + + 
+   map http://finance.yahoo.com/photos/ http://newfinance.yahoo.com/1k.html @plugin=cookie_remap.so @pparam=foo.txt
+
+ + +will become `http://foo.com/what/ever/matches/x/y/z `_ + +An example configuration file +----------------------------- + +---- + + + +* first match "wins" (top down) + + +.. raw:: html + + 
+   #comments
+   #are allowed
+   op:
+     cookie: K
+     operation: exists
+     url: http://www.yahoo.com
+
+   op:
+     cookie: K
+     operation: exists
+     connector: and
+     cookie: K.l
+     operation: not exists
+     sendto: http://www.yahoo.com
+
+   op:
+     cookie: Y
+     operation: bucket
+     bucket: 1/1000
+     connector: and
+     cookie: Y.l
+     operation: regex
+     regex: (.*)
+     connector: and
+     cookie: Y.n
+     operation: string
+     match: foobar
+     sendto: http://cnn.com/$1
+     else: http://yahoo.com
+
+ + +Debugging things +---------------- + +---- + +The easiest way to debug problems with this plugin is to run in the following manner: + + +.. raw:: html + + 
+   bin/traffic_server -T cookie_remap
+
+ + +which will produce output on startup, and for each request. Be aware that this mode of running trafficserver is **extremely** inefficient and should only be used for debugging. + +Initial output +^^^^^^^^^^^^^^ + +---- + +Initially, you will notice output describing each operation and how trafficserver interpreted the information from your configuration file: + + +.. raw:: html + + 
+   [Jul  8 13:08:34.183] Server {3187168} DIAG: (cookie_remap) loading cookie remap configuration file from /homes/ebalsa/dev/yts_mods/cookie_remap/example_config.txt
+   [Jul  8 13:08:34.199] Server {3187168} DIAG: (cookie_remap) ++++operation++++
+   [Jul  8 13:08:34.199] Server {3187168} DIAG: (cookie_remap) sending to: http://finance.yahoo.com/2k.html
+   [Jul  8 13:08:34.199] Server {3187168} DIAG: (cookie_remap) if these operations match:
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)     +++subop+++
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         cookie: B
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         operation: bucket
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         bucket: 1/100
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         taking: 1
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         out of: 100
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)     +++subop+++
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap)         cookie: Y.l
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap)         operation: exists
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap) ++++operation++++
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap) sending to: http://X.existed.com
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap) if these operations match:
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap)     +++subop+++
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap)         cookie: PH.l
+   [Jul  8 13:08:34.202] Server {3187168} DIAG: (cookie_remap)         operation: exists
+   [Jul  8 13:08:34.202] Server {3187168} DIAG: (cookie_remap) # of ops: 2
+
+ + +each operation is enumerated and is more-or-less human readable from the top-down. From now on, each request has information spit out to the console with debugging information on how the cookie_remap is treating this request. + diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 09e814a8034..9eee0dd61e4 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -58,6 +58,7 @@ include experimental/buffer_upload/Makefile.inc include experimental/cache_range_requests/Makefile.inc include experimental/certifier/Makefile.inc include experimental/collapsed_forwarding/Makefile.inc +include experimental/cookie_remap/Makefile.inc include experimental/custom_redirect/Makefile.inc include experimental/fq_pacing/Makefile.inc include experimental/geoip_acl/Makefile.inc diff --git a/plugins/experimental/cookie_remap/Makefile.inc b/plugins/experimental/cookie_remap/Makefile.inc new file mode 100644 index 00000000000..5b44c4f4a4e --- /dev/null +++ b/plugins/experimental/cookie_remap/Makefile.inc @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/cookie_remap/cookie_remap.la + +experimental_cookie_remap_cookie_remap_la_SOURCES = \ + experimental/cookie_remap/cookie_remap.cc \ + experimental/cookie_remap/hash.c \ + experimental/cookie_remap/strip.c \ + experimental/cookie_remap/cookiejar.cc + +experimental_cookie_remap_cookie_remap_la_LDFLAGS = \ + $(AM_LDFLAGS) + +AM_CPPFLAGS += @YAMLCPP_INCLUDES@ + +check_PROGRAMS += \ + cookie_remap/test_cookiejar + +cookie_remap_test_cookiejar_CPPFLAGS = $(AM_CPPFLAGS) -Iexperimental/cookie_remap -I$(abs_top_srcdir)/tests/include +cookie_remap_test_cookiejar_SOURCES = \ + experimental/cookie_remap/unit_tests/cookiejar/unit_test.cc \ + experimental/cookie_remap/strip.c \ + experimental/cookie_remap/cookiejar.cc + +# vim: ft=make ts=8 sw=8 et: diff --git a/plugins/experimental/cookie_remap/README b/plugins/experimental/cookie_remap/README new file mode 100644 index 00000000000..d9064f3ce8e --- /dev/null +++ b/plugins/experimental/cookie_remap/README @@ -0,0 +1,9 @@ +============================================================ +Cookie Based Routing Inside TrafficServer Using cookie_remap +============================================================ + +---- + + +For details please read the official documentation in the Admin Guide section. + diff --git a/plugins/experimental/cookie_remap/cookie_remap.cc b/plugins/experimental/cookie_remap/cookie_remap.cc new file mode 100644 index 00000000000..b5671888bed --- /dev/null +++ b/plugins/experimental/cookie_remap/cookie_remap.cc @@ -0,0 +1,1151 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +//////////////////////////////////////////////////////////////////////////////// +// cookie_remap: ATS plugin to do (simple) cookie based remap rules +// To use this plugin, configure a remap.config rule like +// map http://foo.com http://bar.com @plugin=.../libexec/cookie_remap.so +// @pparam=maps.reg + +#include "cookiejar.h" +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "hash.h" + +using namespace std; + +// for bucketizing + +#define MY_NAME "cookie_remap" +const int OVECCOUNT = 30; // We support $1 - $9 only, and this needs to be 3x that + +#if TS_VERSION_MAJOR > 7 +#define SETHTTPSTATUS(TXN, STATUS) TSHttpTxnStatusSet((TXN), STATUS) +#else +#define SETHTTPSTATUS(TXN, STATUS) TSHttpTxnSetHttpRetStatus((TXN), STATUS) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Helpers for memory management (to make sure pcre uses the TS APIs). +// +inline void * +ink_malloc(size_t s) +{ + return TSmalloc(s); +} + +inline void +ink_free(void *s) +{ + return TSfree(s); +} +/////////////////////////////////////////////////////////////////////////////// +// Class holding one request URL's component, to simplify the code and +// length calculations (we need all of them). +// +struct UrlComponents { + UrlComponents() {} + + ~UrlComponents() + { + if (url != nullptr) + TSfree((void *)url); + } + + void + populate(TSRemapRequestInfo *rri) + { + scheme = TSUrlSchemeGet(rri->requestBufp, rri->requestUrl, &scheme_len); + host = TSUrlHostGet(rri->requestBufp, rri->requestUrl, &host_len); + tspath = TSUrlPathGet(rri->requestBufp, rri->requestUrl, &tspath_len); + query = TSUrlHttpQueryGet(rri->requestBufp, rri->requestUrl, &query_len); + matrix = TSUrlHttpParamsGet(rri->requestBufp, rri->requestUrl, &matrix_len); + port = TSUrlPortGet(rri->requestBufp, rri->requestUrl); + url = TSUrlStringGet(rri->requestBufp, rri->requestUrl, &url_len); + from_path = TSUrlPathGet(rri->requestBufp, rri->mapFromUrl, &from_path_len); + + // based on RFC2396, matrix params are part of path segments so + // we will just + // append them to the path + path.assign(tspath, tspath_len); + if (matrix_len) { + path += ';'; + path.append(matrix, matrix_len); + } + } + + const char *scheme = nullptr; + const char *host = nullptr; + const char *tspath = nullptr; + std::string path{}; + const char *query = nullptr; + const char *matrix = nullptr; + const char *url = nullptr; + const char *from_path = nullptr; + int port = 0; + + int scheme_len = 0; + int host_len = 0; + int tspath_len = 0; + int query_len = 0; + int matrix_len = 0; + int url_len = 0; + int from_path_len = 0; +}; + +void +setup_memory_allocation() +{ + pcre_malloc = &ink_malloc; + pcre_free = &ink_free; +} + +enum operation_type { UNKNOWN = -1, EXISTS = 1, NOTEXISTS, REGEXP, STRING, BUCKET }; + +enum target_type { + COOKIE = 1, + URI, // URI = PATH + QUERY + UNKNOWN_TARGET +}; + +/*************************************************************************************** + Decimal to Hex +converter + +This is a template function which returns a char* array filled with hex digits +when +passed to it a number(can work as a decimal to hex conversion)and will work for +signed +and unsigned: char, short, and integer(long) type parameters passed to it. + +Shortcomings:It won't work for decimal numbers because of presence of +bitshifting in its algorithm. + +Arguments: + * _num is the number to convert to hex + * hdigits two-byte character array, will be populated with the hex number + +***************************************************************************************/ + +template // template usage to allow multiple types of parameters +void +dec_to_hex(type _num, char *hdigits) +{ + const char *hlookup = "0123456789ABCDEF"; // lookup table stores the hex digits into their + // corresponding index. + + if (_num < 0) + _num *= -1; // and make _num positive to clear(zero) the sign bit + + char mask = 0x000f; // mask will clear(zero) out all the bits except lowest 4 + // which represent a single hex digit + + hdigits[1] = hlookup[mask & _num]; + hdigits[0] = hlookup[mask & (_num >> 4)]; + + return; +} + +void +urlencode(std::string &str) +{ + size_t pos = 0; + std::string replacement; + for (; pos < str.length(); pos++) { + if (!isalnum(str[pos])) { + char dec[2]; + dec_to_hex(str[pos], dec); + std::string replacement = "%" + std::string(dec, 2); + str.replace(pos, 1, replacement); + } + } +} + +//---------------------------------------------------------------------------- +class subop +{ +public: + subop() + : cookie(""), + operation(""), + op_type(UNKNOWN), + target(UNKNOWN_TARGET), + str_match(""), + regex(nullptr), + regex_extra(nullptr), + regex_ccount(0), + bucket(""), + how_many(0), + out_of(0) + { + TSDebug(MY_NAME, "subop constructor called"); + } + + ~subop() + { + TSDebug(MY_NAME, "subop destructor called"); + if (regex) + pcre_free(regex); + + if (regex_extra) + pcre_free(regex_extra); + } + + bool + empty() const + { + return (cookie == "" && operation == "" && op_type == UNKNOWN); + } + + void + setCookieName(const std::string &s) + { + cookie = s; + } + + const std::string & + getCookieName() const + { + return cookie; + } + + const std::string & + getOperation() const + { + return operation; + } + + operation_type + getOpType() const + { + return op_type; + } + + target_type + getTargetType() const + { + return target; + } + + void + setOperation(const std::string &s) + { + operation = s; + + if (operation == "string") { + op_type = STRING; + } + if (operation == "regex") { + op_type = REGEXP; + } + if (operation == "exists") { + op_type = EXISTS; + } + if (operation == "not exists") { + op_type = NOTEXISTS; + } + if (operation == "bucket") { + op_type = BUCKET; + } + } + + void + setTarget(const std::string &s) + { + if (s == "uri") + target = URI; + else + target = COOKIE; + } + + void + setStringMatch(const std::string &s) + { + op_type = STRING; + str_match = s; + } + + const std::string & + getStringMatch() const + { + return str_match; + } + + void + setBucket(const std::string &s) + { + int start_pos = s.find("/"); + + op_type = BUCKET; + bucket = s; + how_many = atoi(bucket.substr(0, start_pos).c_str()); + out_of = atoi(bucket.substr(start_pos + 1).c_str()); + } + + int + bucketGetTaking() const + { + return how_many; + } + + int + bucketOutOf() const + { + return out_of; + } + + bool + setRegexMatch(const std::string &s) + { + const char *error_comp = nullptr; + const char *error_study = nullptr; + int erroffset; + + op_type = REGEXP; + regex_string = s; + regex = pcre_compile(regex_string.c_str(), 0, &error_comp, &erroffset, nullptr); + + if (regex == nullptr) { + return false; + } + regex_extra = pcre_study(regex, 0, &error_study); + if ((regex_extra == nullptr) && (error_study != 0)) { + return false; + } + + if (pcre_fullinfo(regex, regex_extra, PCRE_INFO_CAPTURECOUNT, ®ex_ccount) != 0) + return false; + + return true; + } + + const std::string & + getRegexString() const + { + return regex_string; + } + + int + getRegexCcount() const + { + return regex_ccount; + } + + int + regexMatch(const char *str, int len, int ovector[]) const + { + return pcre_exec(regex, // the compiled pattern + regex_extra, // Extra data from study (maybe) + str, // the subject std::string + len, // the length of the subject + 0, // start at offset 0 in the subject + 0, // default options + ovector, // output vector for substring information + OVECCOUNT); // number of elements in the output vector + }; + + void + printSubOp() const + { + TSDebug(MY_NAME, "\t+++subop+++"); + TSDebug(MY_NAME, "\t\tcookie: %s", cookie.c_str()); + TSDebug(MY_NAME, "\t\toperation: %s", operation.c_str()); + if (str_match.size() > 0) { + TSDebug(MY_NAME, "\t\tmatching: %s", str_match.c_str()); + } + if (regex) { + TSDebug(MY_NAME, "\t\tregex: %s", regex_string.c_str()); + } + if (bucket.size() > 0) { + TSDebug(MY_NAME, "\t\tbucket: %s", bucket.c_str()); + TSDebug(MY_NAME, "\t\ttaking: %d", how_many); + TSDebug(MY_NAME, "\t\tout of: %d", out_of); + } + } + +private: + std::string cookie; + std::string operation; + enum operation_type op_type; + enum target_type target; + + std::string str_match; + + pcre *regex; + pcre_extra *regex_extra; + std::string regex_string; + int regex_ccount; + + std::string bucket; + unsigned int how_many; + unsigned int out_of; +}; + +typedef std::vector SubOpQueue; + +//---------------------------------------------------------------------------- +class op +{ +public: + op() { TSDebug(MY_NAME, "op constructor called"); } + + ~op() + { + TSDebug(MY_NAME, "op destructor called"); + for (SubOpQueue::iterator it = subops.begin(); it != subops.end(); ++it) { + delete *it; + } + } + + void + addSubOp(const subop *s) + { + subops.push_back(s); + } + + void + setSendTo(const std::string &s) + { + sendto = s; + } + + const std::string & + getSendTo() const + { + return sendto; + } + + void + setElseSendTo(const std::string &s) + { + else_sendto = s; + } + + void + setStatus(const std::string &s) + { + if (else_sendto.size() > 0) + else_status = static_cast(atoi(s.c_str())); + else + status = static_cast(atoi(s.c_str())); + } + + void + setElseStatus(const std::string &s) + { + else_status = static_cast(atoi(s.c_str())); + } + + void + printOp() const + { + TSDebug(MY_NAME, "++++operation++++"); + TSDebug(MY_NAME, "sending to: %s", sendto.c_str()); + TSDebug(MY_NAME, "if these operations match: "); + + for (SubOpQueue::const_iterator it = subops.begin(); it != subops.end(); ++it) { + (*it)->printSubOp(); + } + if (else_sendto.size() > 0) + TSDebug(MY_NAME, "else: %s", else_sendto.c_str()); + } + + bool + process(CookieJar &jar, std::string &dest, TSHttpStatus &retstat, TSRemapRequestInfo *rri) const + { + if (sendto == "") { + return false; // guessing every operation must have a + // sendto url??? + } + + int retval = 1; + bool cookie_found = false; + std::string c; + std::string cookie_data; + std::string object_name; // name of the thing being processed, + // cookie, or + // request url + + TSDebug(MY_NAME, "starting to process a new operation"); + + for (SubOpQueue::const_iterator it = subops.begin(); it != subops.end(); ++it) { + // subop* s = *it; + int subop_type = (*it)->getOpType(); + target_type target = (*it)->getTargetType(); + + c = (*it)->getCookieName(); + if (c.length()) { + TSDebug(MY_NAME, "processing cookie: %s", c.c_str()); + + size_t period_pos = c.find_first_of("."); + + if (period_pos == std::string::npos) { // not a sublevel + // cookie name + TSDebug(MY_NAME, "processing non-sublevel cookie"); + + cookie_found = jar.get_full(c, cookie_data); + TSDebug(MY_NAME, "full cookie: %s", cookie_data.c_str()); + object_name = c; + } else { // is in the format FOO.BAR + std::string cookie_main = c.substr(0, period_pos); + std::string cookie_subkey = c.substr(period_pos + 1); + + TSDebug(MY_NAME, "processing sublevel cookie"); + TSDebug(MY_NAME, "c key: %s", cookie_main.c_str()); + TSDebug(MY_NAME, "c subkey: %s", cookie_subkey.c_str()); + + cookie_found = jar.get_part(cookie_main, cookie_subkey, cookie_data); + object_name = cookie_main + " . " + cookie_subkey; + } + // invariant: cookie name is in object_name and + // cookie data (if any) is + // in cookie_data + + if (cookie_found == false) { // cookie name or sub-key not found + // inside cookies + if (subop_type == NOTEXISTS) { + TSDebug(MY_NAME, + "cookie %s was not " + "found (and we wanted " + "that)", + object_name.c_str()); + continue; // we can short + // circuit more + // testing + } + TSDebug(MY_NAME, "cookie %s was not found", object_name.c_str()); + retval &= 0; + break; + } else { + // cookie exists + if (subop_type == NOTEXISTS) { // we found the cookie + // but are asking + // for non existence + TSDebug(MY_NAME, + "cookie %s was found, " + "but operation " + "requires " + "non-existence", + object_name.c_str()); + retval &= 0; + break; + } + + if (subop_type == EXISTS) { + TSDebug(MY_NAME, "cookie %s was found", object_name.c_str()); // got what + // we were + // looking + // for + continue; // we can short + // circuit more + // testing + } + } // handled EXISTS / NOTEXISTS subops + + TSDebug(MY_NAME, "processing cookie data: \"%s\"", cookie_data.c_str()); + } else + target = URI; + + // INVARIANT: we now have the data from the cookie (if + // any) inside + // cookie_data and we are here because we need + // to continue processing this suboperation in some way + + if (!rri) { // too dangerous to continue without the + // rri; hopefully that + // never happens + TSDebug(MY_NAME, "request info structure is " + "empty; can't continue " + "processing this subop"); + retval &= 0; + break; + } + + // If the user has specified a cookie in his + // suboperation, use the cookie + // data for matching; + // otherwise, use the request uri (path + query) + std::string request_uri; // only set the value if we + // need it; we might + // match the cookie data instead + const std::string &string_to_match(target == URI ? request_uri : cookie_data); + if (target == URI) { + UrlComponents req_url; + req_url.populate(rri); + + TSDebug(MY_NAME, "process req_url.path = %s", req_url.path.c_str()); + request_uri = req_url.path; + if (request_uri.length() && request_uri[0] != '/') + request_uri.insert(0, 1, '/'); + if (req_url.query_len > 0) { + request_uri += '?'; + request_uri += std::string(req_url.query, req_url.query_len); + } + object_name = "request uri"; + } + + // invariant: we've decided at this point what string + // we'll match, if we + // do matching + + // OPERATION::string matching + if (subop_type == STRING) { + if (string_to_match == (*it)->getStringMatch()) { + TSDebug(MY_NAME, "string match succeeded"); + continue; + } else { + TSDebug(MY_NAME, "string match failed"); + retval &= 0; + break; + } + } + + // OPERATION::regex matching + if (subop_type == REGEXP) { + int ovector[OVECCOUNT]; + int ret = (*it)->regexMatch(string_to_match.c_str(), string_to_match.length(), ovector); + + if (ret >= 0) { + std::string::size_type pos = sendto.find('$'); + std::string::size_type ppos = 0; + + dest.erase(); // we only reset dest if + // there is a successful + // regex + // match + dest.reserve(sendto.size() * 2); // Wild guess at this + // time ... is + // sucks we can't precalculate this + // like regex_remap. + + TSDebug(MY_NAME, "found %d matches", ret); + TSDebug(MY_NAME, + "successful regex " + "match of: %s with %s " + "rewriting string: %s", + string_to_match.c_str(), (*it)->getRegexString().c_str(), sendto.c_str()); + + // replace the $(1-9) in the sendto url + // as necessary + const size_t LAST_IDX_TO_SEARCH(sendto.length() - 2); // otherwise the below loop can + // access "sendto" out of range + while (pos <= LAST_IDX_TO_SEARCH) { + if (isdigit(sendto[pos + 1])) { + int ix = sendto[pos + 1] - '0'; + + if (ix <= (*it)->getRegexCcount()) { // Just skip an illegal regex group + dest += sendto.substr(ppos, pos - ppos); + dest += string_to_match.substr(ovector[ix * 2], ovector[ix * 2 + 1] - ovector[ix * 2]); + ppos = pos + 2; + } else { + TSDebug(MY_NAME, + "bad " + "rewriting " + "string, " + "for group " + "%d: %s", + ix, sendto.c_str()); + } + } + pos = sendto.find('$', pos + 1); + } + dest += sendto.substr(ppos); + continue; // next subop, please + } else { + TSDebug(MY_NAME, + "could not match " + "regular expression " + "%s to %s", + (*it)->getRegexString().c_str(), string_to_match.c_str()); + retval &= 0; + break; + } + } + + // OPERATION::bucket ranges + if (subop_type == BUCKET) { + unsigned int taking = (*it)->bucketGetTaking(); + unsigned int out_of = (*it)->bucketOutOf(); + + uint32_t hash; + + if (taking == 0 || out_of == 0) { + TSDebug(MY_NAME, + "taking %d out of %d " + "makes no sense?!", + taking, out_of); + retval &= 0; + break; + } + + hash = hash_fnv32_buckets(cookie_data.c_str(), cookie_data.size(), out_of); + TSDebug(MY_NAME, + "we hashed this to bucket: %u " + "taking: %u out of: %u", + hash, taking, out_of); + + if (hash < taking) { + TSDebug(MY_NAME, "we hashed in the range, yay!"); + continue; // we hashed in the range + } else { + TSDebug(MY_NAME, "we didnt hash in the " + "range requested, so " + "sad"); + retval &= 0; + break; + } + } + } + + if (retval == 1) { + if (dest.size() == 0) // Unless already set by one of + // the operators (e.g. regex) + dest = sendto; + if (status > 0) { + retstat = status; + } + return true; + } else if (else_sendto.size() > 0 && retval == 0) { + dest = else_sendto; + if (else_status > 0) { + retstat = else_status; + } + return true; + } else { + dest = ""; + return false; + } + } + +private: + SubOpQueue subops{}; + std::string sendto{""}; + std::string else_sendto{""}; + TSHttpStatus status = TS_HTTP_STATUS_NONE; + TSHttpStatus else_status = TS_HTTP_STATUS_NONE; +}; + +typedef std::pair StringPair; +typedef std::vector OpMap; + +//---------------------------------------------------------------------------- +static bool +build_op(op &o, OpMap const &q) +{ + StringPair m; + + subop *sub = new subop(); + + // loop through the array of key->value pairs + for (auto const &pr : q) { + std::string const &key = pr.first; + std::string const &val = pr.second; + + if (key == "cookie") { + if (!sub->empty()) { + TSDebug(MY_NAME, "ERROR: you need to define a connector"); + goto error; + } + sub->setCookieName(val); + } + + if (key == "sendto" || key == "url") { + o.setSendTo(val); + } + + if (key == "else") { + o.setElseSendTo(val); + } + + if (key == "status") { + o.setStatus(val); + } + + if (key == "operation") { + sub->setOperation(val); + } + + if (key == "target") { + sub->setTarget(val); + } + + if (key == "match") { + sub->setStringMatch(val); + } + + if (key == "regex") { + bool ret = sub->setRegexMatch(val); + + if (!ret) { + goto error; + } + } + + if (key == "bucket" || key == "hash") { + sub->setBucket(val); + } + + if (key == "connector") { + o.addSubOp(sub); + sub = new subop(); + } + } + + o.addSubOp(sub); + return true; + +error: + TSDebug(MY_NAME, "error building operation"); + return false; +} + +typedef std::vector OpsQueue; + +//---------------------------------------------------------------------------- +// init +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) +{ + setup_memory_allocation(); + + return TS_SUCCESS; +} + +//---------------------------------------------------------------------------- +// initialization of structures from config parameters +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) +{ + if (argc != 3) { + TSError("arguments not equal to 3: %d", argc); + TSDebug(MY_NAME, "arguments not equal to 3: %d", argc); + return TS_ERROR; + } + + std::string filename(argv[2]); + try { + YAML::Node config = YAML::LoadFile(filename); + + std::unique_ptr ops(new OpsQueue); + OpMap op_data; + + for (YAML::const_iterator it = config.begin(); it != config.end(); ++it) { + const string &name = it->first.as(); + YAML::NodeType::value type = it->second.Type(); + + if (name != "op" || type != YAML::NodeType::Map) { + const string reason = "Top level nodes must be named op and be of type map"; + TSError("Invalid YAML Configuration format for cookie_remap: %s, reason: %s", filename.c_str(), reason.c_str()); + return TS_ERROR; + } + + for (YAML::const_iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { + const YAML::Node &first = it2->first; + const YAML::Node &second = it2->second; + + if (second.Type() != YAML::NodeType::Scalar) { + const string reason = "All op nodes must be of type scalar"; + TSError("Invalid YAML Configuration format for cookie_remap: %s, reason: %s", filename.c_str(), reason.c_str()); + return TS_ERROR; + } + + const string &key = first.as(); + const string &value = second.as(); + op_data.emplace_back(key, value); + } + + if (op_data.size()) { + op *o = new op(); + if (!build_op(*o, op_data)) { + delete o; + + TSError("building operation, check configuration file: %s", filename.c_str()); + return TS_ERROR; + } else { + ops->push_back(o); + } + o->printOp(); + op_data.clear(); + } + } + + TSDebug(MY_NAME, "# of ops: %d", (int)ops->size()); + *ih = static_cast(ops.release()); + } catch (const YAML::Exception &e) { + TSError("YAML::Exception %s when parsing YAML config file %s for cookie_remap", e.what(), filename.c_str()); + return TS_ERROR; + } + + return TS_SUCCESS; +} + +//---------------------------------------------------------------------------- +// called whenever we need to perform substiturions on a string; used to replace +// things like +// $url, $unmatched_path, $cr_req_url, and $cr_url_encode +// returns 0 if no substitutions, 1 otw. +int +cr_substitutions(std::string &obj, TSRemapRequestInfo *rri) +{ + int retval = 0; + + size_t pos = obj.find('$'); + size_t tmp_pos = 0; + size_t last_pos = pos; + + UrlComponents req_url; + req_url.populate(rri); + TSDebug(MY_NAME, "x req_url.path: %s %zu", req_url.path.c_str(), req_url.path.length()); + TSDebug(MY_NAME, "x req_url.url: %s %d", std::string(req_url.url, req_url.url_len).c_str(), req_url.url_len); + + while (pos < obj.length()) { + if (pos + 1 >= obj.length()) + break; // trailing ? + + switch (obj[pos + 1]) { + case 'p': + if (obj.substr(pos + 1, 4) == "path") { + obj.replace(pos, 5, req_url.path); + pos += req_url.path.length(); + } + break; + case 'u': + if (obj.substr(pos + 1, 14) == "unmatched_path") { + // we found $unmatched_path inside the sendto + // url + std::string unmatched_path(req_url.path); + TSDebug(MY_NAME, "unmatched_path: %s", unmatched_path.c_str()); + TSDebug(MY_NAME, "from_path: %s", req_url.from_path); + + tmp_pos = unmatched_path.find(req_url.from_path, 0, req_url.from_path_len); // from patch + if (tmp_pos != std::string::npos) { + unmatched_path.erase(tmp_pos, req_url.from_path_len); + } + TSDebug(MY_NAME, "unmatched_path: %s", unmatched_path.c_str()); + TSDebug(MY_NAME, "obj: %s", obj.c_str()); + obj.replace(pos, 15, unmatched_path); + TSDebug(MY_NAME, "obj: %s", obj.c_str()); + pos += unmatched_path.length(); + } + break; + case 'c': + if (obj.substr(pos + 1, 3) == "cr_") + switch (obj[pos + 4]) { + case 'r': + if (obj.substr(pos + 4, 7) == "req_url") { + // we found $cr_req_url inside + // the sendto url + std::string request_url; + if (req_url.url && req_url.url_len > 0) { + request_url = std::string(req_url.url, req_url.url_len); + } + TSDebug(MY_NAME, "req_url.url: %s %d", std::string(req_url.url, req_url.url_len).c_str(), req_url.url_len); + obj.replace(pos, 11, request_url); + pos += request_url.length(); + } + break; + case 'u': + // note that this allows for variables + // nested in the function, but not + // for functions nested in the function + if (obj.substr(pos + 4, 10) == "urlencode(" && (tmp_pos = obj.find(')', pos + 14)) != std::string::npos) { + std::string str = obj.substr(pos + 14, tmp_pos - pos - 14); // the string to + // encode + cr_substitutions(str, rri); // expand any + // variables as + // necessary + // before + // encoding + urlencode(str); + obj.replace(pos, tmp_pos - pos + 1, str); // +1 for the + // closing + // parens + pos += str.length(); + } + break; + } + break; + } + if (last_pos == pos) + pos++; // don't search the same place again and again + last_pos = pos; + pos = obj.find('$', pos); + } + + return retval; +} + +//---------------------------------------------------------------------------- +// called on each request +// returns 0 on error or failure to match rules, 1 on a match +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) +{ + OpsQueue *ops = (OpsQueue *)ih; + TSHttpStatus status = TS_HTTP_STATUS_NONE; + + UrlComponents req_url; + req_url.populate(rri); + + if (ops == (OpsQueue *)nullptr) { + TSError("serious error with encountered while attempting to " + "cookie_remap"); + TSDebug(MY_NAME, "serious error with encountered while attempting to remap"); + return TSREMAP_NO_REMAP; + } + + // get any query params..we will append that to the answer (possibly) + std::string client_req_query_params; + if (req_url.query_len) { + client_req_query_params = "?"; + client_req_query_params += std::string(req_url.query, req_url.query_len); + } + TSDebug(MY_NAME, "Query Parameters: %s", client_req_query_params.c_str()); + + std::string rewrite_to; + char cookie_str[] = "Cookie"; + TSMLoc field = TSMimeHdrFieldFind(rri->requestBufp, rri->requestHdrp, cookie_str, sizeof(cookie_str) - 1); + + // cookie header doesn't exist + if (field == nullptr) { + TSDebug(MY_NAME, "no cookie header"); + // return TSREMAP_NO_REMAP; + } + + const char *cookie = nullptr; + int cookie_len = 0; + if (field != nullptr) { + cookie = TSMimeHdrFieldValueStringGet(rri->requestBufp, rri->requestHdrp, field, -1, &cookie_len); + } + std::string temp_cookie(cookie, cookie_len); + CookieJar jar; + jar.create(temp_cookie); + + for (OpsQueue::iterator it = ops->begin(); it != ops->end(); ++it) { + TSDebug(MY_NAME, ">>> processing new operation"); + if ((*it)->process(jar, rewrite_to, status, rri)) { + cr_substitutions(rewrite_to, rri); + + size_t pos = 7; // 7 because we want to ignore the // in + // http:// :) + size_t tmp_pos = rewrite_to.find('?', pos); // we don't want to alter the query string + do { + pos = rewrite_to.find("//", pos); + if (pos < tmp_pos) + rewrite_to.erase(pos, 1); // remove one '/' + } while (pos <= rewrite_to.length() && pos < tmp_pos); + + // Add Query Parameters if not already present + if (!client_req_query_params.empty() && rewrite_to.find("?") == std::string::npos) { + rewrite_to.append(client_req_query_params); + } + + TSDebug(MY_NAME, "rewriting to: %s", rewrite_to.c_str()); + + // Maybe set the return status + if (status > TS_HTTP_STATUS_NONE) { + TSDebug(MY_NAME, "Setting return status to %d", status); + SETHTTPSTATUS(txnp, status); + if ((status == TS_HTTP_STATUS_MOVED_PERMANENTLY) || (status == TS_HTTP_STATUS_MOVED_TEMPORARILY)) { + if (rewrite_to.size() > 8192) { + TSError("Redirect in target " + "URL too long"); + SETHTTPSTATUS(txnp, TS_HTTP_STATUS_REQUEST_URI_TOO_LONG); + } else { + const char *start = rewrite_to.c_str(); + int dest_len = rewrite_to.size(); + + if (TS_PARSE_ERROR == TSUrlParse(rri->requestBufp, rri->requestUrl, &start, start + dest_len)) { + SETHTTPSTATUS(txnp, TS_HTTP_STATUS_INTERNAL_SERVER_ERROR); + TSError("can't parse " + "substituted " + "URL string"); + } else { + rri->redirect = 1; + } + } + } + if (field != nullptr) { + TSHandleMLocRelease(rri->requestBufp, rri->requestHdrp, field); + } + if (rri->redirect) { + return TSREMAP_DID_REMAP; + } else { + return TSREMAP_NO_REMAP; + } + } + + const char *start = rewrite_to.c_str(); + + // set the new url + if (TSUrlParse(rri->requestBufp, rri->requestUrl, &start, start + rewrite_to.length()) == TS_PARSE_ERROR) { + SETHTTPSTATUS(txnp, TS_HTTP_STATUS_INTERNAL_SERVER_ERROR); + TSError("can't parse substituted URL string"); + goto error; + } else { + if (field != nullptr) { + TSHandleMLocRelease(rri->requestBufp, rri->requestHdrp, field); + } + return TSREMAP_DID_REMAP; + } + + // Cleanup + error: + if (field != nullptr) { + TSHandleMLocRelease(rri->requestBufp, rri->requestHdrp, field); + } + return TSREMAP_NO_REMAP; + } + } + + TSDebug(MY_NAME, "could not execute ANY of the cookie remap operations... " + "falling back to default in remap.config"); + + if (field != nullptr) { + TSHandleMLocRelease(rri->requestBufp, rri->requestHdrp, field); + } + return TSREMAP_NO_REMAP; +} + +//---------------------------------------------------------------------------- +// unload +void +TSRemapDeleteInstance(void *ih) +{ + OpsQueue *ops = (OpsQueue *)ih; + + TSDebug(MY_NAME, "deleting loaded operations"); + for (OpsQueue::iterator it = ops->begin(); it != ops->end(); ++it) { + delete *it; + } + + delete ops; + + return; +} diff --git a/plugins/experimental/cookie_remap/cookiejar.cc b/plugins/experimental/cookie_remap/cookiejar.cc new file mode 100644 index 00000000000..be46e2c9439 --- /dev/null +++ b/plugins/experimental/cookie_remap/cookiejar.cc @@ -0,0 +1,255 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "cookiejar.h" +#include "strip.h" +#include + +/* allowed cookie-name definition from RFC + * cookie-name = token + * token = + * token = 1* + * separators = "(" | ")" | "<" | ">" | "@" + * | "," | ";" | ":" | "\" | <"> + * | "/" | "[" | "]" | "?" | "=" + * | "{" | "}" | SP | HT + * CTL = + * SP = + * HT = + */ + +static const int rfc_cookie_name_table[256] = { + /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 00-0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 10-1F */ + 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, /* 20-2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 30-3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 40-4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, /* 50-5F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 60-6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, /* 70-7F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 80-8F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 90-9F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* A0-AF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* B0-BF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* C0-CF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* D0-DF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* E0-EF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0-FF */ +}; + +bool +CookieJar::create(const string &strCookie) +{ + if (strCookie.empty()) { + return false; + } + + if (parse(strCookie, "; ", true, true) != 0) { + return false; + } + + return true; +} + +int +CookieJar::parse(const string &arg, const char *sepstr, bool val_check, bool mainElement) +{ + char *arg_copy; + char *cp; + char *key; + + if ((arg_copy = strdup(arg.c_str())) == nullptr) + return -1; + + cp = arg_copy; + char empty[] = ""; + for (key = strsep(&cp, sepstr); key != nullptr; key = strsep(&cp, sepstr)) { + int val_len; + char *val = strchr(key, '='); + char *addme = nullptr; + + if (val) { + /* split key and value */ + *val++ = '\0'; + val_len = strlen(val); + if (val_len > 0) { + /* if we have DQUOTES around our value then drop them */ + + if (val_len > 1 && val[0] == '"' && val[val_len - 1] == '"') { + val[val_len - 1] = '\0'; + addme = val + 1; + + /* update the value length accordingly */ + + val_len -= 2; + } else { + addme = val; /* We have got a valid value eg: "YL=a" */ + } + + /* verify that the value is valid according to our configured + * opton and possibly strip out invalid characters. */ + + if (val_check && verify_value(addme, val_len) != 0) + continue; + } else { + // Empty cookie case eg: "L=" + addme = empty; + } + } else /* If val is nullptr, then no need to add this kv pair to hashtable, skip this key */ + { + continue; + } + + /* we are going to verify the key name for our top level + * cookie names only so we'll use the val_check variables + * to know what we're processing */ + + if (val_check && verify_name(key) != 0) + continue; + + if (mainElement) + addElement(key, addme); + else + addSubElement(key, addme); + } + + free(arg_copy); + return 0; +} + +void +CookieJar::addElement(const char *key, const char *val) +{ + /* The insert method avoids duplicates */ + CookieVal cval; + cval.m_val = val; + m_jar.insert(std::make_pair(key, cval)); +} + +void +CookieJar::addSubElement(const char *key, const char *val) +{ + /* The insert method avoids duplicates */ + m_currentVal->m_subelements.insert(std::make_pair(key, val)); +} + +int +CookieJar::verify_value(char *val, int val_len) +{ + char *buf; + char *data_ptr = nullptr; + char data_buf[1024] = { + 0, + }; + int buf_len; + + if (val_len > static_cast(sizeof(data_buf) - 1)) { + buf_len = val_len + 1; + if ((data_ptr = static_cast(malloc(buf_len))) == nullptr) + return -1; + + buf = data_ptr; + } else { + buf = data_buf; + buf_len = sizeof(data_buf); + } + + if (get_stripped(val, val_len, buf, &buf_len, 0) != STRIP_RESULT_OK) { + if (data_ptr) + free(data_ptr); + return -1; + } + + /* returned buf_len includes the null terminator + * so we need to verify that somehow we didn't end up + * with a bigger buffer than what we started with */ + + if (buf_len > val_len + 1) { + if (data_ptr) + free(data_ptr); + return -1; + } + + memcpy(val, buf, buf_len); + if (data_ptr) + free(data_ptr); + + return 0; +} + +int +CookieJar::verify_name(char *name) +{ + char *p; + + for (p = name; *p; p++) { + /* if we get any invalid characters then return failure + * in order to skip this cookie completely */ + + if (rfc_cookie_name_table[(int)*p] == 0) + return -1; + } + + return 0; +} + +bool +CookieJar::get_full(const string &cookie_name, string &val) +{ + if (m_jar.find(cookie_name) != m_jar.end()) { + val = m_jar[cookie_name].m_val; + return true; + } + return false; +} + +bool +CookieJar::get_part(const string &cookie_name, const string &part_name, string &val) +{ + if (m_jar.empty()) + return false; + + if (m_jar.find(cookie_name) == m_jar.end()) { + /* full cookie not found */ + return false; + } + + CookieVal &fe = m_jar[cookie_name]; + /* check if we need to do lazy evaluation */ + if (fe.parts_inited == false) { + /* since we already validated the value for the full cookie + * there is no need to validate the components again so + * we'll be passing 0/false for the val_check argument */ + + m_currentVal = &fe; + if (parse(fe.m_val, "&", false, false) != 0) + return false; + + fe.parts_inited = true; + m_currentVal = nullptr; + } + + if (fe.m_subelements.find(part_name) == fe.m_subelements.end()) { + return false; /* not found */ + } + + val = fe.m_subelements[part_name]; + return true; +} diff --git a/plugins/experimental/cookie_remap/cookiejar.h b/plugins/experimental/cookie_remap/cookiejar.h new file mode 100644 index 00000000000..65d1c9bd112 --- /dev/null +++ b/plugins/experimental/cookie_remap/cookiejar.h @@ -0,0 +1,62 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef CKREMAP_COOKIEJAR_H_ +#define CKREMAP_COOKIEJAR_H_ + +#include +#include + +using std::string; +using std::unordered_map; + +class CookieJar +{ +public: + CookieJar(){}; + ~CookieJar(){}; + + bool create(const string &strCookie); + + CookieJar(const CookieJar &) = delete; + CookieJar &operator=(const CookieJar &) = delete; + + bool get_full(const string &cookie_name, string &val); + bool get_part(const string &cookie_name, const string &part_name, string &val); + +private: + int verify_name(char *name); + int verify_value(char *val, int val_len); + int parse(const string &arg, const char *sepstr, bool val_check, bool mainElement); + + void addElement(const char *key, const char *val); + void addSubElement(const char *key, const char *val); + + class CookieVal + { + public: + unordered_map m_subelements; + string m_val; + bool parts_inited = 0; + }; + CookieVal *m_currentVal = nullptr; + + unordered_map m_jar; +}; + +#endif // CKREMAP_COOKIEJAR_H_ diff --git a/plugins/experimental/cookie_remap/hash.c b/plugins/experimental/cookie_remap/hash.c new file mode 100644 index 00000000000..a1bdbb5ba4f --- /dev/null +++ b/plugins/experimental/cookie_remap/hash.c @@ -0,0 +1,110 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "hash.h" +#include +#include + +//////////////////////////////////////////////////////////////////////////////// +// Implementation of the Fowler–Noll–Vo hash function // +// More Details at: http://www.isthe.com/chongo/tech/comp/fnv/ // +//////////////////////////////////////////////////////////////////////////////// + +/* + * 32/64 bit magic FNV primes + * The main secret of the algorithm is in these prime numbers + * and their special relation to 2^32 (or 2^64) [a word] + * and 2^8 [a byte]. + */ +#define FNV_32_PRIME ((uint32_t)0x01000193UL) +#define FNV_64_PRIME ((uint64_t)0x100000001b3ULL) + +/* + * The init value is quite arbitrary, but these seem to perform + * well on both web2 and sequential integers represented as strings. + */ +#define FNV1_32_INIT ((uint32_t)33554467UL) +#define FNV1_64_INIT ((uint64_t)0xcbf29ce484222325ULL) + +#define MAX_UINT32 (~((uint32_t)0)) +#define MAX_UINT64 (~((uint64_t)0)) + +#define MASK(x) (((uint32_t)1 << (x)) - 1) + +static uint32_t +fnv32_nbits(const char *buf, int len, int nbits) +{ + uint32_t hash; + hash = hash_fnv32_buf(buf, len); + + if (nbits <= 16) { + hash = ((hash >> nbits) ^ hash) & MASK(nbits); + } else { + hash = (hash >> nbits) ^ (hash & MASK(nbits)); + } + + return hash; +} + +uint32_t +hash_fnv32_buckets(const char *buf, size_t len, uint32_t num_buckets) +{ + uint32_t hash; + uint32_t retry; + int first_bit; + + if (num_buckets < 1) { + return 0; + } + + first_bit = ffs(num_buckets); + + if (num_buckets >> first_bit == 0) { /* Power of two */ + /* Yay we can xor fold */ + hash = fnv32_nbits(buf, len, first_bit - 1); + return hash; + } + + /* Can't xor fold so use the retry method */ + + hash = hash_fnv32_buf(buf, len); + + /* This code ensures there is no bias against larger values */ + retry = (MAX_UINT32 / num_buckets) * num_buckets; + while (hash >= retry) { + hash = (hash * FNV_32_PRIME) + FNV1_32_INIT; + } + + hash %= num_buckets; + return hash; +} + +/* 32-bit version */ +uint32_t +hash_fnv32_buf(const char *buf, size_t len) +{ + uint32_t val; /* initial hash value */ + + for (val = FNV1_32_INIT; len > 0; --len) { + val *= FNV_32_PRIME; + val ^= (uint32_t)(*buf); + ++buf; + } + + return val; +} diff --git a/plugins/experimental/cookie_remap/hash.h b/plugins/experimental/cookie_remap/hash.h new file mode 100644 index 00000000000..daeffc2df30 --- /dev/null +++ b/plugins/experimental/cookie_remap/hash.h @@ -0,0 +1,54 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _CKREMAP_HASH_H_ +#define _CKREMAP_HASH_H_ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * 32-bit Fowler / Noll / Vo (FNV) Hash. + * + * see http://www.isthe.com/chongo/tech/comp/fnv/index.html + */ +uint32_t hash_fnv32_buf(const char *buf, size_t len); + +/** + * Computes an fnv32 hash whose value is less than num_buckets + * + * This functions computes an fnv32 between zero and num_buckets - 1. + * It computes an fnv32 hash and collapses that hash into a smaller + * range using techniques which avoid the bias in a simple mod + * operation. + * + * This function has the best performance (speed and hash distribution) + * if num_buckets is a power of two. + */ +uint32_t hash_fnv32_buckets(const char *buf, size_t len, uint32_t num_buckets); + +#ifdef __cplusplus +} +#endif + +#endif /* _CKREMAP_HASH_H_ */ diff --git a/plugins/experimental/cookie_remap/strip.c b/plugins/experimental/cookie_remap/strip.c new file mode 100644 index 00000000000..d5965730c8a --- /dev/null +++ b/plugins/experimental/cookie_remap/strip.c @@ -0,0 +1,282 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "strip.h" + +static int copy_whitespace(const char **r, const char *in_end, char **w, const char *out_end); + +static int strip_whitespace(const char **r, const char **in_end); + +/* Determine if there is room to store len bytes starting at p for an + * object that ends at maxp. This is not as simple as a less-than + * comparison, because our code may increment p well beyond the end of + * the object it originally pointed to (in complete violation of what + * ANSI C says is legitimate). The result is that p may wrap around. + * This has been observed with using stack buffers as arguments + * from 32 bit programs running on 64-bit RHEL. + */ +#define ROOM(p, len, maxp) (((maxp) - ((p) + (len))) >= 0) + +/* write c into *p if there's room, always incrementing *p. This + * implementation uses a do-loop to avoid several syntactic issues + * when this macro is expanded in the context of if-then-else constructs. + * It is expected that the compiler will optimize away the "while (0)" + */ +#define WRITE_C_IF_ROOM(p, maxp, c) \ + do { \ + if (ROOM(*(p), 1, (maxp))) \ + **(p) = (c); \ + (*(p))++; \ + } while (0) + +/* write s into *p if there's room, always adding slen to *p . This + * implementation uses a do-loop to avoid several syntactic issues + * when this macro is expanded in the context of if-then-else constructs. + * It is expected that the compiler will optimize away the "while 0" + */ +#define WRITE_STR_IF_ROOM(p, maxp, s, slen) \ + do { \ + if (ROOM(*(p), (slen), (maxp))) \ + memcpy(*(p), (s), (slen)); \ + *(p) += (slen); \ + } while (0) + +/* Write count spaces into *p if there's room, always adding count to *p. + * The count argument is set to zero at the end of execution. This + * implementation uses a do-loop to avoid several syntactic issues when + * this macro is expanded in the context of if-then-else constructs. + * It is expected that the compiler will optimize away the "while 0" + */ +#define WRITE_SPACES_IF_ROOM(p, maxp, slen) \ + do { \ + if (ROOM(*(p), (slen), (maxp))) \ + memset(*(p), ' ', (slen)); \ + *(p) += (slen); \ + (slen) = 0; \ + } while (0) + +/* + * File-scope data + */ + +static const unsigned int allowed_flags = (STRIP_FLAG_LEAVE_WHITESP | STRIP_FLAG_STRIP_LOW | STRIP_FLAG_STRIP_HIGH | + STRIP_FLAG_UNSAFE_QUOTES | STRIP_FLAG_UNSAFE_SLASHES | STRIP_FLAG_UNSAFE_SPACES); + +static int +stripped_core(const char *r, const char *in_end, char **w, const char *out_end, unsigned int flags) +{ + int leading = 1; /* haven't yet written a non-space */ + int in_js_entity = 0; /* are we inside a javascript entity? */ + char in_quote_char = '\0'; /* in quoted region? which kind: '\'' or '"' */ + int space = 0; /* number of spaces pending */ + int stripped = 0; /* have we stripped since last output? */ + int in_tag = 0; /* are we inside a tag? */ + + /* parse the string, stripping risky characters/sequences */ + for (/* already established */; r < in_end; r++) { + unsigned char c = *r; + if (in_tag) { + switch (c) { + case '>': + if (!in_quote_char) { + in_tag = 0; + } + break; + + case '"': + case '\'': + if (!in_quote_char) { + in_quote_char = c; + } else if (in_quote_char == c) { + in_quote_char = '\0'; + } + break; + + default: + break; /* eat everything between < and > */ + } + } else if (in_js_entity) { + switch (c) { + case '}': + if (!in_quote_char) { + in_js_entity = 0; + if (r + 1 < in_end && *(r + 1) == ';') { + r++; + } + } + break; + + case '"': + case '\'': + if (!in_quote_char) { + in_quote_char = c; + } else if (in_quote_char == c) { + in_quote_char = '\0'; + } + break; + + default: + break; /* eat everything between < and > */ + } + } else { + if (c == '<') { + in_tag = 1; + stripped = 1; + } else if (c == '&' && r + 1 < in_end && *(r + 1) == '{') { + in_js_entity = 1; + stripped = 1; + r++; + } else if ((c < 0x07 && (flags & STRIP_FLAG_STRIP_LOW)) || (c >= 0x80 && (flags & STRIP_FLAG_STRIP_HIGH)) || + (c == '"' && !(flags & STRIP_FLAG_UNSAFE_QUOTES)) || (c == '\'' && !(flags & STRIP_FLAG_UNSAFE_QUOTES)) || + (c == '\\' && !(flags & STRIP_FLAG_UNSAFE_SLASHES)) || c == '>') { + stripped = 1; + } else if (c == ' ') { + space++; /* don't collapse existing spaces */ + } else { + /* we're ready to write an output character */ + if (leading) { + leading = 0; /* first non-whitespace character */ + stripped = 0; + if (!(flags & STRIP_FLAG_LEAVE_WHITESP)) { + space = 0; + } + } + + /* flush pending spaces */ + if (!space && stripped && !(flags & STRIP_FLAG_UNSAFE_SPACES)) { + space = 1; /* replace stripped sequence with space */ + } + stripped = 0; /* reset until next stripped sequence */ + WRITE_SPACES_IF_ROOM(w, out_end, space); + + /* Process as single character. */ + WRITE_C_IF_ROOM(w, out_end, c); + } + } + } + + /* Restore trailing whitespace if asked */ + if (flags & STRIP_FLAG_LEAVE_WHITESP) + WRITE_SPACES_IF_ROOM(w, out_end, space); + + return STRIP_RESULT_OK; +} + +int +get_stripped(const char *in, ssize_t in_len, char *out, int *out_len, unsigned int flags) +{ + int retval = STRIP_RESULT_OK; + const char *r, *in_end; /* where we read from, read limit */ + char *w, *out_end; /* where we write to, write limit */ + + /* validate params */ + if (in == NULL || in_len < 0 || out_len == NULL || *out_len < 0 || (out == NULL && *out_len > 0) || (flags & (~allowed_flags))) { + if (out != NULL && out_len != NULL && *out_len > 0) { + *out = '\0'; + *out_len = 1; + } + return STRIP_RESULT_BAD_PARAM; + } + + /* make room for null terminator in output and remove if present in in */ + (*out_len) -= out ? 1 : 0; /* make space for '\0' unless NULL out */ + if (in_len > 0 && in[in_len - 1] == '\0') { + in_len--; /* don't count null terminator in input */ + } + + /* establish our read and write limits */ + r = in; + w = out; + in_end = in + in_len; + out_end = out + *out_len; + + /* strip leading and trailing whitespace, unless asked not to */ + if (!(flags & STRIP_FLAG_LEAVE_WHITESP)) { + strip_whitespace(&r, &in_end); + } else { + copy_whitespace(&r, in_end, &w, out_end); + } + + /* handle empty input case (null terminated or not) */ + if ((!(flags & STRIP_FLAG_LEAVE_WHITESP) && r >= in_end) || ((flags & STRIP_FLAG_LEAVE_WHITESP) && in_len == 0)) { + WRITE_C_IF_ROOM(&w, out_end, '\0'); /* make out empty string */ + *out_len = 1; + return STRIP_RESULT_EMPTY_IN; /* input is empty string */ + } + + /* call the core function that does actual checking and stripping */ + retval = stripped_core(r, in_end, &w, out_end, flags); + + /* null terminate */ + out_end += out_end ? 1 : 0; /* undo decrement at start */ + WRITE_C_IF_ROOM(&w, out_end, '\0'); /* try to term at end of output */ + + /* report the required/used length */ + *out_len = w - out; + + /* see if we ran out of space, but were otherwise ok */ + if (w > out_end && retval == STRIP_RESULT_OK) { + retval = STRIP_RESULT_OUTLEN_SMALL; + } + + if (retval != STRIP_RESULT_OK) { + /* return the empty string on all errors */ + WRITE_C_IF_ROOM(&out, out_end, '\0'); /* make out the empty string */ + if (retval != STRIP_RESULT_OUTLEN_SMALL) { + *out_len = 1; /* even if retried, we won't use more than 1 byte */ + } + } + + return retval; +} + +/* + * Copy sequence of whitespace from r to w + */ +static int +copy_whitespace(const char **r, const char *in_end, char **w, const char *out_end) +{ + char c; + while (*r < in_end && (c = **r) && (c == ' ' || c == '\t' || c == '\r' || c == '\n')) { + WRITE_C_IF_ROOM(w, out_end, c); + (*r)++; + } + return 0; +} + +static int +strip_whitespace(const char **r, const char **in_end) +{ + char c; + while (*r < *in_end && (c = **r) && (c == ' ' || c == '\t' || c == '\r' || c == '\n')) { + (*r)++; + } + while (*in_end > *r && (c = *((*in_end) - 1)) && (c == ' ' || c == '\t' || c == '\r' || c == '\n')) { + (*in_end)--; + } + return 0; +} diff --git a/plugins/experimental/cookie_remap/strip.h b/plugins/experimental/cookie_remap/strip.h new file mode 100644 index 00000000000..cf76f67a64d --- /dev/null +++ b/plugins/experimental/cookie_remap/strip.h @@ -0,0 +1,146 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef CKREMAP_IV_H +#define CKREMAP_IV_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* return codes */ +#define STRIP_RESULT_OK 0 /**< success */ +#define STRIP_RESULT_BAD_PARAM -1 /**< one or more invalid arguments */ +#define STRIP_RESULT_OUTLEN_SMALL -2 /**< output buffer not large enough */ +#define STRIP_RESULT_EMPTY_IN -3 /**< in consists solely of whitespace */ + +/* defined flags */ +#define STRIP_FLAG_NONE 0x0 /**< no flags */ +#define STRIP_FLAG_STRIP_LOW 0x1 /**< stripped, html: strip low */ +#define STRIP_FLAG_STRIP_HIGH 0x2 /**< stripped, html: strip high */ +#define STRIP_FLAG_LEAVE_WHITESP 0x4 /**< all: avoid trimming spaces */ +#define STRIP_FLAG_UNSAFE_QUOTES 0x8 /**< html: dont encode quotes */ +#define STRIP_FLAG_UNSAFE_SLASHES 0x10 /**< all: dont encode backslashes */ +#define STRIP_FLAG_UNSAFE_SPACES 0x20 /**< html: stipped tag isnt space */ + +/** Output the input after stripping all characters that are + * unsafe in an HTML context. + * + * This function performs the following treatment: + * + * - strips from a '<' to the next unquoted '>' + * + * - strips "&{" to the next unquoted '}' or "};" + * + * - strips the character '>' + * + * - strips the following characters: '\'', '"', unless the flag + * STRIP_FLAG_UNSAFE_QUOTES is present. + * + * - strips the character '\\' unless the flag STRIP_FLAG_UNSAFE_SLASHES + * is present. + * + * - leaves a single space character in place of each + * sequence of stripped characters if no other space + * preceeded the stripped sequence (e.g., "a b" becomes + * "a b", but "ab" becomes "a b") + * + * @param[in] in character array (string) + * @param[in] in_len number of bytes in character array + * @param[in,out] out (in)storage for the resulting character array, + * (out)contains the null terminated result + * @param[in,out] out_len (in)number of available bytes in out parameter, + * (out)number of bytes used or required in out + * (including NUL) + * @param[in] flags zero or more function-specific flags: + * - STRIP_FLAG_LEAVE_WHITESP - Leave whitespace in place. + * - STRIP_FLAG_STRIP_LOW - Strip all values <= 0x07. + * - STRIP_FLAG_STRIP_HIGH - Strip all values >= 0x80. + * - STRIP_FLAG_UNSAFE_QUOTES - Leave apos and quote in place. + * - STRIP_FLAG_UNSAFE_SLASHES - Leave backslashes in place. + * - STRIP_FLAG_UNSAFE_SPACES - Leave spaces in place. + * @param[in] charset NULL or the charset used to treat the input + * + * @return + * IV_RESULT_OK(0) on success. Non-zero on failure. + * See the complete list above. + * + * @verbatim + * EXAMPLES + * + * Input Output + * ----------------------------- ----------------------------- + * Bob & Mary Bob & Mary + * "a phrase" a phrase + * Alice and Bob's house Alice and Bob s house + * @endverbatim + * + * + * DESIGN NOTES + * + * The function assumes ISO-8859-1 encoding. + * + * - the caller is responsible for managing memory; the code + * does not allocate memory + * + * - The function requires the input length to be specified + * and place the required and/or used length in the + * out_len parameter; this allows them to be efficiently + * used in environments that store something other than + * null terminated strings and also allows you to never + * call a function more than once (if the output buffer is + * too small on the first call, the value of out_len tells + * you how long the buffer needs to be) + * + * - all length parameters specify the length of the + * corresponding string in bytes, not characters + * + * - the input buffer does not need to be null-terminated + * + * - the output is always null-terminated (except when it is + * not possible -- out==NULL || *out_len==0) + * + * - no context is retained between calls + * + * NOTES + * + * - leading and trailing whitespace is stripped by default; + * pass STRIP_FLAG_LEAVE_WHITESP to leave leading and + * trailing whitespace in place + * + * - when called with out_len less than the required + * length, IV_RESULT_OUTLEN_SMALL is returned with the + * required length in out_len and out set to the empty + * string + * + * - can be called with out_len == 0 and out == NULL to + * compute sufficient storage size, which is returned + * in out_len + * + * - when called with out_len == 0, the output is not + * NUL-terminated + */ +int get_stripped(const char *in, ssize_t in_len, char *out, int *out_len, unsigned int flags); + +#ifdef __cplusplus +} +#endif + +#endif /* CKREMAP_IV_H */ diff --git a/plugins/experimental/cookie_remap/unit_tests/cookiejar/unit_test.cc b/plugins/experimental/cookie_remap/unit_tests/cookiejar/unit_test.cc new file mode 100644 index 00000000000..cacf8575a15 --- /dev/null +++ b/plugins/experimental/cookie_remap/unit_tests/cookiejar/unit_test.cc @@ -0,0 +1,250 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file +#include +#include +#include +#include "catch.hpp" +#include "cookiejar.h" + +using std::cout; +using std::endl; +using std::string; + +TEST_CASE("Basic test with ; separated cookies", "[CookieJar]") +{ + std::cerr << "Verify individual cookie crumbs (semicolon separated)" << std::endl; + CookieJar oreo; + bool rc = oreo.create("fp=1;fn=2;sp=3;tl=4"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("fp", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_full("fn", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_full("sp", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_full("tl", val); + REQUIRE(rc == true); + REQUIRE(val == "4"); + + rc = oreo.get_full("doesnotexist", val); + REQUIRE(rc == false); +} + +TEST_CASE("Basic test with space separated cookies", "[CookieJar]") +{ + std::cerr << "Verify individual cookie crumbs (space separated)" << std::endl; + CookieJar oreo; + bool rc = oreo.create("fp=1 fn=2 sp=3 tl=4"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("fp", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_full("fn", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_full("sp", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_full("tl", val); + REQUIRE(rc == true); + REQUIRE(val == "4"); + + rc = oreo.get_full("doesnotexist", val); + REQUIRE(rc == false); +} + +TEST_CASE("Basic test with mixed delimiters", "[CookieJar]") +{ + std::cerr << "Verify individual cookie crumbs (mixed delimiters)" << std::endl; + CookieJar oreo; + bool rc = oreo.create("fp=1;fn=2 ; sp=3 ;; ; tl=4"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("fp", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_full("fn", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_full("sp", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_full("tl", val); + REQUIRE(rc == true); + REQUIRE(val == "4"); + + rc = oreo.get_full("doesnotexist", val); + REQUIRE(rc == false); +} + +TEST_CASE("Test with some empty values", "[CookieJar]") +{ + std::cerr << "Verify empty values" << std::endl; + CookieJar oreo; + bool rc = oreo.create("lastname=whatever;firstname=;age=100;salary=;dept=engineering"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("lastname", val); + REQUIRE(rc == true); + REQUIRE(val == "whatever"); + + rc = oreo.get_full("firstname", val); + REQUIRE(rc == true); + REQUIRE(val == ""); + + rc = oreo.get_full("age", val); + REQUIRE(rc == true); + REQUIRE(val == "100"); + + rc = oreo.get_full("salary", val); + REQUIRE(rc == true); + REQUIRE(val == ""); + + rc = oreo.get_full("dept", val); + REQUIRE(rc == true); + REQUIRE(val == "engineering"); +} + +TEST_CASE("Verify double quotes around values are stripped", "[CookieJar]") +{ + std::cerr << "Verify double quotes around values are stripped" << std::endl; + CookieJar oreo; + bool rc = oreo.create("lang=c;vcs=\"git\""); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("vcs", val); + REQUIRE(rc == true); + REQUIRE(val == "git"); +} + +TEST_CASE("Discard invalid cookie names", "[CookieJar]") +{ + std::cerr << "Discard invalid cookie names" << std::endl; + + // [] cannot be used in cookie names + CookieJar oreo; + bool rc = oreo.create("t=2;x=3;[invalid]=4;valid=5"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("t", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_full("x", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_full("valid", val); + REQUIRE(rc == true); + REQUIRE(val == "5"); + + rc = oreo.get_full("[invalid]", val); + REQUIRE(rc == false); +} + +TEST_CASE("Handle null values", "[CookieJar]") +{ + std::cerr << "Handle null values" << std::endl; + CookieJar oreo; + + // perl's value will be "" + // ancient will not be found because there is no = following it + // = will not be found because there is no = following it + // python will be "modern" + bool rc = oreo.create("perl= ancient =;python=modern"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("perl", val); + REQUIRE(rc == true); + REQUIRE(val == ""); + + rc = oreo.get_full("ancient", val); + REQUIRE(rc == false); + + rc = oreo.get_full("=", val); + REQUIRE(rc == false); + + rc = oreo.get_full("python", val); + REQUIRE(rc == true); + REQUIRE(val == "modern"); +} + +TEST_CASE("Verify subcookies are parsed", "[CookieJar]") +{ + std::cerr << "Verify subcookies are parsed" << std::endl; + CookieJar oreo; + + bool rc = oreo.create("team1=spiderman=1&ironman=2&batman=3;team2=thor=1&wonderwoman=2&antman=3;superhero3=spiderman"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("team1", val); + REQUIRE(rc == true); + REQUIRE(val == "spiderman=1&ironman=2&batman=3"); + + rc = oreo.get_full("superhero3", val); + REQUIRE(rc == true); + REQUIRE(val == "spiderman"); + + rc = oreo.get_part("team1", "spiderman", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_part("team1", "ironman", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_part("team1", "batman", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_part("team2", "thor", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_part("team2", "wonderwoman", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_part("team2", "antman", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); +} diff --git a/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py new file mode 100644 index 00000000000..4e1863a6ccd --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py @@ -0,0 +1,104 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: cookie in bucket or not" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/bucketconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/bucketconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/bucketconfig.txt' +) + +# Cookie value in bucket +tr = Test.AddTestRun("cookie value in bucket") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=333" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie.gold" + +# cookie value not in bucket +tr = Test.AddTestRun("cooke value not in bucket") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py new file mode 100644 index 00000000000..48198996aa1 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py @@ -0,0 +1,71 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: plugin collapses consecutive slashes" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# We only run a server to capture ATS outbound requests +# and verify it collapsed the double // +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/collapseconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/collapseconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/collapseconfig.txt' +) + +tr = Test.AddTestRun("collapse consecutive forward slashes") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/collapseslashes.gold" + diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/bucketconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/bucketconfig.txt new file mode 100644 index 00000000000..2af72d0d1da --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/bucketconfig.txt @@ -0,0 +1,9 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: bucket + bucket: 30/100 + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/collapseconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/collapseconfig.txt new file mode 100644 index 00000000000..11693a0a508 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/collapseconfig.txt @@ -0,0 +1,6 @@ +# This is a test configuration + +# Do a regex against the request url +op: + regex: magic + sendto: http://127.0.0.10:$PORT/i/////////like/cheetos?.done=http://finance.yahoo.com diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/connectorconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/connectorconfig.txt new file mode 100644 index 00000000000..a35a1a6efb2 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/connectorconfig.txt @@ -0,0 +1,16 @@ +# This is a test configuration + +# First 2 ops do a comparison against the cookie and the 3rd +# one does against the url +op: + cookie: fpbeta + operation: string + match: abcd + connector: and + cookie: icecream + operation: exists + connector: and + operation: regex + regex: magic + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/existsconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/existsconfig.txt new file mode 100644 index 00000000000..8b3189a86bc --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/existsconfig.txt @@ -0,0 +1,8 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: exists + sendto: http://127.0.0.10:$PORT/cookieexists + else: http://127.0.0.11:$ALTPORT/cookiedoesntexist diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/matchconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/matchconfig.txt new file mode 100644 index 00000000000..a7b850a6357 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/matchconfig.txt @@ -0,0 +1,8 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + match: magic + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/matchuriconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/matchuriconfig.txt new file mode 100644 index 00000000000..3272826ea71 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/matchuriconfig.txt @@ -0,0 +1,17 @@ +# This is a test configuration + +# Do a regex against the request url +op: + match: dkfdkfkdfkdfkdfkdfk + sendto: http://nothere.com +op: + match: eioreioreioreioeio + sendto: http://nothereeither.com +op: + match: mmmdm,cmvmcnxcxncn + sendto: http://nothereaswell.com +op: + operation: regex + regex: thisispartofthepath + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/matrixconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/matrixconfig.txt new file mode 100644 index 00000000000..263c957359d --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/matrixconfig.txt @@ -0,0 +1,24 @@ +# This is a test configuration + +# Do a regex against the url + +op: + operation: regex + regex: eleventh + sendto: http://127.0.0.10:$PORT/$path/eleventh;matrix=2?query=true + +op: + operation: regex + regex: tenth + sendto: http://127.0.0.10:$PORT/$path/tenth;matrix=2 + +op: + operation: regex + regex: ninth + sendto: http://127.0.0.10:$PORT/$path/ninth?query=true + +op: + operation: regex + regex: eighth + sendto: http://127.0.0.10:$PORT/$path/eighth + diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/notexistsconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/notexistsconfig.txt new file mode 100644 index 00000000000..72a2c19eb5d --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/notexistsconfig.txt @@ -0,0 +1,8 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: not exists + sendto: http://127.0.0.10:$PORT/cookiedoesntexist + else: http://127.0.0.11:$ALTPORT/cookieexists diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/regexconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/regexconfig.txt new file mode 100644 index 00000000000..04e7fdd6d6f --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/regexconfig.txt @@ -0,0 +1,9 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: regex + regex: ilove-(\w+)-(\w+)-(\w+) + sendto: http://127.0.0.10:$PORT/regexmatches?cookies=$1-$2-$3 + else: http://127.0.0.11:$ALTPORT/regexdoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/statusconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/statusconfig.txt new file mode 100644 index 00000000000..6d66e8d12ff --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/statusconfig.txt @@ -0,0 +1,17 @@ +# This is a test configuration +# When status is set, ATS doesn't follow the sendto or else +# It just returns a canned response + +# Do a regex against the cookie +op: + cookie: fpbeta + match: magic + sendto: http://shouldnevergohere.com + status: 205 +op: + cookie: abracadabra + match: magic + sendto: http://shouldnevergohere.com + else: http://shouldnotgohereaswell.com + # When a else is present, status is associated with it + status: 400 diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/subcookie.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/subcookie.txt new file mode 100644 index 00000000000..f34be180867 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/subcookie.txt @@ -0,0 +1,17 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta.a + operation: string + match: 1 + connector: and + cookie: fpbeta.b + operation: string + match: 2 + connector: and + cookie: fpbeta.c + operation: string + match: 3 + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/substituteconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/substituteconfig.txt new file mode 100644 index 00000000000..120adc35f4e --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/substituteconfig.txt @@ -0,0 +1,21 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: exists + sendto: http://127.0.0.10:$PORT/photos/search?query=$path +op: + cookie: oxalpha + operation: exists + sendto: http://127.0.0.10:$PORT/photos/search?query=$unmatched_path +op: + cookie: acgamma + operation: exists + sendto: http://127.0.0.10:$PORT/photos/search/cr_substitutions?query=$cr_urlencode($cr_req_url) +# Regex against url and path is substituted in outgoing path +op: + operation: regex + regex: foobar + sendto: http://127.0.0.10:$PORT/photos/search/$path + diff --git a/tests/gold_tests/pluginTest/cookie_remap/connector.test.py b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py new file mode 100644 index 00000000000..378b880dbeb --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py @@ -0,0 +1,104 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: test connector" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/connectorconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/connectorconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/connectorconfig.txt' +) + +# Positive test case that remaps because all connected operations pass +tr = Test.AddTestRun("cookie value matches") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=abcd icecream=donteat" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie.gold" + +# Negative test case that doesn't remap because not all subops pass +tr = Test.AddTestRun("cookie value doesn't match") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=somethingelse" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py new file mode 100644 index 00000000000..6f07b02631c --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py @@ -0,0 +1,104 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: cookie exists" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookieexists HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntexist HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/existsconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/existsconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/existsconfig.txt' +) + +# Positive test case that remaps because cookie exists +tr = Test.AddTestRun("cookie fpbeta exists") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=fdkfkdfkdfkdkdfkdfk" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/existscookie.gold" + +# Negative test case that doesn't remap because cookie doesn't exist +tr = Test.AddTestRun("cooke fpbeta doesnt exist") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: breadcrumb=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/doesntexistcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold new file mode 100644 index 00000000000..68802f1e736 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold @@ -0,0 +1,3 @@ +`` +Request=GET /i/like/cheetos?.done=http://finance.yahoo.com HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold new file mode 100644 index 00000000000..d137bee4ec8 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold @@ -0,0 +1,3 @@ +`` +Request=GET /cookiedoesntexist HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold new file mode 100644 index 00000000000..9bae354b2c6 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold @@ -0,0 +1,3 @@ +`` +Request=GET /cookieexists HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold new file mode 100644 index 00000000000..fe1811dbd5f --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold @@ -0,0 +1,3 @@ +`` +Request=GET /cookiematches HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold new file mode 100644 index 00000000000..969256d8d40 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold @@ -0,0 +1,3 @@ +`` +Request=GET /cookiematches?a=1&b=2&c=3 HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchelsestatus.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchelsestatus.gold new file mode 100644 index 00000000000..5c4de12cae2 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchelsestatus.gold @@ -0,0 +1,3 @@ +`` +HTTP/1.1 400 Bad Request +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchstatus.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchstatus.gold new file mode 100644 index 00000000000..c396c1356fc --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchstatus.gold @@ -0,0 +1,3 @@ +`` +HTTP/1.1 205 Reset Content +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold new file mode 100644 index 00000000000..dfc27353d5e --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold @@ -0,0 +1,15 @@ +`` +Request=GET /eighth/magic;matrix=1/eighth HTTP/1.1 lookup_key={PATH} +`` +`` +Request=GET /eighth/magic;matrix=1/eighth?hello=10 HTTP/1.1 lookup_key={PATH} +`` +`` +Request=GET /tenth/magic/tenth;matrix=2 HTTP/1.1 lookup_key={PATH} +`` +`` +Request=GET /tenth/magic/tenth;matrix=2?query=10 HTTP/1.1 lookup_key={PATH} +`` +`` +Request=GET /eleventh/magic;matrix=4/eleventh;matrix=2?query=true HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold new file mode 100644 index 00000000000..206e19b875b --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold @@ -0,0 +1,3 @@ +`` +Request=GET /regexdoesntmatch HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold new file mode 100644 index 00000000000..5f484fdd5e0 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold @@ -0,0 +1,3 @@ +`` +Request=GET /regexmatches?cookies=oreos-chipsahoy-icecream HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold new file mode 100644 index 00000000000..b490d41fe95 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold @@ -0,0 +1,9 @@ +`` +Request=GET /photos/search?query=magic HTTP/1.1 lookup_key={PATH} +`` +`` +Request=GET /photos/search?query=/theunmatchedpath HTTP/1.1 lookup_key={PATH} +`` +`` +equest=GET /photos/search/magic/foobar HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold new file mode 100644 index 00000000000..29bb5e517fd --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold @@ -0,0 +1,3 @@ +`` +Request=GET /cookiedoesntmatch HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold new file mode 100644 index 00000000000..6d5dbb16c23 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold @@ -0,0 +1,3 @@ +`` +Request=GET /cookiedoesntmatch?a=1&b=2&c=3 HTTP/1.1 lookup_key={PATH} +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py new file mode 100644 index 00000000000..bafb0ef76ff --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py @@ -0,0 +1,104 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: match cookie" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches?a=1&b=2&c=3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch?a=1&b=2&c=3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/matchconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/matchconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matchconfig.txt' +) + +# Positive test case that remaps because cookie matches +tr = Test.AddTestRun("cookie value matches") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/magic?a=1&b=2&c=3" \ +-H"Cookie: fpbeta=magic" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie2.gold" + +# Negative test case that doesn't remap because cookie doesn't match +tr = Test.AddTestRun("cookie regex doesn't match") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/magic?a=1&b=2&c=3" \ +-H"Cookie: fpbeta=magit" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie2.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py b/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py new file mode 100644 index 00000000000..706f353b441 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py @@ -0,0 +1,103 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: match cookie" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/matchuriconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/matchuriconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matchuriconfig.txt' +) + +# Positive test case, URI matches rule +tr = Test.AddTestRun("URI value matches") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic/thisispartofthepath" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie.gold" + +# Negative test case that doesn't remap because URI doesn't match +tr = Test.AddTestRun("URI value doesn't match") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py new file mode 100644 index 00000000000..9b3c7f32eb4 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py @@ -0,0 +1,148 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: Tests when matrix parameters are present" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# We just need a server to capture ATS outgoing requests +# so that we can verify the remap rules +# That's why I am not adding any canned request/response +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/matrixconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/matrixconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/eighth http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matrixconfig.txt' +) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/ninth http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matrixconfig.txt' +) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/tenth http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matrixconfig.txt' +) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/eleventh http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matrixconfig.txt' +) + +tr = Test.AddTestRun("path is substituted") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/eighth/magic;matrix=1" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("path is substituted when matrix \ + and query is present") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/eighth/magic;matrix=1?hello=10" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Another $path substitution passing matrix \ + and query and replacing query") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/ninth/magic;matrix=5?hello=16" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("$path substitution in sendto and \ + inserting matrix parameters in remap") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/tenth/magic" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("inserting matrix params in remap and \ + passing along query string") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/tenth/magic?query=10" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Another test to verify matrix and query \ + params are passed along") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/eleventh/magic;matrix=4?query=12" \ +-H "Proxy-Connection: keep-alive" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +server.Streams.All = "gold/matrix.gold" + diff --git a/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py new file mode 100644 index 00000000000..6028ec8e004 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py @@ -0,0 +1,103 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: cookie not exists" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiedoesntexist HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookieexists HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/notexistsconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/notexistsconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/notexistsconfig.txt' +) + +# Positive test case that remaps because cookie doesn't exist +tr = Test.AddTestRun("cookie fpbeta doesn't exist") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/doesntexistcookie.gold" + +# Negative test case that doesn't remap because cookie exists +tr = Test.AddTestRun("cooke fpbeta exists") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/existscookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py new file mode 100644 index 00000000000..3e2c8175a7c --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py @@ -0,0 +1,104 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: cookie regex match and substition" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /regexmatches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /regexdoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/regexconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/regexconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/regexconfig.txt' +) + +# Positive test case that remaps because cookie regex matches +tr = Test.AddTestRun("cookie regex matches") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=ilove-oreos-chipsahoy-icecream" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/regexmatches.gold" + +# Negative test case that doesn't remap because cookie regex doesn't match +tr = Test.AddTestRun("cookie regex doesn't match") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/regexdoesntmatch.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py b/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py new file mode 100644 index 00000000000..34c0b344492 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py @@ -0,0 +1,79 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: set HTTP status of the response" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/statusconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/statusconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/statusconfig.txt' +) + +# Plugin sets the HTTP status because first rule matches +tr = Test.AddTestRun("Sets the status to 205") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=magic" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +tr.Streams.All = "gold/matchstatus.gold" + +# Plugin sets the HTTP status because the else rule matches (i.e. no match) +tr = Test.AddTestRun("Sets the else status to 400") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H "Proxy-Connection: keep-alive" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +tr.Streams.All = "gold/matchelsestatus.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py new file mode 100644 index 00000000000..07166891ab9 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py @@ -0,0 +1,100 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: test connector" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/subcookie.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/subcookie.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/subcookie.txt' +) + +# Positive test case that remaps because all connected operations pass +tr = Test.AddTestRun("cookie value matches") +# Unlike in other places I am using a single line string because the & seems to +# be interpreted by the autest framework or the shell (tried escaping with \) +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com/magic" -H"Cookie: fpbeta=a=1&b=2&c=3" -H "Proxy-Connection: keep-alive" --verbose '.format(ts.Variables.port) +#tr.Processes.Default.Command = ''' +#curl +#--proxy 127.0.0.1:{0} +#"http://www.example.com/magic" +#-H\"Cookie: fpbeta=a=1&b=2&c=3\" +#-H "Proxy-Connection: keep-alive" +#--verbose +#'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie.gold" + +# Negative test case that doesn't remap because not all subops pass +tr = Test.AddTestRun("cookie value doesn't match") +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com/magic" -H"Cookie: fpbeta=a=1&b=2&c=4" -H "Proxy-Connection: keep-alive" --verbose '.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py new file mode 100644 index 00000000000..9cfb01d8a69 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py @@ -0,0 +1,116 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: Substitute variables" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/substituteconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/substituteconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/substituteconfig.txt' +) + +tr = Test.AddTestRun("Substitute $path in the dest query") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=abcd" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Substitute $unmatched_path in the dest query") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic/theunmatchedpath" +-H"Cookie: oxalpha=3333" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Substitute $cr_req_url using $cr_urlencode") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: acgamma=dfndfdfd" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Substitute $path as is in outgoing path") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic/foobar" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +server.Streams.All = "gold/substitute.gold" + From 53ae715dd959c917c36c170bd9ad157de8c53fd2 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 23 Jan 2019 12:34:37 -0800 Subject: [PATCH 188/526] Cleans up memcached_remap plugin README and code Removes unfinished code and support for very old versions of ATS --- plugins/experimental/memcached_remap/README | 22 ++-------- .../memcached_remap/memcached_remap.cc | 40 +------------------ .../experimental/memcached_remap/sample.py | 4 +- 3 files changed, 8 insertions(+), 58 deletions(-) diff --git a/plugins/experimental/memcached_remap/README b/plugins/experimental/memcached_remap/README index 449060d73f5..248341f3086 100644 --- a/plugins/experimental/memcached_remap/README +++ b/plugins/experimental/memcached_remap/README @@ -1,7 +1,7 @@ Apache Traffic Server Memcached-based remap plugin ============================================== -This is a plugin which is based on the mysql_remap code (http://svn.apache.org/repos/asf/trafficserver/plugins/mysql_remap/) +This is a plugin which is based on the mysql_remap code (https://github.com/apache/trafficserver/tree/master/plugins/experimental/mysql_remap) that allows us to do dynamic reverse proxy (dynamic remap) based on the information present in the memcached database. @@ -23,27 +23,13 @@ server too (please change things accordingly, if any) sudo yum install memcached python-memcached -NOTE 14-OCT-2015: +2. Compile and Install the memcached_remap plugin + To build this with the ATS configure script when --enable-experimental-plugins is used, you must have libmemcached version 1.0 or greater installed. -2. Compile and Install the memcached_remap plugin - -[user@ memcached_remap]$ gmake -/usr/local/bin/tsxs -o memcached_remap.so memcached_remap.cc - compiling memcached_remap.cc -> memcached_remap.lo - linking -> memcached_remap.so - -[user@ memcached_remap]$ sudo gmake install -/usr/local/bin/tsxs -o memcached_remap.so memcached_remap.cc - compiling memcached_remap.cc -> memcached_remap.lo - linking -> memcached_remap.so -/usr/local/bin/tsxs -i -o memcached_remap.so - installing memcached_remap.so -> -/usr/local/libexec/trafficserver/memcached_remap.so - -3. Start the Traffic Server in Plugin Mode (to debug any issues) +3. Start Traffic Server with logging (to debug any issues) /usr/local/bin/traffic_server -T "memcached_remap" diff --git a/plugins/experimental/memcached_remap/memcached_remap.cc b/plugins/experimental/memcached_remap/memcached_remap.cc index 2cee48ee0f5..e6b2834ca5c 100644 --- a/plugins/experimental/memcached_remap/memcached_remap.cc +++ b/plugins/experimental/memcached_remap/memcached_remap.cc @@ -22,7 +22,6 @@ #include #include -// change this on your box #include // global settings @@ -96,7 +95,7 @@ do_memcached_remap(TSCont contp, TSHttpTxn txnp) if (m_result) free(m_result); TSDebug(PLUGIN_NAME, "\nOUTGOING REQUEST ->\n ::: to_scheme_desc: %s\n ::: to_hostname: %s\n ::: to_port: %d", oscheme, ohost, - oport); // row[0],row[1],row[2]); + oport); TSMimeHdrFieldValueStringSet(reqp, hdr_loc, field_loc, 0, ohost, -1); TSUrlHostSet(reqp, url_loc, ohost, -1); TSUrlSchemeSet(reqp, url_loc, oscheme, -1); @@ -112,7 +111,7 @@ do_memcached_remap(TSCont contp, TSHttpTxn txnp) goto not_found; } - goto free_stuff; // free the result set after processed + goto release_field; // free the result set after processed not_found: // lets build up a nice 404 message for someone @@ -120,13 +119,6 @@ do_memcached_remap(TSCont contp, TSHttpTxn txnp) TSHttpHdrStatusSet(reqp, hdr_loc, TS_HTTP_STATUS_NOT_FOUND); TSHttpTxnStatusSet(txnp, TS_HTTP_STATUS_NOT_FOUND); } -free_stuff: -#if (TS_VERSION_NUMBER < 2001005) - if (request_host) - TSHandleStringRelease(reqp, hdr_loc, request_host); - if (request_scheme) - TSHandleStringRelease(reqp, hdr_loc, request_scheme); -#endif release_field: if (field_loc) { TSHandleMLocRelease(reqp, hdr_loc, field_loc); @@ -166,8 +158,6 @@ TSPluginInit(int argc, const char *argv[]) { TSPluginRegistrationInfo info; memcached_return_t rc; - // FILE *fp; - // char servers_string[8192]; info.plugin_name = const_cast(PLUGIN_NAME); info.vendor_name = const_cast("Apache Software Foundation"); @@ -179,34 +169,8 @@ TSPluginInit(int argc, const char *argv[]) return; } - // parse the configuration file - // TODO: this is still under testing 1.0.2 version should have this feature - /* - if(argc < 1) { - TSError("memcached_remap: you should pass a configuration file as argument to plugin with list of servers.\n"); - return; - } - - fp = fopen(argv[0], "r"); - if(!fp) { - TSError("memcached_remap: Failed to open the configuration file %s\n", argv[0]); - return; - } - - while(!feof(fp)) { - fscanf(fp,"servers=%[^\n] ", servers_string); - } - - fclose(fp); - */ - - // initialize the memcache - // fp = NULL; - // snprintf(servers_string, 1,"%c",'h'); memc = memcached_create(NULL); - // servers = memcached_servers_parse(servers_string); - servers = memcached_server_list_append(NULL, "localhost", 11211, &rc); if (rc != MEMCACHED_SUCCESS) { TSError("[memcached_remap] Plugin registration failed while adding servers.\n"); diff --git a/plugins/experimental/memcached_remap/sample.py b/plugins/experimental/memcached_remap/sample.py index 07f28209ff8..01466cddbd4 100755 --- a/plugins/experimental/memcached_remap/sample.py +++ b/plugins/experimental/memcached_remap/sample.py @@ -25,8 +25,8 @@ mc = memcache.Client(['127.0.0.1:11211'], debug=0) # Add couple of keys -mc.set("http://127.0.0.1:80/", "http://127.0.0.1:8080"); -mc.set("http://localhost:80/", "http://localhost:8080"); +mc.set("http://127.0.0.1:80/", "http://127.0.0.1:8080") +mc.set("http://localhost:80/", "http://localhost:8080") # Print the keys that are saved print "response-1 is '%s'" % (mc.get("http://127.0.0.1:80/")) From 8d671d8729b3113e69424aa16800a6b875e4e12f Mon Sep 17 00:00:00 2001 From: Aaron Canary Date: Tue, 22 Jan 2019 15:44:27 -0600 Subject: [PATCH 189/526] Moved AtomicBit into it's own file. iterate code --- include/tscore/AtomicBit.h | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 include/tscore/AtomicBit.h diff --git a/include/tscore/AtomicBit.h b/include/tscore/AtomicBit.h new file mode 100644 index 00000000000..25d399f5339 --- /dev/null +++ b/include/tscore/AtomicBit.h @@ -0,0 +1,92 @@ +/** + @file AtomicBit + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + +//////////////////////////////////////////// + Implements class AtomicBit +*/ + +#pragma once +#include +#include + +////////////////////////////////////////////////////// +/// AtomicBit for inplace atomic bit operations +/* useful when you refernce a bit packed into a byte (unit_8) as a bool&, + * you want a bit to 'walk and talk' like an std::atomic or std::atomic_flag. + * In practice this is constructed at time of the operation(s), + * storing it would defeat the purpose of packing the bits. + */ +class AtomicBit +{ + std::atomic *_byte_ptr; ///< pointer to the byte + uint8_t const _mask; ///< bitmask of which bit you are using + +public: + // define a bit to perform atomic operations + AtomicBit(std::atomic &byte, const uint8_t mask) : _byte_ptr(&byte), _mask(mask) {} + AtomicBit(uint8_t *byte_ptr, const uint8_t mask) : _byte_ptr(reinterpret_cast *>(byte_ptr)), _mask(mask) {} + + // Atomically set the bit true + // return @c true if the bit was changed, @c false if not. + bool + test_and_set() + { + return compare_exchange(true); + } + + // allows assign by bool + // @return The new value of the bit. + bool + operator=(bool val) + { + compare_exchange(val); + return val; + } + + // allow cast to bool + explicit operator const bool() const { return (*_byte_ptr) & _mask; } + + // allows compare with bool + bool + operator==(bool rhs) const + { + return bool(*this) == rhs; + } + + // Atomically set the bit to `val` + // return @c true if the bit was changed, @c false if not. + bool + compare_exchange(bool val) + { + while (true) { + uint8_t byte_val = *_byte_ptr; + const uint8_t next_byte_val = val ? (byte_val | _mask) : (byte_val & ~_mask); + if (byte_val == next_byte_val) { + return false; + } + if (_byte_ptr->compare_exchange_weak(byte_val, next_byte_val)) { + return true; + } + } + } +}; From 217fbe1474668b45916bf1ab130dcbeb9723f549 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 23 Jan 2019 19:22:42 +0000 Subject: [PATCH 190/526] Allow client_cert and client_key to be specified by relative path --- iocore/net/SSLConfig.cc | 4 +--- iocore/net/SSLSNIConfig.cc | 11 ++++++----- .../experimental/ssl_session_reuse/src/ssl_init.cc | 1 + .../experimental/ssl_session_reuse/src/subscriber.cc | 1 + tests/gold_tests/tls/tls_client_cert2.test.py | 6 ++++-- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index b46ff5f90fb..71d79fb00ce 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -419,9 +419,7 @@ SSLConfigParams::initialize() ssl_client_cert_path = nullptr; REC_ReadConfigStringAlloc(ssl_client_cert_filename, "proxy.config.ssl.client.cert.filename"); REC_ReadConfigStringAlloc(ssl_client_cert_path, "proxy.config.ssl.client.cert.path"); - if (ssl_client_cert_filename && ssl_client_cert_path) { - set_paths_helper(ssl_client_cert_path, ssl_client_cert_filename, &clientCertPathOnly, &clientCertPath); - } + set_paths_helper(ssl_client_cert_path, ssl_client_cert_filename, &clientCertPathOnly, &clientCertPath); ats_free_null(ssl_client_cert_filename); ats_free_null(ssl_client_cert_path); diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 06f15cd0927..80dc8c4b5c1 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -35,6 +35,7 @@ #include "P_SSLConfig.h" #include "tscore/ink_memory.h" #include "tscpp/util/TextView.h" +#include "tscore/I_Layout.h" #include #include @@ -82,11 +83,11 @@ SNIConfigParams::loadSNIConfig() // set the next hop properties SSLConfig::scoped_config params; - auto clientCTX = params->getClientSSL_CTX(); - const char *certFile = item.client_cert.data(); - const char *keyFile = item.client_key.data(); - if (certFile && certFile[0] != '\0') { - clientCTX = params->getCTX(certFile, keyFile, params->clientCACertFilename, params->clientCACertPath); + auto clientCTX = params->getClientSSL_CTX(); + if (!item.client_cert.empty() && !item.client_key.empty()) { + std::string certFilePath = Layout::get()->relative_to(params->clientCertPathOnly, item.client_cert.data()); + std::string keyFilePath = Layout::get()->relative_to(params->clientKeyPathOnly, item.client_key.data()); + clientCTX = params->getCTX(certFilePath.c_str(), keyFilePath.c_str(), params->clientCACertFilename, params->clientCACertPath); } auto nps = next_hop_list.emplace(next_hop_list.end()); diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc index 3cd105f78e2..2333eb39129 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc @@ -23,6 +23,7 @@ */ #include +#include #include "ssl_utils.h" #include "Config.h" #include "common.h" diff --git a/plugins/experimental/ssl_session_reuse/src/subscriber.cc b/plugins/experimental/ssl_session_reuse/src/subscriber.cc index 913aecd3260..6f51709c694 100644 --- a/plugins/experimental/ssl_session_reuse/src/subscriber.cc +++ b/plugins/experimental/ssl_session_reuse/src/subscriber.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include "common.h" #include "subscriber.h" diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py index f50d33051b7..93b1b7b67dc 100644 --- a/tests/gold_tests/tls/tls_client_cert2.test.py +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -68,6 +68,8 @@ 'proxy.config.diags.debug.tags': 'ssl_verify_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', @@ -87,8 +89,8 @@ ts.Disk.ssl_server_name_yaml.AddLines([ '- fqdn: bob.bar.com', - ' client_cert: {0}/signed-bar.pem'.format(ts.Variables.SSLDir), - ' client_key: {0}/signed-bar.key'.format(ts.Variables.SSLDir), + ' client_cert: signed-bar.pem', + ' client_key: signed-bar.key', '- fqdn: bob.*.com', ' client_cert: {0}/signed-foo.pem'.format(ts.Variables.SSLDir), ' client_key: {0}/signed-foo.key'.format(ts.Variables.SSLDir), From 4c36c0ac5df2a4d7b151ce04fb11881107871b9c Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 24 Jan 2019 16:25:23 +0000 Subject: [PATCH 191/526] Remove the timeouts on tls tests and limit the number of exec threads. --- tests/gold_tests/tls/tls.test.py | 2 +- .../tls/tls_check_cert_selection.test.py | 7 +------ tests/gold_tests/tls/tls_client_cert.test.py | 19 +------------------ tests/gold_tests/tls/tls_client_cert2.test.py | 9 +-------- .../tls/tls_client_cert_override.test.py | 7 +------ .../gold_tests/tls/tls_client_verify.test.py | 15 +-------------- .../gold_tests/tls/tls_client_verify2.test.py | 14 +------------- .../tls/tls_client_versions.test.py | 7 +------ .../tls/tls_forward_nonhttp.test.py | 4 +--- tests/gold_tests/tls/tls_hooks_verify.test.py | 2 ++ tests/gold_tests/tls/tls_keepalive.test.py | 4 +--- tests/gold_tests/tls/tls_ticket.test.py | 4 ++-- tests/gold_tests/tls/tls_tunnel.test.py | 11 +---------- .../gold_tests/tls/tls_tunnel_forward.test.py | 6 +----- .../tls/tls_tunnel_plugin_rename.test.py | 4 +--- tests/gold_tests/tls/tls_verify.test.py | 6 +----- tests/gold_tests/tls/tls_verify2.test.py | 9 +-------- tests/gold_tests/tls/tls_verify3.test.py | 8 +------- tests/gold_tests/tls/tls_verify_base.test.py | 7 +------ .../tls/tls_verify_ca_override.test.py | 7 +------ .../tls/tls_verify_not_pristine.test.py | 5 +---- .../tls/tls_verify_override.test.py | 15 +-------------- .../tls/tls_verify_override_base.test.py | 14 +------------- 23 files changed, 25 insertions(+), 161 deletions(-) diff --git a/tests/gold_tests/tls/tls.test.py b/tests/gold_tests/tls/tls.test.py index 58ffdaddabb..4f02cbf1a6f 100644 --- a/tests/gold_tests/tls/tls.test.py +++ b/tests/gold_tests/tls/tls.test.py @@ -75,6 +75,7 @@ # enable ssl port 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -86,4 +87,3 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.Processes.Default.Streams.stdout = "gold/ssl-post.gold" tr.StillRunningAfter = server -tr.Processes.Default.TimeOut = 10 diff --git a/tests/gold_tests/tls/tls_check_cert_selection.test.py b/tests/gold_tests/tls/tls_check_cert_selection.test.py index 92e52c41252..92a0e183a2f 100644 --- a/tests/gold_tests/tls/tls_check_cert_selection.test.py +++ b/tests/gold_tests/tls/tls_check_cert_selection.test.py @@ -59,14 +59,13 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'http|ssl|dns|hostdb', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.url_remap.pristine_host_hdr': 1, 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.dns.resolv_conf': 'NULL' }) @@ -84,7 +83,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=bar.com", "Cert should contain bar.com") tr.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=foo.com", "Cert should not contain foo.com") @@ -95,7 +93,6 @@ tr2.Processes.Default.Command = "curl -v --cacert signer.pem --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=foo.com", "Cert should contain foo.com") @@ -107,7 +104,6 @@ tr2.Processes.Default.Command = "curl -v -k --resolve 'random.server.com:{0}:127.0.0.1' https://random.server.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=random.server.com", "Cert should contain random.server.com") @@ -121,7 +117,6 @@ tr2.Processes.Default.Command = "curl -v -k --cacert signer.pem --resolve 'bad.sni.com:{0}:127.0.0.1' https://bad.sni.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=foo.com", "Cert should contain foo.com") diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index c03e4b882a4..0fcab7be1f2 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -75,6 +75,7 @@ 'proxy.config.ssl.client.cert.filename': 'signed-foo.pem', 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.private_key.filename': 'signed-foo.key', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr' : 1, }) @@ -107,7 +108,6 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:example.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -117,7 +117,6 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") # Should succeed @@ -127,7 +126,6 @@ trbar.StillRunningAfter = server2 trbar.Processes.Default.Command = "curl -H host:bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) trbar.Processes.Default.ReturnCode = 0 -trbar.Processes.Default.TimeOut = 10 trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -137,7 +135,6 @@ trbarfail.StillRunningAfter = server2 trbarfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) trbarfail.Processes.Default.ReturnCode = 0 -trbarfail.Processes.Default.TimeOut = 10 trbarfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") tr2 = Test.AddTestRun("Update config files") @@ -154,8 +151,6 @@ # recreate the records.config with the cert filename changed tr2.Disk.File(recordspath, id = "records_config", typename="ats:config:records"), tr2.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl|http', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), @@ -174,7 +169,6 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket tr2.Processes.Default.Env = ts.Env tr2.Processes.Default.ReturnCode = 0 -tr2.Processes.Default.TimeOut = 10 tr2reload = Test.AddTestRun("Reload config") tr2reload.StillRunningAfter = ts @@ -184,7 +178,6 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket tr2reload.Processes.Default.Env = ts.Env tr2reload.Processes.Default.ReturnCode = 0 -tr2reload.Processes.Default.TimeOut = 10 #Should succeed @@ -196,7 +189,6 @@ tr3bar.StillRunningAfter = server2 tr3bar.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr3bar.Processes.Default.ReturnCode = 0 -tr3bar.Processes.Default.TimeOut = 10 tr3bar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -206,7 +198,6 @@ tr3barfail.StillRunningAfter = server2 tr3barfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr3barfail.Processes.Default.ReturnCode = 0 -tr3barfail.Processes.Default.TimeOut = 10 tr3barfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") #Should succeed @@ -217,7 +208,6 @@ tr3.StillRunningAfter = server2 tr3.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr3.Processes.Default.ReturnCode = 0 -tr3.Processes.Default.TimeOut = 10 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -227,7 +217,6 @@ tr3fail.StillRunningAfter = server2 tr3fail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr3fail.Processes.Default.ReturnCode = 0 -tr3fail.Processes.Default.TimeOut = 10 tr3fail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") @@ -244,7 +233,6 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket trupdate.Processes.Default.Env = ts.Env trupdate.Processes.Default.ReturnCode = 0 -trupdate.Processes.Default.TimeOut = 10 trreload = Test.AddTestRun("Reload config after renaming certs") trreload.StillRunningAfter = ts @@ -253,7 +241,6 @@ trreload.Processes.Default.Command = 'traffic_ctl config reload' trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 -trreload.Processes.Default.TimeOut = 10 #Should succeed tr4bar = Test.AddTestRun("Make request with renamed bar cert to second server") @@ -264,7 +251,6 @@ tr4bar.StillRunningAfter = server2 tr4bar.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr4bar.Processes.Default.ReturnCode = 0 -tr4bar.Processes.Default.TimeOut = 10 tr4bar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -274,7 +260,6 @@ tr4barfail.StillRunningAfter = server2 tr4barfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr4barfail.Processes.Default.ReturnCode = 0 -tr4barfail.Processes.Default.TimeOut = 10 tr4barfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") #Should succeed @@ -284,7 +269,6 @@ tr4.StillRunningAfter = server2 tr4.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr4.Processes.Default.ReturnCode = 0 -tr4.Processes.Default.TimeOut = 10 tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -294,6 +278,5 @@ tr4fail.StillRunningAfter = server2 tr4fail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr4fail.Processes.Default.ReturnCode = 0 -tr4fail.Processes.Default.TimeOut = 10 tr4fail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py index 93b1b7b67dc..35cd0b642e3 100644 --- a/tests/gold_tests/tls/tls_client_cert2.test.py +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -64,8 +64,6 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl_verify_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), @@ -73,6 +71,7 @@ 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr' : 1, }) @@ -110,7 +109,6 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:bob.bar.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -120,7 +118,6 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:bob.bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") # Should succeed @@ -130,7 +127,6 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:bob.foo.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -140,7 +136,6 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:bob.foo.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") # Should succeed @@ -150,7 +145,6 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:random.bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -160,6 +154,5 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:random.bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") diff --git a/tests/gold_tests/tls/tls_client_cert_override.test.py b/tests/gold_tests/tls/tls_client_cert_override.test.py index 70be6131b5a..25a82cd848a 100644 --- a/tests/gold_tests/tls/tls_client_cert_override.test.py +++ b/tests/gold_tests/tls/tls_client_cert_override.test.py @@ -64,8 +64,6 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), @@ -75,6 +73,7 @@ 'proxy.config.ssl.client.cert.filename': 'signed-foo.pem', 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.private_key.filename': 'signed-foo.key', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr' : 1, }) @@ -105,7 +104,6 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:example.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -115,7 +113,6 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/badcase1'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 10 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") # Should succeed @@ -125,7 +122,6 @@ trbar.StillRunningAfter = server2 trbar.Processes.Default.Command = "curl -H host:bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) trbar.Processes.Default.ReturnCode = 0 -trbar.Processes.Default.TimeOut = 10 trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") #Should fail @@ -135,7 +131,6 @@ trbarfail.StillRunningAfter = server2 trbarfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/badcase2'.format(ts.Variables.port) trbarfail.Processes.Default.ReturnCode = 0 -trbarfail.Processes.Default.TimeOut = 10 trbarfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py index 2406d28cfad..866c5bfab28 100644 --- a/tests/gold_tests/tls/tls_client_verify.test.py +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -44,8 +44,6 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), @@ -54,6 +52,7 @@ 'proxy.config.url_remap.pristine_host_hdr' : 1, 'proxy.config.ssl.client.certification_level': 2, 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.TLSv1_3': 0 }) @@ -84,7 +83,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") tr = Test.AddTestRun("Connect to foo.com with bad cert") @@ -95,7 +93,6 @@ tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) # Should fail with badly signed certs tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") tr = Test.AddTestRun("Connect to foo.com with cert") @@ -105,7 +102,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-foo.pem --key signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bob.bar.com without cert") @@ -113,7 +109,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") tr = Test.AddTestRun("Connect to bob.bar.com with cert") @@ -123,7 +118,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-bar.pem --key signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bob.bar.com with bad cert") @@ -133,7 +127,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bob.foo.com without cert") @@ -141,7 +134,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") tr = Test.AddTestRun("Connect to bob.foo.com with cert") @@ -151,7 +143,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-foo.pem --key signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bob.foo.com with bad cert") @@ -161,7 +152,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bar.com without cert") @@ -169,7 +159,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert handshake failure", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bar.com with cert") @@ -179,7 +168,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") tr = Test.AddTestRun("Connect to bar.com with bad cert") @@ -189,7 +177,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") diff --git a/tests/gold_tests/tls/tls_client_verify2.test.py b/tests/gold_tests/tls/tls_client_verify2.test.py index fd70d865c3d..e55376d8f2f 100644 --- a/tests/gold_tests/tls/tls_client_verify2.test.py +++ b/tests/gold_tests/tls/tls_client_verify2.test.py @@ -44,8 +44,6 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), @@ -54,6 +52,7 @@ 'proxy.config.url_remap.pristine_host_hdr' : 1, 'proxy.config.ssl.client.certification_level': 0, 'proxy.config.ssl.CA.cert.path': '', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir) }) @@ -84,7 +83,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to foo.com with cert") @@ -94,7 +92,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-foo.pem --key signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bob.bar.com without cert") @@ -102,7 +99,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bob.bar.com with cert") @@ -112,7 +108,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-bar.pem --key signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") tr = Test.AddTestRun("Connect to bob.bar.com with bad cert") @@ -122,7 +117,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("error", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bob.foo.com without cert") @@ -130,7 +124,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bob.foo.com with cert") @@ -140,7 +133,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-foo.pem --key signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") tr = Test.AddTestRun("Connect to bob.foo.com with bad cert") @@ -150,7 +142,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ContainsExpression("error", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bar.com without cert") @@ -158,7 +149,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") tr = Test.AddTestRun("Connect to bar.com with cert") @@ -168,7 +158,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bar.com with bad cert") @@ -178,6 +167,5 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert unknown ca", "TLS handshake should succeed") diff --git a/tests/gold_tests/tls/tls_client_versions.test.py b/tests/gold_tests/tls/tls_client_versions.test.py index 77bad0d77c0..54de7086628 100644 --- a/tests/gold_tests/tls/tls_client_versions.test.py +++ b/tests/gold_tests/tls/tls_client_versions.test.py @@ -52,8 +52,6 @@ ) ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -63,6 +61,7 @@ 'proxy.config.url_remap.pristine_host_hdr': 1, 'proxy.config.ssl.TLSv1': 0, 'proxy.config.ssl.TLSv1_1': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.TLSv1_2': 1 }) @@ -81,7 +80,6 @@ tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 35 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ContainsExpression("ssl_choose_client_version:unsupported protocol", "Should not allow TLSv1_2") # Target foo.com for TLSv1. Should succeed @@ -89,14 +87,12 @@ tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 # Target bar.com for TLSv1. Should fail tr = Test.AddTestRun("bar.com TLSv1") tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 35 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ContainsExpression("alert protocol version", "Should not allow TLSv1_0") # Target bar.com for TLSv1_2. Should succeed @@ -104,5 +100,4 @@ tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 diff --git a/tests/gold_tests/tls/tls_forward_nonhttp.test.py b/tests/gold_tests/tls/tls_forward_nonhttp.test.py index 8516dc1030c..90c947732dc 100644 --- a/tests/gold_tests/tls/tls_forward_nonhttp.test.py +++ b/tests/gold_tests/tls/tls_forward_nonhttp.test.py @@ -45,14 +45,13 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), 'proxy.config.http.connect_ports': '{0} 4444'.format(ts.Variables.ssl_port), 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -69,7 +68,6 @@ tr.ReturnCode = 0 tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 testout_path = os.path.join(Test.RunDirectory, "test.out") tr.Disk.File(testout_path, id = "testout") tr.Processes.Default.Streams.All += Testers.IncludesExpression("This is a reply", "s_client should get response") diff --git a/tests/gold_tests/tls/tls_hooks_verify.test.py b/tests/gold_tests/tls/tls_hooks_verify.test.py index c3d694d221d..3c49f35da49 100644 --- a/tests/gold_tests/tls/tls_hooks_verify.test.py +++ b/tests/gold_tests/tls/tls_hooks_verify.test.py @@ -38,6 +38,7 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ + # Test looks for debug output from the plugin 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'ssl_verify_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), @@ -47,6 +48,7 @@ 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.verify.server.policy': 'ENFORCED', 'proxy.config.ssl.client.verify.server.properties': 'NONE', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) diff --git a/tests/gold_tests/tls/tls_keepalive.test.py b/tests/gold_tests/tls/tls_keepalive.test.py index 92fa45edbc6..e420d54aec8 100644 --- a/tests/gold_tests/tls/tls_keepalive.test.py +++ b/tests/gold_tests/tls/tls_keepalive.test.py @@ -36,14 +36,13 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl_hook_test|log', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), 'proxy.config.ssl.TLSv1_3': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.log.max_secs_per_buffer': 1 }) @@ -102,4 +101,3 @@ ts.Disk.squid_log.Content = "gold/accesslog.gold" -tr.Processes.Default.TimeOut = 10 diff --git a/tests/gold_tests/tls/tls_ticket.test.py b/tests/gold_tests/tls/tls_ticket.test.py index 355c5fed1ed..29c9b34406a 100644 --- a/tests/gold_tests/tls/tls_ticket.test.py +++ b/tests/gold_tests/tls/tls_ticket.test.py @@ -66,6 +66,7 @@ 'proxy.config.http.server_ports': '{0}:proto=http2;http:ssl'.format(ts.Variables.ssl_port), 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.server.session_ticket.enable': '1', 'proxy.config.ssl.server.ticket_key.filename': '../../file.ticket' }) @@ -76,6 +77,7 @@ 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.server.session_ticket.enable': '1', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.server.ticket_key.filename': '../../file.ticket' }) @@ -89,7 +91,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) path1 = tr.Processes.Default.Streams.stdout.AbsPath tr.StillRunningAfter = server -tr.Processes.Default.TimeOut = 10 # Pull out session created in tr to test for session id in tr2 def checkSession(ev) : @@ -123,5 +124,4 @@ def checkSession(ev) : tr2.Processes.Default.StartBefore(Test.Processes.ts2, ready=When.PortOpen(ts2.Variables.ssl_port)) tr2.ReturnCode = 0 path2 = tr2.Processes.Default.Streams.stdout.AbsPath -tr2.Processes.Default.TimeOut = 10 tr2.Processes.Default.Streams.All.Content = Testers.Lambda(checkSession) diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index 1db455ea05b..a33c7c43349 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -60,8 +60,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -70,6 +68,7 @@ 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -92,7 +91,6 @@ tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") @@ -104,7 +102,6 @@ tr.Processes.Default.Command = "curl -v --resolve 'bob.bar.com:{0}:127.0.0.1' -k https://bob.bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") @@ -116,7 +113,6 @@ tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") @@ -125,7 +121,6 @@ tr.Processes.Default.Command = "curl -v -k https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") @@ -148,7 +143,6 @@ tr.Processes.Default.Env = ts.Env tr.Processes.Default.Command = 'echo Updated configs' tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 10 trreload = Test.AddTestRun("Reload config") trreload.StillRunningAfter = ts @@ -158,7 +152,6 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 -trreload.Processes.Default.TimeOut = 10 # Should termimate on traffic_server (not tunnel) tr = Test.AddTestRun("foo.com no Tunnel-test") @@ -166,7 +159,6 @@ # Wait for the reload to complete tr.DelayStart = 5 tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -176,7 +168,6 @@ tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Terminate on Traffic Server") diff --git a/tests/gold_tests/tls/tls_tunnel_forward.test.py b/tests/gold_tests/tls/tls_tunnel_forward.test.py index a9e2cec6f6e..93ca7329ea4 100644 --- a/tests/gold_tests/tls/tls_tunnel_forward.test.py +++ b/tests/gold_tests/tls/tls_tunnel_forward.test.py @@ -64,8 +64,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -74,6 +72,7 @@ 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -96,7 +95,6 @@ tr.Processes.Default.StartBefore(server_random) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") @@ -107,7 +105,6 @@ tr2.Processes.Default.Command = "curl -v --http1.1 -H 'host:bar.com' --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server_bar -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") @@ -119,7 +116,6 @@ tr3.Processes.Default.Command = "curl --http1.1 -v -k -H 'host:random.com' https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr3.ReturnCode = 0 tr3.StillRunningAfter = server_random -tr3.Processes.Default.TimeOut = 10 tr3.StillRunningAfter = ts tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") diff --git a/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py index 6341aeca0c6..db03cc8d90b 100644 --- a/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py +++ b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py @@ -62,8 +62,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -72,6 +70,7 @@ 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -93,7 +92,6 @@ tr.Processes.Default.StartBefore(server_random) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server_random -tr.Processes.Default.TimeOut = 10 tr.StillRunningAfter = ts tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") diff --git a/tests/gold_tests/tls/tls_verify.test.py b/tests/gold_tests/tls/tls_verify.test.py index cc88c99163f..d5b63e641b8 100644 --- a/tests/gold_tests/tls/tls_verify.test.py +++ b/tests/gold_tests/tls/tls_verify.test.py @@ -71,8 +71,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -83,6 +81,7 @@ 'proxy.config.ssl.client.verify.server.properties': 'SIGNATURE', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -109,7 +108,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2 = Test.AddTestRun("Override-enforcing-Test") @@ -117,7 +115,6 @@ tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.StillRunningAfter = ts -tr2.Processes.Default.TimeOut = 10 tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr3 = Test.AddTestRun("Override-enforcing-Test-fail-name-check") @@ -126,7 +123,6 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 10 # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") diff --git a/tests/gold_tests/tls/tls_verify2.test.py b/tests/gold_tests/tls/tls_verify2.test.py index 8a4742da659..ca818ac559b 100644 --- a/tests/gold_tests/tls/tls_verify2.test.py +++ b/tests/gold_tests/tls/tls_verify2.test.py @@ -71,8 +71,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -83,6 +81,7 @@ 'proxy.config.ssl.client.verify.server.properties': 'ALL', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -116,14 +115,12 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr2 = Test.AddTestRun("override-disabled") tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -132,7 +129,6 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 10 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr4 = Test.AddTestRun("override-permissive-bad-name") @@ -140,7 +136,6 @@ tr4.ReturnCode = 0 tr4.StillRunningAfter = server tr4.StillRunningAfter = ts -tr4.Processes.Default.TimeOut = 10 tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr5 = Test.AddTestRun("default-enforce-bad-sig") @@ -149,7 +144,6 @@ tr5.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr5.StillRunningAfter = server tr5.StillRunningAfter = ts -tr5.Processes.Default.TimeOut = 10 tr6 = Test.AddTestRun("default-enforce-fail") tr6.Processes.Default.Command = "curl -k -H \"host: bad_foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) @@ -157,7 +151,6 @@ tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr6.StillRunningAfter = server tr6.StillRunningAfter = ts -tr6.Processes.Default.TimeOut = 10 # No name checking for the sig-only permissive override for bad_bar diff --git a/tests/gold_tests/tls/tls_verify3.test.py b/tests/gold_tests/tls/tls_verify3.test.py index 81a7798e2bb..9668a489910 100644 --- a/tests/gold_tests/tls/tls_verify3.test.py +++ b/tests/gold_tests/tls/tls_verify3.test.py @@ -71,8 +71,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl|http|url', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -83,6 +81,7 @@ 'proxy.config.ssl.client.verify.server.properties': 'ALL', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -110,7 +109,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr = Test.AddTestRun("my.foo.com Permissive-Test log failure") @@ -118,7 +116,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -126,7 +123,6 @@ tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -136,7 +132,6 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 10 tr3 = Test.AddTestRun("random.bar.com override-no-test") tr3.Processes.Default.Command = "curl -v -k --resolve 'random.bar.com:{0}:127.0.0.1' https://random.bar.com:{0}".format(ts.Variables.ssl_port) @@ -144,7 +139,6 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 10 # Over riding the built in ERROR check since we expect tr3 to fail diff --git a/tests/gold_tests/tls/tls_verify_base.test.py b/tests/gold_tests/tls/tls_verify_base.test.py index baa373fba0c..8279b22a7d3 100644 --- a/tests/gold_tests/tls/tls_verify_base.test.py +++ b/tests/gold_tests/tls/tls_verify_base.test.py @@ -71,8 +71,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -83,6 +81,7 @@ 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.client.sni_policy': 'host' }) @@ -108,7 +107,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr = Test.AddTestRun("Permissive-Test with logged failure") @@ -116,7 +114,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -124,7 +121,6 @@ tr2.Processes.Default.Command = "curl -v -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -134,7 +130,6 @@ tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 10 # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: SNI \(bad_bar.com\) not in certificate. Action=Terminate", "Make sure bad_bar name checked failed.") diff --git a/tests/gold_tests/tls/tls_verify_ca_override.test.py b/tests/gold_tests/tls/tls_verify_ca_override.test.py index 749ebbf43bb..bb622d5f697 100644 --- a/tests/gold_tests/tls/tls_verify_ca_override.test.py +++ b/tests/gold_tests/tls/tls_verify_ca_override.test.py @@ -73,8 +73,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -85,6 +83,7 @@ 'proxy.config.ssl.client.verify.server.properties': 'SIGNATURE', 'proxy.config.ssl.client.CA.cert.path': '/tmp', 'proxy.config.ssl.client.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -100,7 +99,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) tr.StillRunningAfter = server1 tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 # Should succed. No message tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -108,7 +106,6 @@ tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/badcase1".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server1 -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about name mismatch tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -117,7 +114,6 @@ tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/case2".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server2 -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about signature tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -129,6 +125,5 @@ tr3.StillRunningAfter = ts # Should succeed. No error messages tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.TimeOut = 10 diff --git a/tests/gold_tests/tls/tls_verify_not_pristine.test.py b/tests/gold_tests/tls/tls_verify_not_pristine.test.py index 529ebfbc8da..8153008f760 100644 --- a/tests/gold_tests/tls/tls_verify_not_pristine.test.py +++ b/tests/gold_tests/tls/tls_verify_not_pristine.test.py @@ -61,8 +61,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'http|ssl|dns|hostdb', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -75,6 +73,7 @@ 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', 'proxy.config.url_remap.pristine_host_hdr': 0, 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.dns.resolv_conf': 'NULL' }) @@ -95,7 +94,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") # foo.com in. bar.com out. Should not verify @@ -103,7 +101,6 @@ tr2.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should not have succeeded") diff --git a/tests/gold_tests/tls/tls_verify_override.test.py b/tests/gold_tests/tls/tls_verify_override.test.py index 626e8587bff..010e0d327d2 100644 --- a/tests/gold_tests/tls/tls_verify_override.test.py +++ b/tests/gold_tests/tls/tls_verify_override.test.py @@ -91,8 +91,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -106,6 +104,7 @@ 'proxy.config.url_remap.pristine_host_hdr': 1, 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.client.sni_policy': 'remap' }) @@ -128,7 +127,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 # Should succed. No message tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -136,7 +134,6 @@ tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about name mismatch tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -145,7 +142,6 @@ tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about signature tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -157,7 +153,6 @@ tr3.StillRunningAfter = ts # Should succeed. No error messages tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.TimeOut = 10 tr3 = Test.AddTestRun("override-foo") tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/override".format(ts.Variables.port) @@ -166,7 +161,6 @@ tr3.StillRunningAfter = ts # Should succeed. No error messages tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.TimeOut = 10 tr4 = Test.AddTestRun("override-bar-disabled") tr4.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" http://127.0.0.1:{0}/overridedisabled".format(ts.Variables.port) @@ -175,7 +169,6 @@ tr4.StillRunningAfter = ts # Succeed. No error messages tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr4.Processes.Default.TimeOut = 10 tr5 = Test.AddTestRun("override-bar-signature-enforced") tr5.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overridesignature".format(ts.Variables.port) @@ -183,7 +176,6 @@ tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr5.StillRunningAfter = server tr5.StillRunningAfter = ts -tr5.Processes.Default.TimeOut = 10 tr6 = Test.AddTestRun("override-bar-enforced") tr6.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overrideenforced".format(ts.Variables.port) @@ -192,7 +184,6 @@ tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr6.StillRunningAfter = server tr6.StillRunningAfter = ts -tr6.Processes.Default.TimeOut = 10 # Should succeed tr = Test.AddTestRun("foo-to-bar-sni-policy-remap") @@ -200,7 +191,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") # Should fail @@ -209,7 +199,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") # Should fail @@ -218,7 +207,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") # Should succeed @@ -227,7 +215,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") # Over riding the built in ERROR check since we expect some cases to fail diff --git a/tests/gold_tests/tls/tls_verify_override_base.test.py b/tests/gold_tests/tls/tls_verify_override_base.test.py index 8f35718154c..244e4d3cbd9 100644 --- a/tests/gold_tests/tls/tls_verify_override_base.test.py +++ b/tests/gold_tests/tls/tls_verify_override_base.test.py @@ -89,8 +89,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -101,6 +99,7 @@ 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), 'proxy.config.dns.resolv_conf': 'NULL' }) @@ -123,7 +122,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 # Should succed. No message tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -131,7 +129,6 @@ tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about name mismatch tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -140,7 +137,6 @@ tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 10 tr2.StillRunningAfter = ts # Should succeed, but will be message in log about signature tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -152,7 +148,6 @@ tr3.StillRunningAfter = ts # Should succeed. No error messages tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.TimeOut = 10 tr4 = Test.AddTestRun("override-bar-disabled") tr4.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" http://127.0.0.1:{0}/overridedisabled".format(ts.Variables.port) @@ -161,7 +156,6 @@ tr4.StillRunningAfter = ts # Succeed. No error messages tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr4.Processes.Default.TimeOut = 10 tr5 = Test.AddTestRun("override-bar-signature-enforced") tr5.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overridesignature".format(ts.Variables.port) @@ -169,7 +163,6 @@ tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr5.StillRunningAfter = server tr5.StillRunningAfter = ts -tr5.Processes.Default.TimeOut = 10 tr6 = Test.AddTestRun("override-bar-enforced") tr6.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overrideenforced".format(ts.Variables.port) @@ -178,7 +171,6 @@ tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr6.StillRunningAfter = server tr6.StillRunningAfter = ts -tr6.Processes.Default.TimeOut = 10 # Should succeed tr = Test.AddTestRun("foo-to-bar-sni-policy-remap") @@ -186,7 +178,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") # Should fail @@ -195,7 +186,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") # Should fail @@ -204,7 +194,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") # Should succeed @@ -213,7 +202,6 @@ tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") From 90e51a2c47209a5342fbefb63128e9d6d9cfbc05 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Thu, 24 Jan 2019 18:33:11 +0000 Subject: [PATCH 192/526] Updating uri_signing docs to reflect new RFC changes --- plugins/experimental/uri_signing/README.md | 34 +++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/plugins/experimental/uri_signing/README.md b/plugins/experimental/uri_signing/README.md index f4fef0c85bd..5dc8789465b 100644 --- a/plugins/experimental/uri_signing/README.md +++ b/plugins/experimental/uri_signing/README.md @@ -2,7 +2,7 @@ URI Signing Plugin ================== This remap plugin implements the draft URI Signing protocol documented here: -https://tools.ietf.org/html/draft-ietf-cdni-uri-signing-12 . +https://tools.ietf.org/html/draft-ietf-cdni-uri-signing-16 . It takes a single argument: the name of a config file that contains key information. @@ -17,6 +17,8 @@ this plugin gets the URI. Config ------ +### Keys + The config file should be a JSON object that maps issuer names to JWK-sets. Exactly one of these JWK-sets must have an additional member indicating the renewal key. @@ -75,6 +77,26 @@ It's worth noting that multiple issuers can provide `auth_directives`. Each issuer will be processed in order and any issuer can provide access to a path. +### Token Stripping + +When The boolean strip_token parameter is set to true, the plugin removes the +token from both the url that is sent upstream to the origin and the url that +is used as the cache key. It can be set like this: + + { + "Kabletown URI Authority": { + "renewal_kid": "Second Key", + "strip_token" : true, + "auth_directives": [ + ⋮ + ] + "keys": [ + ⋮ + ] + } + +The strip_token parameter defaults to false and should be set by only one issuer. + Usage ----- @@ -94,12 +116,14 @@ Path parameters will not be searched for JWTs. The following claims are understood: - `iss`: Must be present. The issuer is used to locate the key for verification. - - `sub`: Validated last, after key verification. **Only `uri-regex` is supported!** + - `sub`: May be present, but is not validated. - `exp`: Expired tokens are not valid. - `iat`: May be present, but is not validated. - `cdniv`: Must be missing or 1. - - `cdnistt`: If present, must be 1. + - `cdniuc`: Validated last, after key verificationD. **Only `regex` is supported!** - `cdniets`: If cdnistt is 1, this must be present and non-zero. + - `cdnistt`: If present, must be 1. + - `cdnistd`: If present, must be 0. ### Unsupported Claims @@ -108,8 +132,10 @@ These claims are not supported. If they are present, the token will not validate - `aud` - `nbf` - `jti` + - `cdnicrit` + - `cdniip` -In addition, the `sub` containers of `uri`, `uri-pattern`, and `uri-hash` are +In addition, the `cdniuc` container of `hash` is **not supported**. ### Token Renewal From aa74886bb16b788d60143803fa5227334fb87067 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 23 Jan 2019 19:38:24 -0700 Subject: [PATCH 193/526] Removes the echo from installing pre-commit --- Makefile.am | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 133c31debe1..e04aa9aa65a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -96,7 +96,6 @@ install-examples: examples $(abs_top_srcdir)/.git/hooks/pre-commit: $(abs_top_srcdir)/tools/git/pre-commit @if [ -d $(abs_top_srcdir)/.git/hooks ]; then \ - echo; echo "\tNote: Installing ATS .git/hooks/pre-commit"; echo; \ cp $(abs_top_srcdir)/tools/git/pre-commit $(abs_top_srcdir)/.git/hooks/pre-commit; \ fi From c777312b7993586378dd76b7c55408c719b2330e Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 7 Jan 2019 20:33:30 -0700 Subject: [PATCH 194/526] Changes the return codes to be RecErrT --- lib/records/I_RecCore.h | 45 ++++++----- lib/records/RecCore.cc | 101 +++++++++++++------------ src/traffic_manager/traffic_manager.cc | 2 +- 3 files changed, 77 insertions(+), 71 deletions(-) diff --git a/lib/records/I_RecCore.h b/lib/records/I_RecCore.h index e3db5ea1920..b9157679367 100644 --- a/lib/records/I_RecCore.h +++ b/lib/records/I_RecCore.h @@ -146,37 +146,36 @@ RecErrT RecSetRecordString(const char *name, const RecString rec_string, RecSour bool inc_version = true); RecErrT RecSetRecordCounter(const char *name, RecCounter rec_counter, RecSourceT source, bool lock = true, bool inc_version = true); -int RecGetRecordInt(const char *name, RecInt *rec_int, bool lock = true); -int RecGetRecordFloat(const char *name, RecFloat *rec_float, bool lock = true); -int RecGetRecordString(const char *name, char *buf, int buf_len, bool lock = true); -int RecGetRecordString_Xmalloc(const char *name, RecString *rec_string, bool lock = true); -int RecGetRecordCounter(const char *name, RecCounter *rec_counter, bool lock = true); +RecErrT RecGetRecordInt(const char *name, RecInt *rec_int, bool lock = true); +RecErrT RecGetRecordFloat(const char *name, RecFloat *rec_float, bool lock = true); +RecErrT RecGetRecordString(const char *name, char *buf, int buf_len, bool lock = true); +RecErrT RecGetRecordString_Xmalloc(const char *name, RecString *rec_string, bool lock = true); +RecErrT RecGetRecordCounter(const char *name, RecCounter *rec_counter, bool lock = true); // Convenience to allow us to treat the RecInt as a single byte internally -int RecGetRecordByte(const char *name, RecByte *rec_byte, bool lock = true); +RecErrT RecGetRecordByte(const char *name, RecByte *rec_byte, bool lock = true); // Convenience to allow us to treat the RecInt as a bool internally -int RecGetRecordBool(const char *name, RecBool *rec_byte, bool lock = true); +RecErrT RecGetRecordBool(const char *name, RecBool *rec_byte, bool lock = true); //------------------------------------------------------------------------ // Record Attributes Reading //------------------------------------------------------------------------ typedef void (*RecLookupCallback)(const RecRecord *, void *); -int RecLookupRecord(const char *name, RecLookupCallback callback, void *data, bool lock = true); -int RecLookupMatchingRecords(unsigned rec_type, const char *match, RecLookupCallback callback, void *data, bool lock = true); - -int RecGetRecordType(const char *name, RecT *rec_type, bool lock = true); -int RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock = true); -int RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lock = true); -int RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock = true); - -int RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock = true); -int RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock = true); -int RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock = true); -int RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock = true); -int RecGetRecordSource(const char *name, RecSourceT *source, bool lock = true); - -int RecGetRecordAccessType(const char *name, RecAccessT *secure, bool lock = true); -int RecSetRecordAccessType(const char *name, RecAccessT secure, bool lock = true); +RecErrT RecLookupRecord(const char *name, RecLookupCallback callback, void *data, bool lock = true); +RecErrT RecLookupMatchingRecords(unsigned rec_type, const char *match, RecLookupCallback callback, void *data, bool lock = true); + +RecErrT RecGetRecordType(const char *name, RecT *rec_type, bool lock = true); +RecErrT RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock = true); +RecErrT RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lock = true); +RecErrT RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock = true); +RecErrT RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock = true); +RecErrT RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock = true); +RecErrT RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock = true); +RecErrT RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock = true); +RecErrT RecGetRecordSource(const char *name, RecSourceT *source, bool lock = true); + +RecErrT RecGetRecordAccessType(const char *name, RecAccessT *secure, bool lock = true); +RecErrT RecSetRecordAccessType(const char *name, RecAccessT secure, bool lock = true); //------------------------------------------------------------------------ // Signal and Alarms diff --git a/lib/records/RecCore.cc b/lib/records/RecCore.cc index 6f83226e533..388a69be580 100644 --- a/lib/records/RecCore.cc +++ b/lib/records/RecCore.cc @@ -359,32 +359,35 @@ RecRegisterConfigUpdateCb(const char *name, RecConfigUpdateCb update_cb, void *c //------------------------------------------------------------------------- // RecGetRecordXXX //------------------------------------------------------------------------- -int +RecErrT RecGetRecordInt(const char *name, RecInt *rec_int, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_INT, &data, lock)) == REC_ERR_OKAY) { *rec_int = data.rec_int; } return err; } -int +RecErrT RecGetRecordFloat(const char *name, RecFloat *rec_float, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_FLOAT, &data, lock)) == REC_ERR_OKAY) { *rec_float = data.rec_float; } return err; } -int +RecErrT RecGetRecordString(const char *name, char *buf, int buf_len, bool lock) { - int err = REC_ERR_OKAY; + RecErrT err = REC_ERR_OKAY; + if (lock) { ink_rwlock_rdlock(&g_records_rwlock); } @@ -411,44 +414,48 @@ RecGetRecordString(const char *name, char *buf, int buf_len, bool lock) return err; } -int +RecErrT RecGetRecordString_Xmalloc(const char *name, RecString *rec_string, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_STRING, &data, lock)) == REC_ERR_OKAY) { *rec_string = data.rec_string; } return err; } -int +RecErrT RecGetRecordCounter(const char *name, RecCounter *rec_counter, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_COUNTER, &data, lock)) == REC_ERR_OKAY) { *rec_counter = data.rec_counter; } return err; } -int +RecErrT RecGetRecordByte(const char *name, RecByte *rec_byte, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_INT, &data, lock)) == REC_ERR_OKAY) { *rec_byte = data.rec_int; } return err; } -int +RecErrT RecGetRecordBool(const char *name, RecBool *rec_bool, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_INT, &data, lock)) == REC_ERR_OKAY) { *rec_bool = 0 != data.rec_int; } @@ -459,10 +466,10 @@ RecGetRecordBool(const char *name, RecBool *rec_bool, bool lock) // RecGetRec Attributes //------------------------------------------------------------------------- -int +RecErrT RecLookupRecord(const char *name, void (*callback)(const RecRecord *, void *), void *data, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -484,7 +491,7 @@ RecLookupRecord(const char *name, void (*callback)(const RecRecord *, void *), v return err; } -int +RecErrT RecLookupMatchingRecords(unsigned rec_type, const char *match, void (*callback)(const RecRecord *, void *), void *data, bool lock) { int num_records; @@ -514,10 +521,10 @@ RecLookupMatchingRecords(unsigned rec_type, const char *match, void (*callback)( return REC_ERR_OKAY; } -int +RecErrT RecGetRecordType(const char *name, RecT *rec_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -539,10 +546,10 @@ RecGetRecordType(const char *name, RecT *rec_type, bool lock) return err; } -int +RecErrT RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -568,10 +575,10 @@ RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock) return err; } -int +RecErrT RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -597,10 +604,10 @@ RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lo return err; } -int +RecErrT RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -629,10 +636,10 @@ RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock) return err; } -int +RecErrT RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -658,10 +665,10 @@ RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock) return err; } -int +RecErrT RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -687,10 +694,10 @@ RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock) return err; } -int +RecErrT RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -716,10 +723,10 @@ RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock) return err; } -int +RecErrT RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock) { - int err; + RecErrT err; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -767,10 +774,10 @@ RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock) return err; } -int +RecErrT RecGetRecordAccessType(const char *name, RecAccessT *access, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -792,10 +799,10 @@ RecGetRecordAccessType(const char *name, RecAccessT *access, bool lock) return err; } -int +RecErrT RecSetRecordAccessType(const char *name, RecAccessT access, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -817,10 +824,10 @@ RecSetRecordAccessType(const char *name, RecAccessT access, bool lock) return err; } -int +RecErrT RecGetRecordSource(const char *name, RecSourceT *source, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -1086,8 +1093,8 @@ REC_readInteger(const char *name, bool *found, bool lock) { ink_assert(name); RecInt _tmp = 0; - bool _found; - _found = (RecGetRecordInt(name, &_tmp, lock) == REC_ERR_OKAY); + bool _found = (RecGetRecordInt(name, &_tmp, lock) == REC_ERR_OKAY); + if (found) { *found = _found; } @@ -1099,8 +1106,8 @@ REC_readFloat(char *name, bool *found, bool lock) { ink_assert(name); RecFloat _tmp = 0.0; - bool _found; - _found = (RecGetRecordFloat(name, &_tmp, lock) == REC_ERR_OKAY); + bool _found = (RecGetRecordFloat(name, &_tmp, lock) == REC_ERR_OKAY); + if (found) { *found = _found; } @@ -1112,8 +1119,8 @@ REC_readCounter(char *name, bool *found, bool lock) { ink_assert(name); RecCounter _tmp = 0; - bool _found; - _found = (RecGetRecordCounter(name, &_tmp, lock) == REC_ERR_OKAY); + bool _found = (RecGetRecordCounter(name, &_tmp, lock) == REC_ERR_OKAY); + if (found) { *found = _found; } @@ -1125,8 +1132,8 @@ REC_readString(const char *name, bool *found, bool lock) { ink_assert(name); RecString _tmp = nullptr; - bool _found; - _found = (RecGetRecordString_Xmalloc(name, &_tmp, lock) == REC_ERR_OKAY); + bool _found = (RecGetRecordString_Xmalloc(name, &_tmp, lock) == REC_ERR_OKAY); + if (found) { *found = _found; } diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc index f7aaad20af1..b4b7aa96777 100644 --- a/src/traffic_manager/traffic_manager.cc +++ b/src/traffic_manager/traffic_manager.cc @@ -548,7 +548,7 @@ main(int argc, const char **argv) init_dirs(); // setup critical directories, needs LibRecords - if (RecGetRecordString("proxy.config.admin.user_id", userToRunAs, sizeof(userToRunAs)) != TS_ERR_OKAY || + if (RecGetRecordString("proxy.config.admin.user_id", userToRunAs, sizeof(userToRunAs)) != REC_ERR_OKAY || strlen(userToRunAs) == 0) { mgmt_fatal(0, "proxy.config.admin.user_id is not set\n"); } From bf01bd89a24a5713e2b8d67121e319932b6973a5 Mon Sep 17 00:00:00 2001 From: Jesse Zhang Date: Wed, 23 Jan 2019 13:00:40 -0600 Subject: [PATCH 195/526] Autest test extension modification using opensourced testing tools Removes MicroServer, MicroDNS, Traffic-Replay, and SessionValidation out of tests/tools into their independent BitBucket repo together with Autest. All the relevant test extensions and the affected tests have been modified to work with the new setup. tests/bootstrap.py has also been updated to automatically install all the dependencies for the tests to run properly. Updated tests/README.md and cleaned up --- tests/README.md | 29 +- tests/bootstrap.py | 1 + .../gold_tests/autest-site/microDNS.test.ext | 13 +- .../autest-site/microserver.test.ext | 180 ++--- .../autest-site/traffic_replay.test.ext | 91 +++ .../chunked_encoding/chunked_encoding.test.py | 2 +- tests/gold_tests/h2/gold/post_chunked.gold | 2 +- tests/gold_tests/h2/http2.test.py | 15 +- .../gold_tests/headers/cachedIMSRange.test.py | 10 +- tests/gold_tests/remap/remap_https.test.py | 2 +- .../tls/tls_check_cert_selection.test.py | 2 +- tests/gold_tests/tls/tls_client_cert.test.py | 9 +- tests/gold_tests/tls/tls_client_cert2.test.py | 8 +- .../tls/tls_client_cert_override.test.py | 12 +- tests/gold_tests/tls/tls_hooks_verify.test.py | 6 +- tests/gold_tests/tls/tls_tunnel.test.py | 10 +- .../gold_tests/tls/tls_tunnel_forward.test.py | 4 +- .../tls/tls_tunnel_plugin_rename.test.py | 6 +- tests/gold_tests/tls/tls_verify.test.py | 10 +- tests/gold_tests/tls/tls_verify2.test.py | 10 +- tests/gold_tests/tls/tls_verify3.test.py | 12 +- .../tls/tls_verify_ca_override.test.py | 8 +- .../tls/tls_verify_not_pristine.test.py | 4 +- .../tls/tls_verify_override.test.py | 28 +- .../tls/tls_verify_override_base.test.py | 26 +- .../gold_tests/tls_hooks/tls_hooks13.test.py | 2 +- .../gold_tests/tls_hooks/tls_hooks14.test.py | 2 +- .../gold_tests/tls_hooks/tls_hooks15.test.py | 2 +- tests/tools/lib/IPConstants.py | 48 -- tests/tools/lib/result.py | 117 --- tests/tools/microDNS/uDNS.py | 207 ----- tests/tools/microServer/README.md | 49 -- tests/tools/microServer/uWServer.py | 734 ------------------ .../ssl/server.crt | 0 .../ssl/server.pem | 0 tests/tools/sessionvalidation/__init__.py | 17 - tests/tools/sessionvalidation/badsession.py | 35 - tests/tools/sessionvalidation/request.py | 48 -- tests/tools/sessionvalidation/response.py | 49 -- tests/tools/sessionvalidation/session.py | 45 -- .../sessionvalidation/sessionvalidation.py | 259 ------ tests/tools/sessionvalidation/transaction.py | 40 - tests/tools/traffic-replay/Config.py | 34 - tests/tools/traffic-replay/NonSSL.py | 192 ----- tests/tools/traffic-replay/RandomReplay.py | 91 --- tests/tools/traffic-replay/SSLReplay.py | 233 ------ tests/tools/traffic-replay/Scheduler.py | 88 --- tests/tools/traffic-replay/WorkerTask.py | 49 -- tests/tools/traffic-replay/__main__.py | 44 -- tests/tools/traffic-replay/extractHeader.py | 91 --- tests/tools/traffic-replay/h2Replay.py | 331 -------- tests/tools/traffic-replay/mainProcess.py | 76 -- tests/unit_tests/Makefile.am | 1 + 53 files changed, 300 insertions(+), 3084 deletions(-) create mode 100644 tests/gold_tests/autest-site/traffic_replay.test.ext delete mode 100644 tests/tools/lib/IPConstants.py delete mode 100644 tests/tools/lib/result.py delete mode 100644 tests/tools/microDNS/uDNS.py delete mode 100644 tests/tools/microServer/README.md delete mode 100644 tests/tools/microServer/uWServer.py rename tests/tools/{microServer => microserver}/ssl/server.crt (100%) rename tests/tools/{microServer => microserver}/ssl/server.pem (100%) delete mode 100644 tests/tools/sessionvalidation/__init__.py delete mode 100644 tests/tools/sessionvalidation/badsession.py delete mode 100644 tests/tools/sessionvalidation/request.py delete mode 100644 tests/tools/sessionvalidation/response.py delete mode 100644 tests/tools/sessionvalidation/session.py delete mode 100644 tests/tools/sessionvalidation/sessionvalidation.py delete mode 100644 tests/tools/sessionvalidation/transaction.py delete mode 100644 tests/tools/traffic-replay/Config.py delete mode 100644 tests/tools/traffic-replay/NonSSL.py delete mode 100644 tests/tools/traffic-replay/RandomReplay.py delete mode 100644 tests/tools/traffic-replay/SSLReplay.py delete mode 100644 tests/tools/traffic-replay/Scheduler.py delete mode 100644 tests/tools/traffic-replay/WorkerTask.py delete mode 100644 tests/tools/traffic-replay/__main__.py delete mode 100644 tests/tools/traffic-replay/extractHeader.py delete mode 100644 tests/tools/traffic-replay/h2Replay.py delete mode 100644 tests/tools/traffic-replay/mainProcess.py diff --git a/tests/README.md b/tests/README.md index 619c0c04249..8d0702973d6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,35 +6,34 @@ This directory contains different tests for Apache Trafficserver. It is recommen ## Layout The current layout is: -**gold_tests/** - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest) -**tools/** - contain programs used to help with testing. - -In the future a directory called **"unit/"** will be added for adding unit tests based on some standardized testing system. - +**gold_tests/** - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest) +**tools/** - contains programs used to help with testing. +**include/** - contains headers used for unit testing. ## Scripts -To help with easy running of the tests, there is a autest.sh and bootstrap.py file. +To help with easy running of the tests, there is autest.sh and bootstrap.py. ### autest.sh -This file is a simple wrapper that will call the AuTest program in a python virtualenv. If the virtualenv is not setup it will try to install system. That will set up the Reusable Gold Testing System on most systems in a Python virtual environment. The wrapper add some basic options to the command to point to the location of the tests. Add --help for more details on options for running autest test system. +This file is a simple wrapper that will call the Reusable Gold Testing System (Autest) program in a python virtualenv. If the virtualenv is not setup, the script will try to install it on the system. That will set up the Autest on most systems in a Python virtual environment. The wrapper add some basic options to the command to point to the location of the tests. Use --help for more details on options for running Autest. ### bootstrap.py -This script should try to install python35 or better on the system, and needed python packages for running the tests. +This script will try to install python35 or better on the system, and the needed python packages for running the tests. -# Advance setup +# Advanced setup -AuTest can be install manually instead of using the wrapper script. The advange of this is that it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done. +AuTest and the relevant tools can be install manually instead of using the wrapper script. The advange of this is that it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done. 1. run the bootstrap script then source the path with a "source ./env-test/bin/activate" command. At this point autest command should run without the wrapper script 2. The other way is to make sure you install python 3.5 or better on your system. From there install these python packages ( ie pip install ): - hyper - - git+https://bitbucket.org/dragon512/reusable-gold-testing-system.git + - git+https://bitbucket.org/autestsuite/reusable-gold-testing-system.git + - [traffic-replay](https://bitbucket.org/autestsuite/trafficreplay/src/master/) (This will automatically install [MicroDNS](https://bitbucket.org/autestsuite/microdns/src/master/), [MicroServer](https://bitbucket.org/autestsuite/microserver/src/master/), [TrafficReplayLibrary](https://bitbucket.org/autestsuite/trafficreplaylibrary/src/master/), and dnslib as part of the dependencies.) # Writting tests for AuTest When writting for the AuTest system please refer to the current documenation on the [online wiki](https://bitbucket.org/dragon512/reusable-gold-testing-system/wiki/Home) for general use of the system. ## Documenation of AuTest extension for ATS. -Autest allows for the creation of extension to help specilaize and simplify test writting for a given application domian. Minus API addition the extension code will check that python 3.5 or better is used. There is also a new command line argumented added: +Autest allows for the creation of extensions to help specialize and simplify test writing for a given application domain. Minus API addition the extension code will check that python 3.5 or better is used. There is also a new command line argumented added specifically for Trafficserver: --ats-bin < path to bin directory > @@ -45,7 +44,7 @@ This command line argument will point to your build of ATS you want to test. At * command - optional argument defining what process to use. Defaults to traffic_server. * select_ports - have the testing system auto select the ports to use for this instance of ATS -This function will define a sandbox for an instance of trafficserver to run under. The function will return a AuTest process object that will have a number of files and variables define for making it easier to define a test. +This function will define a sandbox for an instance of trafficserver to run under. The function will return a AuTest process object that will have a number of files and variables defined to make it easier for test definition. #### Environment The environment of the process will have a number of added environment variables to control trafficserver running the in the sandbox location correctly. This can be used to easily setup other commands that should run under same environment. @@ -64,7 +63,7 @@ tr.Processes.Default.Env=ts.Env ``` #### Variables -These are the current variable that are define dynamically +These are the current variables that are defined dynamically for Trafficserver port - the ipv4 port to listen on portv6 - the ipv4 port to listen on @@ -72,7 +71,7 @@ manager_port - the manager port used. This is set even is select_port is False admin_port - the admin port used. This is set even is select_port is False #### File objects -A number of file object are define to help with adding values to a given configuration value to for a test, or testing a value exists in a log file. File that are defined currently are: +A number of file objects are defined to help with adding values to a given configuration value to for a test, or testing a value exists in a log file. File that are defined currently are: ##### log files * squid.log diff --git a/tests/bootstrap.py b/tests/bootstrap.py index de9b04e3e61..cfef1a133cd 100755 --- a/tests/bootstrap.py +++ b/tests/bootstrap.py @@ -31,6 +31,7 @@ "requests", "dnslib", "httpbin", + "traffic-replay" # this should install TRLib, MicroServer, MicroDNS, Traffic-Replay ] diff --git a/tests/gold_tests/autest-site/microDNS.test.ext b/tests/gold_tests/autest-site/microDNS.test.ext index e45a496c090..fdeffe00088 100644 --- a/tests/gold_tests/autest-site/microDNS.test.ext +++ b/tests/gold_tests/autest-site/microDNS.test.ext @@ -16,12 +16,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ports import get_port import json import os import sys +import trlib.ipconstants as IPConstants +from ports import get_port + # AddRecord registers a list of ip address against hostname + + def AddRecord(hostname, list_ip_addr): record = dict() @@ -69,7 +73,6 @@ def addRecords(self, records=None, jsonFile=None): def MakeDNServer(obj, name, filename="dns_file.json", port=False, ip='INADDR_LOOPBACK', rr=False, default=None, options={}): - server_path = os.path.join(obj.Variables.AtsTestToolsDir, 'microDNS/uDNS.py') data_dir = os.path.join(obj.RunDirectory, name) filepath = os.path.join(data_dir, filename) obj.Variables.zone_file = filepath @@ -90,7 +93,7 @@ def MakeDNServer(obj, name, filename="dns_file.json", port=False, ip='INADDR_LOO p = obj.Processes.Process(name) if (port == False): port = get_port(p, "Port") - command = "python3 {0} {1} {2} {3}".format(server_path, ip, port, filepath) + command = "microdns {0} {1} {2}".format(ip, port, filepath) if rr: command += " --rr" @@ -101,10 +104,6 @@ def MakeDNServer(obj, name, filename="dns_file.json", port=False, ip='INADDR_LOO p.Variables.DataDir = data_dir p.ReturnCode = 0 - # to get the IP keywords in tools/lib - sys.path.append(obj.Variables.AtsTestToolsDir) - import lib.IPConstants as IPConstants - if IPConstants.isIPv6(ip): p.Ready = When.PortOpenv6(port) else: diff --git a/tests/gold_tests/autest-site/microserver.test.ext b/tests/gold_tests/autest-site/microserver.test.ext index 6fe93df9f52..bef027f09c7 100644 --- a/tests/gold_tests/autest-site/microserver.test.ext +++ b/tests/gold_tests/autest-site/microserver.test.ext @@ -16,14 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from autest.api import AddWhenFunction -from ports import get_port import json import socket import ssl import time import sys +from autest.api import AddWhenFunction +from ports import get_port + +import trlib.ipconstants as IPConstants +from trlib import Transaction, Request, Response, Session + +DEFAULT_LOOKUP_KEY = '{PATH}' + def addMethod(self, testName, request_header, functionName): return @@ -52,51 +58,15 @@ def getHeaderFieldVal(request_header, field): return val # addResponse adds customized response with respect to request_header. request_header and response_header are both dictionaries +def addResponse(self, filename, request_header, response_header): + client_request = Request.fromRequestLine(request_header["headers"], request_header["body"], None if "options" not in request_header else request_header["options"]) + server_response = Response.fromRequestLine(response_header["headers"], response_header["body"], None if "options" not in response_header else response_header["options"]) + # timestamp field is left None because that needs to be revised for better implementation + txn = Transaction(client_request, None, server_response, None, None, None) -def addResponse(self, filename, request_header, response_header): - requestline = request_header["headers"].split("\r\n")[0] - host_ = "" - path_ = "" - if requestline: - url_part = requestline.split(" ") - if len(url_part) > 1: - if url_part[1].startswith("http"): - path_ = url_part[1].split("/", 2)[2] - host_, path_ = path_.split("/", 1) - else: - path_ = url_part[1].split("/", 1)[1] - - kpath = "" - - argsList = [] - keyslist = self.Variables.lookup_key.split("}") - for keystr in keyslist: - if keystr == '{PATH': - kpath = kpath + path_ - continue - if keystr == '{HOST': - kpath = kpath + host_ - continue - if keystr == '': # empty - continue - stringk = keystr.replace("{%", "") - argsList.append(stringk) - KeyList = [] - for argsL in argsList: - field_val = getHeaderFieldVal(request_header, argsL) - if field_val != None: - KeyList.append(field_val) - rl = "".join(KeyList) + kpath - txn = dict() - txn["timestamp"] = "" - txn["uuid"] = rl - txn["request"] = request_header - txn["response"] = response_header absFilepath = os.path.join(self.Variables.DataDir, filename) addTransactionToSession(txn, absFilepath) - # absFilepath=os.path.abspath(filename) - # self.Setup.CopyAs(absFilepath,self.Variables.DataDir) return # adds transaction in json format to the specified file @@ -110,18 +80,25 @@ def addTransactionToSession(txn, JFile): jf = open(JFile, 'r') jsondata = json.load(jf) + # hard coding only 1 session per file + # since for the purpose of testing, we don't need multiple sessions in a file if jsondata == None: - jsondata = dict() - jsondata["version"] = '0.2' - jsondata["timestamp"] = "1234567890.098" - jsondata["encoding"] = "url_encoded" - jsondata["txns"] = list() - jsondata["txns"].append(txn) + jsondata = {} + jsondata["sessions"] = [] + + jsondata["sessions"].append(Session(JFile.split("/")[-1], None, None, [txn]).toJSON()) + jsondata["meta"] = {} + jsondata["meta"]["version"] = "1.0" else: - jsondata["txns"].append(txn) + # hardcoding 0 because for testing we only have 1 session + jsondata["sessions"][0]["transactions"].append(txn.toJSON()) + with open(JFile, 'w+') as jf: jf.write(json.dumps(jsondata)) +def addSessionFromFiles(self, session_dir): + self.Setup.Copy(session_dir, self.Variables.DataDir) + # make headers with the key and values provided def makeHeader(self, requestString, **kwargs): @@ -132,73 +109,83 @@ def makeHeader(self, requestString, **kwargs): return headerStr -def uServerUpAndRunning(host, port, isSsl, isIPv6, clientcert='', clientkey=''): +def uServerUpAndRunning(serverHost, port, isSsl, isIPv6, request, clientcert='', clientkey=''): if isIPv6: plain_sock = socket.socket(socket.AF_INET6) else: plain_sock = socket.socket(socket.AF_INET) - if isSsl: - if clientcert != '' or clientkey != '': - sock = ssl.wrap_socket(plain_sock, keyfile=clientkey, certfile=clientcert) - else: - sock = ssl.wrap_socket(plain_sock) + if isSsl: + if clientcert != '' or clientkey != '': + sock = ssl.wrap_socket(plain_sock, keyfile=clientkey, certfile=clientcert) + else: + sock = ssl.wrap_socket(plain_sock) else: - sock = plain_sock + sock = plain_sock try: - sock.connect((host, port)) + sock.connect((serverHost, port)) except ConnectionRefusedError: return False - sock.sendall("GET /ruok HTTP/1.1\r\nHost: {}\r\n\r\n".format(host).encode()) - decoded_output='' + sock.sendall(request.encode()) + decoded_output = '' while True: + host.WriteDebug("??") output = sock.recv(4096) # suggested bufsize from docs.python.org + host.WriteDebug("!!") if len(output) <= 0: break else: - decoded_output+=output.decode() + decoded_output += output.decode() sock.close() sock = None - expected_response="HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 4\r\n\r\nimok" + expected_response = "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 4\r\n\r\nimok" if decoded_output == expected_response: return True - raise RuntimeError('\n'.join([ - 'Got invalid response from microserver:', - '----', - decoded_output, - '----'])) -AddWhenFunction(uServerUpAndRunning) + host.WriteError('\n'.join([ + 'Got invalid response from microserver:', + '----', + decoded_output, + '----'])) + + +AddWhenFunction(uServerUpAndRunning) -def MakeOriginServer(obj, name, port=False, ip='INADDR_LOOPBACK', delay=False, ssl=False, lookup_key='{PATH}', mode='test', options={}, clientcert='', clientkey=''): - # to get the IP keywords in tools/lib - sys.path.append(obj.Variables.AtsTestToolsDir) - import lib.IPConstants as IPConstants - server_path = os.path.join(obj.Variables.AtsTestToolsDir, 'microServer/uWServer.py') +def MakeOriginServer(obj, name, port=None, s_port=None, ip='INADDR_LOOPBACK', delay=None, ssl=False, lookup_key=DEFAULT_LOOKUP_KEY, clientcert='', clientkey='', both=False, options={}): data_dir = os.path.join(obj.RunDirectory, name) - # create Process p = obj.Processes.Process(name) - if (port == False): - port = get_port(p, "Port") - ipaddr = IPConstants.getIP(ip) - if (delay == False): - delay = 0 + command = "microserver --data-dir {0} --ip_address {1} --lookupkey '{2}'".format(data_dir, ipaddr, lookup_key) + + if delay: + command += " --delay {0}".format(delay) + + if both or ssl: + if not s_port: + s_port = get_port(p, "SSL_Port") - command = "python3 {0} --data-dir {1} --port {2} --ip_address {3} --delay {4} -m test --lookupkey '{5}' -m {6}".format( - server_path, data_dir, port, ipaddr, delay, lookup_key, mode) + command += " --both" if both else " --ssl" + key = clientkey if clientkey else os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.pem") + cert = clientcert if clientcert else os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.crt") + command += " --key {0}".format(key) + command += " --cert {0}".format(cert) + command += " --s_port {0}".format(s_port) - if ssl: - command += " --ssl True" + # this might break if user specifies both both and ssl + if not ssl: # in both or HTTP only mode + if not port: + port = get_port(p, "Port") + + command += " --port {0}".format(port) for flag, value in options.items(): - command += " {} {}".format(flag, value) + command += " {} {}".format(flag, value if value else '') p.Command = command p.Setup.MakeDir(data_dir) @@ -206,20 +193,33 @@ def MakeOriginServer(obj, name, port=False, ip='INADDR_LOOPBACK', delay=False, s p.Variables.lookup_key = lookup_key AddMethodToInstance(p, addResponse) AddMethodToInstance(p, addTransactionToSession) + AddMethodToInstance(p, addSessionFromFiles) - # Set up health check. - addResponse(p, "healthcheck.json", { - "headers": "GET /ruok HTTP/1.1\r\nHost: {}\r\n\r\n".format(ipaddr), + custom_lookup_header = '' + keys = lookup_key.split("}") + + for key in keys: + if key not in ['{PATH', '{URL', '{HOST', '{%Host']: + k = key.replace("{%", "") + + if len(k) > 0: + custom_lookup_header += '{0}: healthcheck\r\n'.format(k) + + healthcheck_request = { + "headers": "GET /ruok HTTP/1.1\r\nHost: {0}\r\n{1}\r\n".format(ipaddr, custom_lookup_header), "timestamp": "1469733493.993", "body": "" - }, { + } + + # Set up health check. + addResponse(p, "healthcheck.json", healthcheck_request, { "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "imok", - "options": "skipHooks" + "options": {"skipHooks": None} }) - p.Ready = When.uServerUpAndRunning(ipaddr, port, ssl, IPConstants.isIPv6(ip), clientcert=clientcert, clientkey=clientkey) + p.Ready = When.uServerUpAndRunning(ipaddr, s_port if ssl else port, ssl, IPConstants.isIPv6(ip), healthcheck_request["headers"], clientcert=clientcert, clientkey=clientkey) p.ReturnCode = Any(None, 0) return p diff --git a/tests/gold_tests/autest-site/traffic_replay.test.ext b/tests/gold_tests/autest-site/traffic_replay.test.ext new file mode 100644 index 00000000000..340fae93d48 --- /dev/null +++ b/tests/gold_tests/autest-site/traffic_replay.test.ext @@ -0,0 +1,91 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# default 'mixed' for connection type since it doesn't hurt +def Replay(obj, name, replay_dir, key=None, cert=None, conn_type='mixed', options={}): + # ATS setup - one line because we leave records and remap config to user + ts = obj.MakeATSProcess("ts", select_ports=False) # select ports can be disabled once we add ssl port selection in extension + + # TEMP + ts.Variables.ssl_port = 4443 + + ts.addSSLfile(os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.pem")) + ts.addSSLfile(os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.crt")) + + ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.pem' + ) + + # MicroServer setup - NOTE: expand to multiple microserver in future? + server = obj.MakeOriginServer("server", both=True, lookup_key='{%uuid}') + server.addSessionFromFiles(replay_dir) + + # MicroDNS setup + dns = obj.MakeDNServer("dns", default=['127.0.0.1']) + + # Traffic Replay setup + data_dir = os.path.join(obj.RunDirectory, name) + + # NOTE: we are forcing mixed connection types for now for the sake of simplicity + + # if conn_type != 'nossl': + # if not key: + # host.WriteError("Must provide SSL key to traffic-replay.") + + # if not cert: + # host.WriteError("Must provide SSL key to traffic-replay.") + + # if not ts.Variables.ssl_port: + # host.WriteError("Must set traffic server with an ssl port") + + # NOTE: does this need change? + hostIP = '127.0.0.1' + + if not key: + key = os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.pem") + + if not cert: + cert = os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.crt") + + command = 'traffic-replay --log_dir {0} --type {1} --verify --host {2} --port {3} --s_port {4} '.format(data_dir, conn_type, hostIP, ts.Variables.port, ts.Variables.ssl_port) + + if key: + command += "-k {0} ".format(key) + + if cert: + command += "--ca_cert {0} ".format(cert) + + if options: + for flag, value in options.items(): + command += "{} {} ".format(flag, value if value else '') + + tr = obj.AddTestRun(name) + tr.Command = command + # tr.Command = "echo Hi" + tr.Setup.MakeDir(data_dir) + tr.Setup.Copy(replay_dir, data_dir) + tr.Processes.Default.StartBefore(server) + tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.ssl_port)) + tr.Processes.Default.StartBefore(dns) + tr.ReturnCode = Any(None, 0) + tr.Processes.Default.Streams.All = Testers.ExcludesExpression("FAIL", "No fails allowed.") + + # return all the stuff in case user wants to do extra optimization + return (ts, server, dns, tr) + +AddTestRunSet(Replay) diff --git a/tests/gold_tests/chunked_encoding/chunked_encoding.test.py b/tests/gold_tests/chunked_encoding/chunked_encoding.test.py index f511a215ece..5803d626a26 100644 --- a/tests/gold_tests/chunked_encoding/chunked_encoding.test.py +++ b/tests/gold_tests/chunked_encoding/chunked_encoding.test.py @@ -83,7 +83,7 @@ 'map http://www.yetanotherexample.com http://127.0.0.1:{0}'.format(server3.Variables.Port) ) ts.Disk.remap_config.AddLine( - 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.Port, ts.Variables.ssl_port) + 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.SSL_Port, ts.Variables.ssl_port) ) diff --git a/tests/gold_tests/h2/gold/post_chunked.gold b/tests/gold_tests/h2/gold/post_chunked.gold index ad471007bd7..0ff06d17219 100644 --- a/tests/gold_tests/h2/gold/post_chunked.gold +++ b/tests/gold_tests/h2/gold/post_chunked.gold @@ -1 +1 @@ -0123456789 \ No newline at end of file +abbbbbbbbb \ No newline at end of file diff --git a/tests/gold_tests/h2/http2.test.py b/tests/gold_tests/h2/http2.test.py index 747ae0c8cba..5a841f54486 100644 --- a/tests/gold_tests/h2/http2.test.py +++ b/tests/gold_tests/h2/http2.test.py @@ -126,14 +126,15 @@ tr.Processes.Default.Streams.stdout = "gold/chunked.gold" tr.StillRunningAfter = server +# NOTE: Skipping this test run because traffic-replay doesn't currently support H2 # Test Case 4: Multiple request -client_path = os.path.join(Test.Variables.AtsTestToolsDir, 'traffic-replay/') -tr = Test.AddTestRun() -tr.Processes.Default.Command = "python3 {0} -type {1} -log_dir {2} -port {3} -host '127.0.0.1' -s_port {4} -v -colorize False".format( - client_path, 'h2', server.Variables.DataDir, ts.Variables.port, ts.Variables.ssl_port) -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Streams.stdout = "gold/replay.gold" -tr.StillRunningAfter = server +# client_path = os.path.join(Test.Variables.AtsTestToolsDir, 'traffic-replay/') +# tr = Test.AddTestRun() +# tr.Processes.Default.Command = "python3 {0} -type {1} -log_dir {2} -port {3} -host '127.0.0.1' -s_port {4} -v -colorize False".format( +# client_path, 'h2', server.Variables.DataDir, ts.Variables.port, ts.Variables.ssl_port) +# tr.Processes.Default.ReturnCode = 0 +# tr.Processes.Default.Streams.stdout = "gold/replay.gold" +# tr.StillRunningAfter = server # Test Case 5:h2_active_timeout tr = Test.AddTestRun() diff --git a/tests/gold_tests/headers/cachedIMSRange.test.py b/tests/gold_tests/headers/cachedIMSRange.test.py index da93f95be3d..a8dabae52aa 100644 --- a/tests/gold_tests/headers/cachedIMSRange.test.py +++ b/tests/gold_tests/headers/cachedIMSRange.test.py @@ -41,16 +41,16 @@ server.addResponse("sessionlog.json", request_header, response_header) # IMS revalidation request request_IMS_header = {"headers": "GET / HTTP/1.1\r\nUID: IMS\r\nIf-Modified-Since: Tue, 08 May 2018 15:49:41 GMT\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_IMS_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993"} +response_IMS_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": None} server.addResponse("sessionlog.json", request_IMS_header, response_IMS_header) # EtagFill -request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagFill\r\n\r\n", "timestamp": "1469733493.993"} +request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagFill\r\n\r\n", "timestamp": "1469733493.993", "body": None} response_etagfill_header = {"headers": "HTTP/1.1 200 OK\r\nETag: myetag\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} server.addResponse("sessionlog.json", request_etagfill_header, response_etagfill_header) # INM revalidation -request_INM_header = {"headers": "GET /etag HTTP/1.1\r\nUID: INM\r\nIf-None-Match: myetag\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993"} -response_INM_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nETag: myetag\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993"} +request_INM_header = {"headers": "GET /etag HTTP/1.1\r\nUID: INM\r\nIf-None-Match: myetag\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": None} +response_INM_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nETag: myetag\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": None} server.addResponse("sessionlog.json", request_INM_header, response_INM_header) # object changed to 0 byte @@ -59,7 +59,7 @@ server.addResponse("sessionlog.json", request_noBody_header, response_noBody_header) # etag object now is a 404. Yeah, 404s don't usually have Cache-Control, but, ATS's default is to cache 404s for a while. -request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagError\r\n\r\n", "timestamp": "1469733493.993"} +request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagError\r\n\r\n", "timestamp": "1469733493.993", "body": None} response_etagfill_header = {"headers": "HTTP/1.1 404 Not Found\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_etagfill_header, response_etagfill_header) diff --git a/tests/gold_tests/remap/remap_https.test.py b/tests/gold_tests/remap/remap_https.test.py index 661863d5c34..efaa444b33d 100644 --- a/tests/gold_tests/remap/remap_https.test.py +++ b/tests/gold_tests/remap/remap_https.test.py @@ -61,7 +61,7 @@ 'map https://www.example.com:{1} http://127.0.0.1:{0}'.format(server.Variables.Port, ts.Variables.ssl_port) ) ts.Disk.remap_config.AddLine( - 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.Port, ts.Variables.ssl_port) + 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.SSL_Port,ts.Variables.ssl_port) ) diff --git a/tests/gold_tests/tls/tls_check_cert_selection.test.py b/tests/gold_tests/tls/tls_check_cert_selection.test.py index 92a0e183a2f..f1eaa8fde94 100644 --- a/tests/gold_tests/tls/tls_check_cert_selection.test.py +++ b/tests/gold_tests/tls/tls_check_cert_selection.test.py @@ -48,7 +48,7 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map / https://foo.com:{1}'.format(ts.Variables.ssl_port, server.Variables.Port)) + 'map / https://foo.com:{1}'.format(ts.Variables.ssl_port, server.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLines([ 'dest_ip=127.0.0.1 ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key', diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index 0fcab7be1f2..394b8abc506 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -29,8 +29,9 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=False) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) -server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": "true"}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) -server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": "true"}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +# --clientverify: "" empty string because microserver does store_true for argparse, but options is a dictionary +server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) server.Setup.Copy("ssl/signer.pem") server.Setup.Copy("ssl/signer2.pem") server.Setup.Copy("ssl/signed-foo.pem") @@ -84,10 +85,10 @@ ) ts.Disk.remap_config.AddLine( - 'map /case1 https://127.0.0.1:{0}/'.format(server.Variables.Port) + 'map /case1 https://127.0.0.1:{0}/'.format(server.Variables.SSL_Port) ) ts.Disk.remap_config.AddLine( - 'map /case2 https://127.0.0.1:{0}/'.format(server2.Variables.Port) + 'map /case2 https://127.0.0.1:{0}/'.format(server2.Variables.SSL_Port) ) ts.Disk.ssl_server_name_yaml.AddLine( diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py index 35cd0b642e3..a2411362cf5 100644 --- a/tests/gold_tests/tls/tls_client_cert2.test.py +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -29,8 +29,8 @@ ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=False) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) -server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": "true"}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) -server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": "true"}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) server.Setup.Copy("ssl/signer.pem") server.Setup.Copy("ssl/signer2.pem") server.Setup.Copy("ssl/signed-foo.pem") @@ -80,10 +80,10 @@ ) ts.Disk.remap_config.AddLine( - 'map /case1 https://127.0.0.1:{0}/'.format(server.Variables.Port) + 'map /case1 https://127.0.0.1:{0}/'.format(server.Variables.SSL_Port) ) ts.Disk.remap_config.AddLine( - 'map /case2 https://127.0.0.1:{0}/'.format(server2.Variables.Port) + 'map /case2 https://127.0.0.1:{0}/'.format(server2.Variables.SSL_Port) ) ts.Disk.ssl_server_name_yaml.AddLines([ diff --git a/tests/gold_tests/tls/tls_client_cert_override.test.py b/tests/gold_tests/tls/tls_client_cert_override.test.py index 25a82cd848a..e2fe10dd577 100644 --- a/tests/gold_tests/tls/tls_client_cert_override.test.py +++ b/tests/gold_tests/tls/tls_client_cert_override.test.py @@ -29,8 +29,8 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=False) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) -server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": "true"}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) -server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": "true"}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) server.Setup.Copy("ssl/signer.pem") server.Setup.Copy("ssl/signer2.pem") server.Setup.Copy("ssl/signed-foo.pem") @@ -82,16 +82,16 @@ ) ts.Disk.remap_config.AddLine( - 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.Port, "signed-foo.pem", "signed-foo.key") + 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.SSL_Port, "signed-foo.pem", "signed-foo.key") ) ts.Disk.remap_config.AddLine( - 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.Port, "signed2-foo.pem", "signed-foo.key") + 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.SSL_Port, "signed2-foo.pem", "signed-foo.key") ) ts.Disk.remap_config.AddLine( - 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.Port, "signed2-foo.pem", "signed-foo.key") + 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.SSL_Port, "signed2-foo.pem", "signed-foo.key") ) ts.Disk.remap_config.AddLine( - 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.Port, "signed-foo.pem", "signed-foo.key") + 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.SSL_Port, "signed-foo.pem", "signed-foo.key") ) # Should succeed diff --git a/tests/gold_tests/tls/tls_hooks_verify.test.py b/tests/gold_tests/tls/tls_hooks_verify.test.py index 3c49f35da49..50d94047bfe 100644 --- a/tests/gold_tests/tls/tls_hooks_verify.test.py +++ b/tests/gold_tests/tls/tls_hooks_verify.test.py @@ -57,13 +57,13 @@ ) ts.Disk.remap_config.AddLine( - 'map https://foo.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.Port,ts.Variables.ssl_port) + 'map https://foo.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port, ts.Variables.ssl_port) ) ts.Disk.remap_config.AddLine( - 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.Port,ts.Variables.ssl_port) + 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port, ts.Variables.ssl_port) ) ts.Disk.remap_config.AddLine( - 'map https://random.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.Port,ts.Variables.ssl_port) + 'map https://random.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port, ts.Variables.ssl_port) ) ts.Disk.ssl_server_name_yaml.AddLine( diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index a33c7c43349..007077b7aee 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -64,7 +64,7 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), - 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_foo.Variables.Port,server_bar.Variables.Port), + 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_foo.Variables.SSL_Port,server_bar.Variables.SSL_Port), 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', @@ -77,11 +77,11 @@ # empty SNI should tunnel to server_bar ts.Disk.ssl_server_name_yaml.AddLines([ '- fqdn: foo.com', - " tunnel_route: localhost:{0}".format(server_foo.Variables.Port), + " tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port), "- fqdn: bob.*.com", - " tunnel_route: localhost:{0}".format(server_foo.Variables.Port), + " tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port), "- fqdn: ''", # No SNI sent - " tunnel_route: localhost:{0}".format(server_bar.Variables.Port) + " tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port) ]) tr = Test.AddTestRun("foo.com Tunnel-test") @@ -135,7 +135,7 @@ tr.Disk.File(snipath, id = "ssl_server_name_yaml", typename="ats:config"), tr.Disk.ssl_server_name_yaml.AddLines([ '- fqdn: bar.com', - " tunnel_route: localhost:{0}".format(server_bar.Variables.Port), + " tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port), ]) tr.StillRunningAfter = ts tr.StillRunningAfter = server_foo diff --git a/tests/gold_tests/tls/tls_tunnel_forward.test.py b/tests/gold_tests/tls/tls_tunnel_forward.test.py index 93ca7329ea4..f8068e7800d 100644 --- a/tests/gold_tests/tls/tls_tunnel_forward.test.py +++ b/tests/gold_tests/tls/tls_tunnel_forward.test.py @@ -68,7 +68,7 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), - 'proxy.config.http.connect_ports': '{0} {1} {2} {3}'.format(ts.Variables.ssl_port,server_foo.Variables.Port,server_bar.Variables.Port,server_random.Variables.Port), + 'proxy.config.http.connect_ports': '{0} {1} {2} {3}'.format(ts.Variables.ssl_port, server_foo.Variables.SSL_Port, server_bar.Variables.Port, server_random.Variables.Port), 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', @@ -80,7 +80,7 @@ # bar.com should terminate. Forward its tcp stream to server_bar ts.Disk.ssl_server_name_yaml.AddLines([ "- fqdn: 'foo.com'", - " tunnel_route: 'localhost:{0}'".format(server_foo.Variables.Port), + " tunnel_route: 'localhost:{0}'".format(server_foo.Variables.SSL_Port), "- fqdn: 'bar.com'", " forward_route: 'localhost:{0}'".format(server_bar.Variables.Port), "- fqdn: ''", #default case diff --git a/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py index db03cc8d90b..229c4af9c5e 100644 --- a/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py +++ b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py @@ -66,7 +66,7 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), - 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_bar.Variables.Port,server_random.Variables.Port), + 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_bar.Variables.SSL_Port,server_random.Variables.SSL_Port), 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', @@ -79,9 +79,9 @@ # newname should tunnel to server_bar ts.Disk.ssl_server_name_yaml.AddLines([ "- fqdn: newname", - " tunnel_route: localhost:{0}".format(server_bar.Variables.Port), + " tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port), "- fqdn: ''", #default case - " tunnel_route: localhost:{0}".format(server_random.Variables.Port), + " tunnel_route: localhost:{0}".format(server_random.Variables.SSL_Port), ]) # Plugin should add "newname" to the empty sni and go to _bar instead of random.com diff --git a/tests/gold_tests/tls/tls_verify.test.py b/tests/gold_tests/tls/tls_verify.test.py index d5b63e641b8..609f56c278f 100644 --- a/tests/gold_tests/tls/tls_verify.test.py +++ b/tests/gold_tests/tls/tls_verify.test.py @@ -54,15 +54,15 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' diff --git a/tests/gold_tests/tls/tls_verify2.test.py b/tests/gold_tests/tls/tls_verify2.test.py index ca818ac559b..197b3c88add 100644 --- a/tests/gold_tests/tls/tls_verify2.test.py +++ b/tests/gold_tests/tls/tls_verify2.test.py @@ -54,15 +54,15 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) + 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' diff --git a/tests/gold_tests/tls/tls_verify3.test.py b/tests/gold_tests/tls/tls_verify3.test.py index 9668a489910..3999f657788 100644 --- a/tests/gold_tests/tls/tls_verify3.test.py +++ b/tests/gold_tests/tls/tls_verify3.test.py @@ -54,15 +54,15 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map https://foo.com:{1}/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port, ts.Variables.ssl_port)) + 'map https://foo.com:{1}/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port, ts.Variables.ssl_port)) ts.Disk.remap_config.AddLine( - 'map https://bob.foo.com:{1}/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port, ts.Variables.ssl_port)) + 'map https://bob.foo.com:{1}/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port, ts.Variables.ssl_port)) ts.Disk.remap_config.AddLine( - 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port, ts.Variables.ssl_port)) + 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port, ts.Variables.ssl_port)) ts.Disk.remap_config.AddLine( - 'map https://bob.bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port,ts.Variables.ssl_port)) + 'map https://bob.bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port,ts.Variables.ssl_port)) ts.Disk.remap_config.AddLine( - 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' @@ -103,10 +103,10 @@ tr.Setup.Copy("ssl/signed-bar.pem") tr.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.Processes.Default.StartBefore(server_foo) tr.Processes.Default.StartBefore(server_bar) tr.Processes.Default.StartBefore(server) -tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") diff --git a/tests/gold_tests/tls/tls_verify_ca_override.test.py b/tests/gold_tests/tls/tls_verify_ca_override.test.py index bb622d5f697..a2c5acf0047 100644 --- a/tests/gold_tests/tls/tls_verify_ca_override.test.py +++ b/tests/gold_tests/tls/tls_verify_ca_override.test.py @@ -54,16 +54,16 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.Port, ts.Variables.SSLDir, "signer.pem") + 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.SSL_Port, ts.Variables.SSLDir, "signer.pem") ) ts.Disk.remap_config.AddLine( - 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.Port, ts.Variables.SSLDir, "signer2.pem") + 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.SSL_Port, ts.Variables.SSLDir, "signer2.pem") ) ts.Disk.remap_config.AddLine( - 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.Port, ts.Variables.SSLDir, "signer2.pem") + 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.SSL_Port, ts.Variables.SSLDir, "signer2.pem") ) ts.Disk.remap_config.AddLine( - 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.Port, ts.Variables.SSLDir, "signer.pem") + 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.SSL_Port, ts.Variables.SSLDir, "signer.pem") ) ts.Disk.ssl_multicert_config.AddLine( diff --git a/tests/gold_tests/tls/tls_verify_not_pristine.test.py b/tests/gold_tests/tls/tls_verify_not_pristine.test.py index 8153008f760..e416d94d436 100644 --- a/tests/gold_tests/tls/tls_verify_not_pristine.test.py +++ b/tests/gold_tests/tls/tls_verify_not_pristine.test.py @@ -50,9 +50,9 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map https://bar.com:{0}/ https://foo.com:{1}'.format(ts.Variables.ssl_port, server_foo.Variables.Port)) + 'map https://bar.com:{0}/ https://foo.com:{1}'.format(ts.Variables.ssl_port, server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://foo.com:{0}/ https://bar.com:{1}'.format(ts.Variables.ssl_port, server_foo.Variables.Port)) + 'map https://foo.com:{0}/ https://bar.com:{1}'.format(ts.Variables.ssl_port, server_foo.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' diff --git a/tests/gold_tests/tls/tls_verify_override.test.py b/tests/gold_tests/tls/tls_verify_override.test.py index 010e0d327d2..fe7147c8dd4 100644 --- a/tests/gold_tests/tls/tls_verify_override.test.py +++ b/tests/gold_tests/tls/tls_verify_override.test.py @@ -56,33 +56,33 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map http://foo.com/basictobar https://bar.com:{0}'.format(server_bar.Variables.Port)) + 'map http://foo.com/basictobar https://bar.com:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.Port)) + 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.Port)) + 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /basic https://random.com:{0}'.format(server.Variables.Port)) + 'map /basic https://random.com:{0}'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.Port)) + 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.Port)) + 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.Port)) + 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.Port)) + 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.Port)) + 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.Port)) + 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' diff --git a/tests/gold_tests/tls/tls_verify_override_base.test.py b/tests/gold_tests/tls/tls_verify_override_base.test.py index 244e4d3cbd9..d14763411ea 100644 --- a/tests/gold_tests/tls/tls_verify_override_base.test.py +++ b/tests/gold_tests/tls/tls_verify_override_base.test.py @@ -56,31 +56,31 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.Port)) + 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.Port)) + 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.Port)) + 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /basic https://127.0.0.1:{0}'.format(server.Variables.Port)) + 'map /basic https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.Port)) + 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.Port)) + 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.Port)) + 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.Port)) + 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.Port)) + 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.Port)) + 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' diff --git a/tests/gold_tests/tls_hooks/tls_hooks13.test.py b/tests/gold_tests/tls_hooks/tls_hooks13.test.py index 9388221bce6..704a111cab3 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks13.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks13.test.py @@ -53,7 +53,7 @@ ) ts.Disk.remap_config.AddLine( - 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.Port) + 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.SSL_Port) ) Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-out_start=1 -out_close=2') diff --git a/tests/gold_tests/tls_hooks/tls_hooks14.test.py b/tests/gold_tests/tls_hooks/tls_hooks14.test.py index b411f65c6fe..a8c8d3d6acf 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks14.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks14.test.py @@ -53,7 +53,7 @@ ) ts.Disk.remap_config.AddLine( - 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.Port) + 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.SSL_Port) ) Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-out_start_delay=2') diff --git a/tests/gold_tests/tls_hooks/tls_hooks15.test.py b/tests/gold_tests/tls_hooks/tls_hooks15.test.py index ed6e687cda1..9194802f46b 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks15.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks15.test.py @@ -53,7 +53,7 @@ ) ts.Disk.remap_config.AddLine( - 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.Port) + 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.SSL_Port) ) Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-close=2 -out_close=1') diff --git a/tests/tools/lib/IPConstants.py b/tests/tools/lib/IPConstants.py deleted file mode 100644 index 0531137162b..00000000000 --- a/tests/tools/lib/IPConstants.py +++ /dev/null @@ -1,48 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ipaddress - -# convenience functions - -IPkw = {'INADDR_LOOPBACK':'127.0.0.1', - 'IN6ADDR_LOOPBACK':'::1', - 'INADDR_ANY':'0.0.0.0', - 'IN6ADDR_ANY':'::'} - -def isIPv6(addr): - if addr in IPkw: - addr = IPkw[addr] - - - return ipaddress.ip_address(addr).version == 6 - - -def isIPv4(addr): - if addr in IPkw: - addr = IPkw[addr] - - return ipaddress.ip_address(addr).version == 4 - - -def getIP(addr): - if addr in IPkw: - addr = IPkw[addr] - - return str(ipaddress.ip_address(addr)) - diff --git a/tests/tools/lib/result.py b/tests/tools/lib/result.py deleted file mode 100644 index 7322e65f050..00000000000 --- a/tests/tools/lib/result.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - - -class TermColors: - ''' Collection of colors for printing out to terminal ''' - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - ENDC = '\033[0m' - - -ignoredFields = {'age', 'set-cookie', 'server', 'date', 'last-modified', - 'via', 'expires', 'cache-control', 'vary', 'connection'} # all lower case - - -class Result(object): - ''' Result encapsulates the result of a single session replay ''' - - def __init__(self, test_name, expected_response, received_response, recv_resp_body=None): - ''' expected_response and received_response can be any datatype the caller wants as long as they are the same datatype ''' - self._test_name = test_name - self._expected_response = expected_response - self._received_response = received_response - self._received_response_body = recv_resp_body - - def getTestName(self): - return self._test_name - - def getResultBool(self): - return self._expected_response == self._received_response - - def getRespBody(self): - if self._received_response_body: - return self._received_response_body - else: - return "" - - def Compare(self, received_dict, expected_dict, src=None): - global ignoredFields - # print("RECEIVED") - # print(received_dict) - # print("RECIEVED CACHE CONTROL") - # print(received_dict['Cache-Control'.lower()]) - # print("EXPECTED") - # print(expected_dict) - try: - for key in received_dict: - # print(key) - if key.lower() in expected_dict and key.lower() not in ignoredFields: - # print("{0} ==? {1}".format(expected_dict[key.lower()],received_dict[key])) - if received_dict[key.lower()] != expected_dict[key.lower()]: - print("{0}Difference in the field \"{1}\": \n received:\n{2}\n expected:\n{3}{4}".format( - TermColors.FAIL, key, received_dict[key], expected_dict[key], TermColors.ENDC)) - return False - if key.lower() == 'content-length' and self._received_response_body: - if int(received_dict[key.lower()]) != len(self._received_response_body): - print("{0}Difference in received content length and actual body length \ncontent-length: {1} \nbody: {2}\nbody length: {3}{4}".format( - TermColors.FAIL, received_dict[key.lower()], self._received_response_body, len( - self._received_response_body, TermColors.ENDC) - )) - return False - - except: - e = sys.exc_info() - if src: - print("In {0}: ".format(src), end='') - print("Error in comparing key ", e, key, "expected", expected_dict[key.lower()], "received", received_dict[key]) - return False - return True - - def getResult(self, received_dict, expected_dict, colorize=False): - global ignoredFields - retval = False - ''' Return a nicely formatted result string with color if requested ''' - if self.getResultBool() and self.Compare(received_dict, expected_dict, self._test_name): - if colorize: - outstr = "{0}PASS{1}".format( - TermColors.OKGREEN, TermColors.ENDC) - - else: - outstr = "PASS" - - retval = True - - else: - if colorize: - outstr = "{0}FAIL{1}: expected {2}, received {3}, session file: {4}".format( - TermColors.FAIL, TermColors.ENDC, self._expected_response, self._received_response, self._test_name) - - else: - outstr = "FAIL: expected {0}, received {1}".format( - self._expected_response, self._received_response) - - return (retval, outstr) diff --git a/tests/tools/microDNS/uDNS.py b/tests/tools/microDNS/uDNS.py deleted file mode 100644 index 297cdcb714d..00000000000 --- a/tests/tools/microDNS/uDNS.py +++ /dev/null @@ -1,207 +0,0 @@ -# coding=utf-8 - -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import sys -import time -import threading -import traceback -import socketserver -import argparse -import codecs -import json -from dnslib import * - -sys.path.append( - os.path.normpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..' - ) - ) -) - -import lib.IPConstants as IPConstants - -TTL = 60 * 5 # completely arbitrary TTL value -round_robin = False -default_records = list() -records = dict() - -class DomainName(str): - def __getattr__(self, item): - return DomainName(item + '.' + self) - - -class BaseRequestHandler(socketserver.BaseRequestHandler): - - def get_data(self): - raise NotImplementedError - - def send_data(self, data): - raise NotImplementedError - - def handle(self): - now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f') - print("\n\n%s request %s (%s %s):" % (self.__class__.__name__[:3], now, self.client_address[0], - self.client_address[1])) - try: - data = self.get_data() - self.send_data(dns_response(data)) - except Exception: - traceback.print_exc(file=sys.stderr) - - -class TCPRequestHandler(BaseRequestHandler): - - def get_data(self): - data = self.request.recv(8192).strip() - sz = int(codecs.encode(data[:2], 'hex'), 16) - if sz < len(data) - 2: - raise Exception("Wrong size of TCP packet") - elif sz > len(data) - 2: - raise Exception("Too big TCP packet") - return data[2:] - - def send_data(self, data): - sz = codecs.decode(hex(len(data))[2:].zfill(4), 'hex') - return self.request.sendall(sz + data) - - -class UDPRequestHandler(BaseRequestHandler): - - def get_data(self): - return self.request[0].strip() - - def send_data(self, data): - return self.request[1].sendto(data, self.client_address) - - -def build_domain_mappings(path): - with open(path) as f: - zone_file = json.load(f) - - for domain in zone_file['mappings']: - for d in iter(domain.keys()): - # this loop only runs once, kind of a hack to access the only key in the dict - domain_name = DomainName(d) - print("Domain name:", domain_name) - # we can test using python's built-in ipaddress module, but this should suffice - records[domain_name] = [A(x) if ":" not in x else AAAA(x) for x in domain[domain_name]] - print(records[domain_name]) - - if 'otherwise' in zone_file: - default_records.extend([A(d) if ":" not in d else AAAA(d) for d in zone_file['otherwise']]) - - -def add_authoritative_records(reply, domain): - # ns1 and ns1 are hardcoded in, change if necessary - reply.add_auth(RR(rname=domain, rtype=QTYPE.NS, rclass=1, ttl=TTL, rdata=NS(domain.ns1))) - reply.add_auth(RR(rname=domain, rtype=QTYPE.NS, rclass=1, ttl=TTL, rdata=NS(domain.ns2))) - - -def dns_response(data): - ''' dns_response takes in the raw bytes from the socket and does all the logic behind what - RRs get returned as the response ''' - global default_records, records, TTL, round_robin - - request = DNSRecord.parse(data) - print(request) - - reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q) - qname = request.q.qname - qn = str(qname) - qtype = request.q.qtype - qt = QTYPE[qtype] - found_specific = False - - # first look for a specific mapping - for domain, rrs in records.items(): - if domain == qn or qn.endswith('.' + domain): - # we are the authoritative name server for this domain and all subdomains - for rdata in rrs: - # only include requested record types (ie. A, MX, etc) - rqt = rdata.__class__.__name__ - if qt in ['*', rqt]: - found_specific = True - reply.add_answer(RR(rname=qname, rtype=getattr(QTYPE, str(rqt)), rclass=1, ttl=TTL, rdata=rdata)) - - # rotate the A entries if round robin is on - if round_robin: - a_records = [x for x in rrs if type(x) == A] - records[domain] = a_records[1:] + a_records[:1] # rotate list - break - - # else if a specific mapping is not found, return default A-records - if not found_specific: - for a in default_records: - found_specific = True - reply.add_answer(RR(rname=qname, rtype=QTYPE.A, rclass=1, ttl=TTL, rdata=a)) - - if round_robin: - default_records = default_records[1:] + default_records[:1] - - if not found_specific: - reply.header.set_rcode(3) - - print("---- Reply: ----\n", reply) - return reply.pack() - - -if __name__ == '__main__': - # handle cmd line args - parser = argparse.ArgumentParser() - parser.add_argument("ip", type=str, help="Interface") - parser.add_argument("port", type=int, help="port uDNS should listen on") - parser.add_argument("zone_file", help="path to zone file") - parser.add_argument("--rr", action='store_true', - help='round robin load balances if multiple IP addresses are present for 1 domain') - args = parser.parse_args() - - if IPConstants.isIPv6(args.ip): - # *UDPServer derives from TCPServer, so setting one will affect the other - socketserver.TCPServer.address_family = socket.AF_INET6 - - # exit(1) - - if args.rr: - round_robin = True - build_domain_mappings(args.zone_file) - - ipaddr = IPConstants.getIP(args.ip) - - servers = [ - socketserver.ThreadingUDPServer((ipaddr, args.port), UDPRequestHandler), - socketserver.ThreadingTCPServer((ipaddr, args.port), TCPRequestHandler), - ] - - print("Starting DNS on address {0} port {1}...".format(ipaddr, args.port)) - for s in servers: - thread = threading.Thread(target=s.serve_forever) # that thread will start one more thread for each request - thread.daemon = True # exit the server thread when the main thread terminates - thread.start() - - try: - while 1: - time.sleep(1) - sys.stderr.flush() - sys.stdout.flush() - - except KeyboardInterrupt: - print("Got SigINT") - # pass - finally: - for s in servers: - s.shutdown() diff --git a/tests/tools/microServer/README.md b/tests/tools/microServer/README.md deleted file mode 100644 index a7681a378b0..00000000000 --- a/tests/tools/microServer/README.md +++ /dev/null @@ -1,49 +0,0 @@ -uWServer -======== - -uWServer is a mock HTTP server that takes predefined set of sessions for serving response to HTTP requests. Each session includes one or more transactions. A transaction is composed of an HTTP request and an HTTP response. -uWServer accepts session data in JSON fromat only. - - -Command: ----------------- - -`python3.5 uWServer.py --data-dir ` - -Options: ------------ - -To see the options please run `python3.5 uWServer.py --help` - -Session Definitions: --------------------- - -Example session: - -``` -{ - "encoding": "url_encoded", - "version": "0.2", - "txns": [ - { - "response": { - "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", - "body": "", - "timestamp": "1469733493.993" - }, - "request": { - "headers": "GET / HTTP/1.1\r\nHost: www.example.test\r\n\r\n", - "body": "", - "timestamp": "1469733493.993" - }, - "uuid": "", - "timestamp": "" - } - ], - "timestamp": "1234567890.098" -} -``` - -Each session should be in its own file, and any number of files may be created to define sessions. - -The `response` map may include an `options` string, which is a comma-delimited list of options to be enabled. Currently the only option supported is `skipHooks`, which will ignore any hooks created for the matching request/response pair. See **Options**. diff --git a/tests/tools/microServer/uWServer.py b/tests/tools/microServer/uWServer.py deleted file mode 100644 index f2dfb383283..00000000000 --- a/tests/tools/microServer/uWServer.py +++ /dev/null @@ -1,734 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import string -import http.client -import cgi -import time -import sys -import json -import os -import threading -from ipaddress import ip_address -from http.server import BaseHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn, ForkingMixIn, BaseServer -from http import HTTPStatus -import argparse -import ssl -import socket -import importlib.util -import time -test_mode_enabled = True -lookup_key_ = "{PATH}" -__version__ = "1.1" - - -sys.path.append( - os.path.normpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..' - ) - ) -) - -import sessionvalidation.sessionvalidation as sv -import lib.IPConstants as IPConstants - - -SERVER_PORT = 5005 # default port -SERVER_DELAY = 0 # default delay -HTTP_VERSION = 'HTTP/1.1' -G_replay_dict = {} - -count = 0 - -# Simple class to hold lists of callbacks associated with a key. - - -class HookSet: - # Helper class to provide controlled access to the HookSet to the loading module. - class Registrar: - def __init__(self, hook_set): - self.hooks = hook_set - - def register(self, hook, cb): - self.hooks.register(hook, cb) - - def __init__(self): - self.hooks = {} - self.modules = [] - self.registrar = HookSet.Registrar(self) - # Define all the valid hooks here. - for item in ['ReadRequestHook']: - if isinstance(item, list): - hook = item[0] - label = item[1] - else: - hook = label = item - exec("HookSet.{} = '{}'".format(label, hook)) - exec("HookSet.Registrar.{} = '{}'".format(label, hook)) - self.hooks[hook] = [] - - def load(self, source): - try: - spec = importlib.util.spec_from_file_location('Observer', source) - mod = importlib.util.module_from_spec(spec) - mod.Hooks = self.registrar - spec.loader.exec_module(mod) - except ImportError: - print("Failed to import {}".format(source)) - else: - self.modules.append(mod) - - # Add a callback cb to the hook. - # Error if the hook isn't defined. - def register(self, hook, cb): - if hook in self.hooks: - self.hooks[hook].append(cb) - else: - raise ValueError("{} is not a valid hook name".format(hook)) - - # Invoke a hook. Pass on any additional arguments to the callback. - def invoke(self, hook, *args, **kwargs): - cb_list = self.hooks[hook] - if cb_list == None: - raise ValueError("{} is not a valid hook name to invoke".format(hook)) - else: - for cb in cb_list: - cb(*args, **kwargs) - - -class ThreadingServer(ThreadingMixIn, HTTPServer): - '''This class forces the creation of a new thread on each connection''' - - def __init__(self, local_addr, handler_class, options): - HTTPServer.__init__(self, local_addr, handler_class) - self.hook_set = HookSet() - if (options.load): - self.hook_set.load(options.load) - - -class ForkingServer(ForkingMixIn, HTTPServer): - '''This class forces the creation of a new process on each connection''' - pass - - -class SSLServer(ThreadingMixIn, HTTPServer): - def __init__(self, server_address, HandlerClass, options): - BaseServer.__init__(self, server_address, HandlerClass) - pwd = os.path.dirname(os.path.realpath(__file__)) - keys = os.path.join(pwd, options.key) - certs = os.path.join(pwd, options.cert) - clientCA = os.path.join(pwd, options.clientCA) - self.options = options - self.hook_set = HookSet() - - self.daemon_threads = True - self.protocol_version = 'HTTP/1.1' - - if options.load: - self.hook_set.load(options.load) - - print ("clientverify={0}".format(options.clientverify)) - - if options.clientverify: - self.socket = ssl.wrap_socket(socket.socket(self.address_family, self.socket_type), - keyfile=keys, certfile=certs, server_side=True, cert_reqs=ssl.CERT_REQUIRED, ca_certs=clientCA) - else: - self.socket = ssl.wrap_socket(socket.socket(self.address_family, self.socket_type), - keyfile=keys, certfile=certs, server_side=True) - - self.server_bind() - self.server_activate() - print("Port Configured for SSL communication") - - -class MyHandler(BaseHTTPRequestHandler): - def handleExpect100Continue(self, contentLength, chunked=False): - print("....expect", contentLength) - self.wfile.write(bytes('HTTP/1.1 100 Continue\r\n\r\n', 'UTF-8')) - if(not chunked): - message = self.rfile.read(contentLength) - else: - readChunks() - - def getLookupKey(self, requestline): - global lookup_key_ - kpath = "" - path = "" - print("Request={0} lookup_key={1}".format(requestline, lookup_key_)) - url_part = requestline.split(" ") - if url_part: - if url_part[1].startswith("http"): - path = url_part[1].split("/", 2)[2] - host_, path = path.split("/", 1) - else: - path = url_part[1].split("/", 1)[1] - argsList = [] - keyslist = lookup_key_.split("}") - for keystr in keyslist: - if keystr == '{PATH': - kpath = kpath + path - continue # do not include path in the list of header fields - if keystr == '{HOST': - kpath = kpath + host_ - continue - stringk = keystr.replace("{%", "") - argsList.append(stringk) - KeyList = [] - for argsL in argsList: - if len(argsL) > 0: - val = self.headers.get(argsL) - if val: - field_val, __ = cgi.parse_header(val) - else: - field_val = None - if field_val != None: - KeyList.append(field_val) - key = "".join(KeyList) + kpath - print("lookup key", key, len(key)) - - return key - - def parseRequestline(self, requestline): - testName = None - return testName - - def testMode(self, requestline): - print(requestline) - key = self.parseRequestline(requestline) - - self.send_response(200) - self.send_header('Connection', 'close') - self.end_headers() - - def get_response_code(self, header): - # this could totally go wrong - return int(header.split(' ')[1]) - - def generator(self): - yield 'micro' - yield 'server' - yield 'apache' - yield 'traffic' - yield 'server' - - def send_response(self, code, message=None): - ''' Override `send_response()`'s tacking on of server and date header lines. ''' - self.send_response_only(code, message) - - def createDummyBodywithLength(self, numberOfbytes): - if numberOfbytes == 0: - return None - body = 'a' - while numberOfbytes != 1: - body += 'b' - numberOfbytes -= 1 - return body - - def writeChunkedData(self): - for chunk in self.generator(): - response_string = bytes('%X\r\n%s\r\n' % (len(chunk), chunk), 'UTF-8') - self.wfile.write(response_string) - response_string = bytes('0\r\n\r\n', 'UTF-8') - self.wfile.write(response_string) - - def readChunks(self): - raw_data = b'' - raw_size = self.rfile.readline(65537) - size = str(raw_size, 'UTF-8').rstrip('\r\n') - # print("==========================================>",size) - size = int(size, 16) - while size > 0: - chunk = self.rfile.read(size + 2) # 2 for reading /r/n - raw_data += chunk - raw_size = self.rfile.readline(65537) - size = str(raw_size, 'UTF-8').rstrip('\r\n') - size = int(size, 16) - chunk = self.rfile.readline(65537) # read the extra blank newline \r\n after the last chunk - - def send_header(self, keyword, value): - """Send a MIME header to the headers buffer.""" - if self.request_version != 'HTTP/0.9': - if not hasattr(self, '_headers_buffer'): - self._headers_buffer = [] - self._headers_buffer.append( - ("%s: %s\r\n" % (keyword, value)).encode('UTF-8', 'strict')) # original code used latin-1.. seriously? - - if keyword.lower() == 'connection': - if value.lower() == 'close': - self.close_connection = True - elif value.lower() == 'keep-alive': - self.close_connection = False - - def parse_request(self): - """Parse a request (internal). - - The request should be stored in self.raw_requestline; the results - are in self.command, self.path, self.request_version and - self.headers. Any matching response is in self.response. - - Return True for success, False for failure; on failure, an - error is sent back. - - """ - - global count, test_mode_enabled, G_replay_dict - - self.command = None # set in case of error on the first line - self.request_version = version = self.default_request_version - self.close_connection = True - print("Raw request {0}".format(self.raw_requestline)) - requestline = str(self.raw_requestline, 'UTF-8') - requestline = requestline.rstrip('\r\n') - self.requestline = requestline - - # Examine the headers and look for a Connection directive. - try: - self.headers = http.client.parse_headers(self.rfile, - _class=self.MessageClass) - key = self.getLookupKey(self.requestline) - self.resp = G_replay_dict[key] if key in G_replay_dict else None - - if self.resp is None or 'skipHooks' not in self.resp.getOptions(): - self.server.hook_set.invoke(HookSet.ReadRequestHook, self.headers) - # read message body - if self.headers.get('Content-Length') != None: - bodysize = int(self.headers.get('Content-Length')) - #print("length of the body is",bodysize) - message = self.rfile.read(bodysize) - #print("message body",message) - elif self.headers.get('Transfer-Encoding', "") == 'chunked': - # print(self.headers) - self.readChunks() - except http.client.LineTooLong: - self.send_error( - HTTPStatus.BAD_REQUEST, - "Line too long") - return False - except http.client.HTTPException as err: - self.send_error( - HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE, - "Too many headers", - str(err) - ) - return False - - words = requestline.split() - if len(words) == 3: - command, path, version = words - if version[:5] != 'HTTP/': - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request version (%r)" % version) - return False - try: - base_version_number = version.split('/', 1)[1] - version_number = base_version_number.split(".") - # RFC 2145 section 3.1 says there can be only one "." and - # - major and minor numbers MUST be treated as - # separate integers; - # - HTTP/2.4 is a lower version than HTTP/2.13, which in - # turn is lower than HTTP/12.3; - # - Leading zeros MUST be ignored by recipients. - if len(version_number) != 2: - raise ValueError - version_number = int(version_number[0]), int(version_number[1]) - except (ValueError, IndexError): - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request version (%r)" % version) - return False - if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1": - self.close_connection = False - if version_number >= (2, 0): - self.send_error( - HTTPStatus.HTTP_VERSION_NOT_SUPPORTED, - "Invalid HTTP Version (%s)" % base_version_number) - return False - elif len(words) == 2: - command, path = words - self.close_connection = True - if command != 'GET': - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad HTTP/0.9 request type (%r)" % command) - return False - elif not words: - count += 1 - print("bla bla on 157 {0} => {1}".format(count, self.close_connection)) - return False - else: - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request syntax (%r)" % requestline) - return False - self.command, self.path, self.request_version = command, path, version - - conntype = self.headers.get('Connection', "") - if conntype.lower() == 'close': - self.close_connection = True - elif (conntype.lower() == 'keep-alive' and - self.protocol_version >= "HTTP/1.1"): - self.close_connection = False - - return True - - def do_GET(self): - global G_replay_dict, test_mode_enabled - # skip time delay for the autest ready check - if test_mode_enabled and self.requestline != "GET /ruok HTTP/1.1": - time.sleep(time_delay) - - try: - response_string = None - chunkedResponse = False - if self.resp is None: - self.send_response(404) - self.send_header('Server', 'MicroServer') - self.send_header('Connection', 'close') - self.end_headers() - return - - else: - headers = self.resp.getHeaders().split('\r\n') - - # set status codes - status_code = self.get_response_code(headers[0]) - self.send_response(status_code) - - # set headers - for header in headers[1:]: # skip first one b/c it's response code - if header == '': - continue - elif 'Content-Length' in header: - if 'Access-Control' in header: # skipping Access-Control-Allow-Credentials, Access-Control-Allow-Origin, Content-Length - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - self.send_header(header_field, header_field_val) - continue - lengthSTR = header.split(':')[1] - length = lengthSTR.strip(' ') - if test_mode_enabled: # the length of the body is given priority in test mode rather than the value in Content-Length. But in replay mode Content-Length gets the priority - if not (self.resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester - self.send_header('Content-Length', str(length)) - else: - self.send_header('Content-Length', str(length)) - response_string = self.createDummyBodywithLength(int(length)) - continue - if 'Transfer-Encoding' in header: - self.send_header('Transfer-Encoding', 'Chunked') - response_string = '%X\r\n%s\r\n' % (len('ats'), 'ats') - chunkedResponse = True - continue - - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - self.send_header(header_field, header_field_val) - # End for - if test_mode_enabled: - if self.resp.getBody(): - length = len(bytes(self.resp.getBody(), 'UTF-8')) - response_string = self.resp.getBody() - self.send_header('Content-Length', str(length)) - self.end_headers() - - if (chunkedResponse): - self.writeChunkedData() - elif response_string != None and response_string != '': - self.wfile.write(bytes(response_string, 'UTF-8')) - except: - e = sys.exc_info() - print("Error", e, self.headers) - self.send_response(400) - self.send_header('Connection', 'close') - self.end_headers() - - def do_HEAD(self): - if self.resp is None: - self.send_response(404) - self.send_header('Connection', 'close') - self.end_headers() - return - - headers = self.resp.getHeaders().split('\r\n') - - # set status codes - status_code = self.get_response_code(headers[0]) - self.send_response(status_code) - - # set headers - for header in headers[1:]: # skip first one b/c it's response code - if header == '': - continue - elif 'Content-Length' in header: - self.send_header('Content-Length', '0') - continue - - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - self.send_header(header_field, header_field_val) - - self.end_headers() - - def do_POST(self): - response_string = None - chunkedResponse = False - global test_mode_enabled - try: - - if self.resp is None: - self.send_response(404) - self.send_header('Connection', 'close') - self.end_headers() - return - else: - resp_headers = self.resp.getHeaders().split('\r\n') - # set status codes - status_code = self.get_response_code(resp_headers[0]) - #print("response code",status_code) - self.send_response(status_code) - #print("reposen is ",resp_headers) - # set headers - for header in resp_headers[1:]: # skip first one b/c it's response code - - if header == '': - continue - elif 'Content-Length' in header: - if 'Access-Control' in header: # skipping Access-Control-Allow-Credentials, Access-Control-Allow-Origin, Content-Length - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - self.send_header(header_field, header_field_val) - continue - - lengthSTR = header.split(':')[1] - length = lengthSTR.strip(' ') - if test_mode_enabled: # the length of the body is given priority in test mode rather than the value in Content-Length. Otherwise, Content-Length gets the priority - if not (self.resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester - self.send_header('Content-Length', str(length)) - else: - self.send_header('Content-Length', str(length)) - response_string = self.createDummyBodywithLength(int(length)) - continue - if 'Transfer-Encoding' in header: - self.send_header('Transfer-Encoding', 'Chunked') - response_string = '%X\r\n%s\r\n' % (len('microserver'), 'microserver') - chunkedResponse = True - continue - - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - #print("{0} === >{1}".format(header_field, header_field_val)) - self.send_header(header_field, header_field_val) - # End for loop - if test_mode_enabled: - if self.resp.getBody(): - length = len(bytes(self.resp.getBody(), 'UTF-8')) - response_string = self.resp.getBody() - self.send_header('Content-Length', str(length)) - self.end_headers() - - if (chunkedResponse): - self.writeChunkedData() - elif response_string != None and response_string != '': - self.wfile.write(bytes(response_string, 'UTF-8')) - except: - e = sys.exc_info() - print("Error", e, self.headers) - self.send_response(400) - self.send_header('Connection', 'close') - self.end_headers() - - -def populate_global_replay_dictionary(sessions): - ''' Populates the global dictionary of {uuid (string): reponse (Response object)} ''' - global G_replay_dict - for session in sessions: - for txn in session.getTransactionIter(): - G_replay_dict[txn._uuid] = txn.getResponse() - - print("size", len(G_replay_dict)) - -# tests will add responses to the dictionary where key is the testname - - -def addResponseHeader(key, response_header): - G_replay_dict[key] = response_header - - -def _path(exists, arg): - path = os.path.abspath(arg) - if not os.path.exists(path) and exists: - msg = '"{0}" is not a valid path'.format(path) - raise argparse.ArgumentTypeError(msg) - return path - - -def _bool(arg): - - opt_true_values = set(['y', 'yes', 'true', 't', '1', 'on', 'all']) - opt_false_values = set(['n', 'no', 'false', 'f', '0', 'off', 'none', None]) - - tmp = arg.lower() if arg is not None else None - if tmp in opt_true_values: - return True - elif tmp in opt_false_values: - return False - else: - msg = 'Invalid value Boolean value : "{0}"\n Valid options are {1}'.format(arg, - opt_true_values | opt_false_values) - raise ValueError(msg) - - -def _argparse_bool(arg): - try: - return _bool(arg) - except ValueError as ve: - raise argparse.ArgumentTypeError(ve) - - -def main(): - global test_mode_enabled - parser = argparse.ArgumentParser() - - parser.add_argument("--data-dir", "-d", - type=lambda x: _path(True, x), - required=True, - help="Directory with data file" - ) - - parser.add_argument("--ip_address", "-ip", - type=str, - default='INADDR_LOOPBACK', - help="IP address of the interface to serve on" - ) - - parser.add_argument("--port", "-p", - type=int, - default=SERVER_PORT, - help="Port to use") - - parser.add_argument("--delay", "-dy", - type=float, - default=SERVER_DELAY, - help="Response delay") - - parser.add_argument("--timeout", "-t", - type=float, - default=None, - help="socket time out in seconds") - - parser.add_argument('-V', '--version', action='version', version='%(prog)s {0}'.format(__version__)) - - parser.add_argument("--mode", "-m", - type=str, - default="test", - help="Mode of operation") - parser.add_argument("--ssl", "-ssl", - type=str, - default="False", - help="SSL port") - parser.add_argument("--key", "-k", - type=str, - default="ssl/server.pem", - help="key for ssl connnection") - parser.add_argument("--cert", "-cert", - type=str, - default="ssl/server.crt", - help="certificate") - parser.add_argument("--clientCA", - type=str, - default="", - help="CA for client certificates") - parser.add_argument("--clientverify", "-cverify", - type=_argparse_bool, - default=False, - help="verify client cert") - parser.add_argument("--load", - dest='load', - type=str, - default='', - help="A file which will install observers on hooks") - parser.add_argument("--lookupkey", - type=str, - default="{PATH}", - help="format string used as a key for response lookup: \ - example: \"{%%Host}{%%Server}{PATH}\", \"{HOST}{PATH}\", \"{PATH}\"\ - All the args preceded by %% are header fields in the request\ - The only two acceptable arguments which are not header fields are : fqdn (represented by HOST) and the url path (represented by PATH) in a request line.\ - Example: given a client request as << GET /some/resource/location HTTP/1.1\nHost: hahaha.com\n\n >>, if the user wishes the host field and the path to be used for the response lookup\ - then the required format will be {%%Host}{PATH}") - - args = parser.parse_args() - global time_delay - time_delay = args.delay - - # set up global dictionary of {uuid (string): response (Response object)} - s = sv.SessionValidator(args.data_dir) - populate_global_replay_dictionary(s.getSessionIter()) - print("Dropped {0} sessions for being malformed".format(len(s.getBadSessionList()))) - - # start server - try: - socket_timeout = args.timeout - test_mode_enabled = args.mode == "test" - global lookup_key_ - lookup_key_ = args.lookupkey - MyHandler.protocol_version = HTTP_VERSION - - if IPConstants.isIPv6(args.ip_address): - print("Server running on IPv6") - HTTPServer.address_family = socket.AF_INET6 - - if args.ssl == "True" or args.ssl == "true": - server = SSLServer((IPConstants.getIP(args.ip_address), args.port), MyHandler, args) - else: - server = ThreadingServer((IPConstants.getIP(args.ip_address), args.port), MyHandler, args) - - server.timeout = 5 - print("Started server on port {0}".format(args.port)) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.daemon = True - server_thread.start() - - try: - while 1: - time.sleep(1) - sys.stderr.flush() - sys.stdout.flush() - except KeyboardInterrupt: - print("\n=== ^C received, shutting down microservers ===") - server.shutdown() - server_thread.join() - - except KeyboardInterrupt: - print("\n=== ^C received, shutting down httpserver ===") - server.socket.close() - # s_server.socket.close() - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/tests/tools/microServer/ssl/server.crt b/tests/tools/microserver/ssl/server.crt similarity index 100% rename from tests/tools/microServer/ssl/server.crt rename to tests/tools/microserver/ssl/server.crt diff --git a/tests/tools/microServer/ssl/server.pem b/tests/tools/microserver/ssl/server.pem similarity index 100% rename from tests/tools/microServer/ssl/server.pem rename to tests/tools/microserver/ssl/server.pem diff --git a/tests/tools/sessionvalidation/__init__.py b/tests/tools/sessionvalidation/__init__.py deleted file mode 100644 index bcbf6855425..00000000000 --- a/tests/tools/sessionvalidation/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/tools/sessionvalidation/badsession.py b/tests/tools/sessionvalidation/badsession.py deleted file mode 100644 index 7f55de276f0..00000000000 --- a/tests/tools/sessionvalidation/badsession.py +++ /dev/null @@ -1,35 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class BadSession(object): - ''' - Session encapsulates a single BAD user session. Bad meaning that for some reason the session is invalid. - - _filename is the filename of the bad JSON session - _reason is a string with some kind of explanation on why the session was bad - ''' - - def __repr__(self): - return "".format( - self._filename, self._reason - ) - - def __init__(self, filename, reason): - self._filename = filename - self._reason = reason diff --git a/tests/tools/sessionvalidation/request.py b/tests/tools/sessionvalidation/request.py deleted file mode 100644 index 39598d7962b..00000000000 --- a/tests/tools/sessionvalidation/request.py +++ /dev/null @@ -1,48 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import hashlib - - -class Request(object): - ''' Request encapsulates a single request from the UA ''' - - def getTimestamp(self): - return self._timestamp - - def getHeaders(self): - return self._headers - - def getBody(self): - return self._body - - def getHeaderMD5(self): - ''' Returns the MD5 hash of the headers - - This is used to do a unique mapping to a request/response transaction ''' - return hashlib.md5(self._headers.encode()).hexdigest() - - def __repr__(self): - # return str(self._timestamp) - return "".format( - str(self._timestamp), str(self._headers), str(self._body) - ) - - def __init__(self, timestamp, headers, body): - self._timestamp = timestamp - self._headers = headers - self._body = body diff --git a/tests/tools/sessionvalidation/response.py b/tests/tools/sessionvalidation/response.py deleted file mode 100644 index faa5f97255f..00000000000 --- a/tests/tools/sessionvalidation/response.py +++ /dev/null @@ -1,49 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - - -class Response(object): - ''' Response encapsulates a single request from the UA ''' - - def getTimestamp(self): - return self._timestamp - - def getHeaders(self): - return self._headers - - def getBody(self): - return self._body - - def getOptions(self): - return self._options - - def __repr__(self): - return "".format( - self._timestamp, self._headers, self._body, self._options - ) - - def __init__(self, timestamp, headers, body, options_string): - self._timestamp = timestamp - self._headers = headers - self._body = body - if options_string: - self._options = re.compile(r'\s*,\s*').split(options_string) - else: - self._options = list() diff --git a/tests/tools/sessionvalidation/session.py b/tests/tools/sessionvalidation/session.py deleted file mode 100644 index e8bb0e2e463..00000000000 --- a/tests/tools/sessionvalidation/session.py +++ /dev/null @@ -1,45 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sessionvalidation.transaction as transaction - - -class Session(object): - ''' Session encapsulates a single user session ''' - - def getTransactionList(self): - ''' Returns a list of transaction objects ''' - return self._transaction_list - - def getTransactionIter(self): - ''' Returns an iterator of transaction objects ''' - return iter(self._transaction_list) - - def returnFirstTransaction(self): - return self._transaction_list[0] - - def __repr__(self): - return "".format( - self._filename, self._version, self._timestamp, self._encoding, repr(self._transaction_list) - ) - - def __init__(self, filename, version, timestamp, transaction_list, encoding=None): - self._filename = filename - self._version = version - self._timestamp = timestamp - self._encoding = encoding - self._transaction_list = transaction_list diff --git a/tests/tools/sessionvalidation/sessionvalidation.py b/tests/tools/sessionvalidation/sessionvalidation.py deleted file mode 100644 index 7ff97e6facf..00000000000 --- a/tests/tools/sessionvalidation/sessionvalidation.py +++ /dev/null @@ -1,259 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import os - -import sessionvalidation.session as session -import sessionvalidation.transaction as transaction -import sessionvalidation.request as request -import sessionvalidation.response as response - -# valid_HTTP_request_methods = ['GET', 'POST', 'HEAD'] -# custom_HTTP_request_methods = ['PULL'] # transaction monitor plugin for ATS may have custom methods -allowed_HTTP_request_methods = ['GET', 'POST', 'HEAD', 'PULL'] -G_CUSTOM_METHODS = False -G_VERBOSE_LOG = True - - -def _verbose_print(msg, verbose_on=False): - ''' Print msg if verbose_on is set to True or G_VERBOSE_LOG is set to True''' - if verbose_on or G_VERBOSE_LOG: - print(msg) - - -class SessionValidator(object): - ''' - SessionValidator parses, validates, and exports an API for a given set of JSON sessions generated from Apache Traffic Server - - SessionValidator is initialized with a path to a directory of JSON sessions. It then automatically parses and validates all the - session in the directory. After initialization, the user may use the provided API - - TODO : - Provide a list of guaranteed fields for each type of object (ie a Transaction has a request and a response, a request has ...) - ''' - - def parse(self): - ''' - Constructs Session objects from JSON files on disk and stores objects into _sessions - - All sessions missing required fields (ie. a session timestamp, a response for every request, etc) are - dropped and the filename is stored inside _bad_sessions - ''' - - log_filenames = [os.path.join(self._json_log_dir, f) for f in os.listdir( - self._json_log_dir) if os.path.isfile(os.path.join(self._json_log_dir, f))] - - for fname in log_filenames: - with open(fname) as f: - # first attempt to load the JSON - try: - sesh = json.load(f) - except: - self._bad_sessions.append(fname) - _verbose_print("Warning: JSON parse error on file={0}".format(fname)) - print("Warning: JSON parse error on file={0}".format(fname)) - continue - - # then attempt to extract all the required fields from the JSON - try: - session_timestamp = sesh['timestamp'] - session_version = sesh['version'] - session_txns = list() - for txn in sesh['txns']: - # create transaction Request object - txn_request = txn['request'] - - txn_request_body = '' - if 'body' in txn_request: - txn_request_body = txn_request['body'] - txn_request_obj = request.Request(txn_request['timestamp'], txn_request['headers'], txn_request_body) - # Create transaction Response object - txn_response = txn['response'] - txn_response_body = '' - if 'body' in txn_response: - txn_response_body = txn_response['body'] - txn_response_obj = response.Response(txn_response['timestamp'], txn_response['headers'], txn_response_body, - txn_response.get('options')) - - # create Transaction object - txn_obj = transaction.Transaction(txn_request_obj, txn_response_obj, txn['uuid']) - session_txns.append(txn_obj) - session_obj = session.Session(fname, session_version, session_timestamp, session_txns) - - except KeyError as e: - self._bad_sessions.append(fname) - print("Warning: parse error on key={0} for file={1}".format(e, fname)) - _verbose_print("Warning: parse error on key={0} for file={1}".format(e, fname)) - continue - - self._sessions.append(session_obj) - - def validate(self): - ''' Prunes out all the invalid Sessions in _sessions ''' - - good_sessions = list() - - for sesh in self._sessions: - if SessionValidator.validateSingleSession(sesh): - good_sessions.append(sesh) - else: - self._bad_sessions.append(sesh._filename) - - self._sessions = good_sessions - - @staticmethod - def validateSingleSession(sesh): - ''' Takes in a single Session object as input, returns whether or not the Session is valid ''' - - retval = True - - try: - # first validate fields - if not sesh._filename: - _verbose_print("bad session filename") - retval = False - elif not sesh._version: - _verbose_print("bad session version") - retval = False - elif float(sesh._timestamp) <= 0: - _verbose_print("bad session timestamp") - retval = False - elif not bool(sesh.getTransactionList()): - _verbose_print("session has no transaction list") - retval = False - - # validate Transactions now - for txn in sesh.getTransactionIter(): - if not SessionValidator.validateSingleTransaction(txn): - retval = False - - except ValueError as e: - _verbose_print("most likely an invalid session timestamp") - retval = False - - return retval - - @staticmethod - def validateSingleTransaction(txn): - ''' Takes in a single Transaction object as input, and returns whether or not the Transaction is valid ''' - - txn_req = txn.getRequest() - txn_resp = txn.getResponse() - retval = True - - #valid_HTTP_request_methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'OPTIONS', 'CONNECT', 'PATCH'] - # we can later uncomment the previous line to support more HTTP methods - valid_HTTP_versions = ['HTTP/1.0', 'HTTP/1.1', 'HTTP/2.0'] - - try: - # validate request first - if not txn_req: - _verbose_print("no transaction request") - retval = False - elif txn_req.getBody() == None: - _verbose_print("transaction body is set to None") - retval = False - elif float(txn_req.getTimestamp()) <= 0: - _verbose_print("invalid transaction request timestamp") - retval = False - elif txn_req.getHeaders().split()[0] not in allowed_HTTP_request_methods: - _verbose_print("invalid HTTP method for transaction {0}".format(txn_req.getHeaders().split()[0])) - retval = False - elif not txn_req.getHeaders().endswith("\r\n\r\n"): - _verbose_print("transaction request headers didn't end with \\r\\n\\r\\n") - retval = False - elif txn_req.getHeaders().split()[2] not in valid_HTTP_versions: - _verbose_print("invalid HTTP version in request") - retval = False - - # if the Host header is not present and vaild we reject this transaction - found_host = False - for header in txn_req.getHeaders().split('\r\n'): - split_header = header.split(' ') - if split_header[0] == 'Host:': - found_host = True - host_header_no_space = len(split_header) == 1 - host_header_with_space = len(split_header) == 2 and split_header[1] == '' - if host_header_no_space or host_header_with_space: - found_host = False - if not found_host: - print("missing host", txn_req) - _verbose_print("transaction request Host header doesn't have specified host") - retval = False - - # now validate response - if not txn_resp: - _verbose_print("no transaction response") - retval = False - elif txn_resp.getBody() == None: - _verbose_print("transaction response body set to None") - retval = False - elif float(txn_resp.getTimestamp()) <= 0: - _verbose_print("invalid transaction response timestamp") - retval = False - elif txn_resp.getHeaders().split()[0] not in valid_HTTP_versions: - _verbose_print("invalid HTTP response header") - retval = False - elif not txn_resp.getHeaders().endswith("\r\n\r\n"): - _verbose_print("transaction response headers didn't end with \\r\\n\\r\\n") - retval = False - - # if any of the 3xx responses have bodies, then the must reject this transaction, since 3xx - # errors by definition can't have bodies - response_line = txn_resp.getHeaders().split('\r\n')[0] - response_code = response_line.split(' ')[1] - if response_code.startswith('3') and txn_resp.getBody(): - _verbose_print("transaction response was 3xx and had a body") - retval = False - - except ValueError as e: - _verbose_print("most likely an invalid transaction timestamp") - retval = False - - except IndexError as e: - _verbose_print("most likely a bad transaction header") - retval = False - - return retval - - def getSessionList(self): - ''' Returns the list of Session objects ''' - return self._sessions - - def getSessionIter(self): - ''' Returns an iterator of the Session objects ''' - return iter(self._sessions) - - def getBadSessionList(self): - ''' Returns a list of bad session filenames (list of strings) ''' - return self._bad_sessions - - def getBadSessionListIter(self): - ''' Returns an iterator of bad session filenames (iterator of strings) ''' - return iter(self._bad_sessions) - - def __init__(self, json_log_dir, allow_custom=False): - global valid_HTTP_request_methods - global G_CUSTOM_METHODS - G_CUSTOM_METHODS = allow_custom - self._json_log_dir = json_log_dir - self._bad_sessions = list() # list of filenames - self._sessions = list() # list of _good_ session objects - - self.parse() - self.validate() diff --git a/tests/tools/sessionvalidation/transaction.py b/tests/tools/sessionvalidation/transaction.py deleted file mode 100644 index 19950abea69..00000000000 --- a/tests/tools/sessionvalidation/transaction.py +++ /dev/null @@ -1,40 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sessionvalidation.request as request -import sessionvalidation.response as response - - -class Transaction(object): - ''' Tranaction encapsulates a single UA transaction ''' - - def getRequest(self): - return self._request - - def getResponse(self): - return self._response - - def __repr__(self): - return "".format( - self._uuid, self._request, self._response - ) - - def __init__(self, request, response, uuid): - self._request = request - self._response = response - self._uuid = uuid diff --git a/tests/tools/traffic-replay/Config.py b/tests/tools/traffic-replay/Config.py deleted file mode 100644 index 48d3fc3919d..00000000000 --- a/tests/tools/traffic-replay/Config.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# SSL config -ca_certs = None -keyfile = None - -# Proxy config -proxy_host = "127.0.0.1" -proxy_ssl_port = 443 -proxy_nonssl_port = 8080 - -# process and thread config -nProcess = 4 -nThread = 4 - -# colorize output -colorize = True diff --git a/tests/tools/traffic-replay/NonSSL.py b/tests/tools/traffic-replay/NonSSL.py deleted file mode 100644 index 3d6b85b090d..00000000000 --- a/tests/tools/traffic-replay/NonSSL.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket -import requests -import os -from threading import Thread -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import extractHeader -import mainProcess -import json -import gzip -bSTOP = False - - -def createDummyBodywithLength(numberOfbytes): - if numberOfbytes <= 0: - return None - body = 'a' - while numberOfbytes != 1: - body += 'b' - numberOfbytes -= 1 - return body - - -def handleResponse(response, *args, **kwargs): - print(response.status_code) - # resp=args[0] - #expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2) - #expected_output = (int(expected_output_split[1]), str( expected_output_split[2])) - #r = result.Result(session_filename, expected_output[0], response.status_code) - # print(r.getResultString(colorize=True)) -# make sure len of the message body is greater than length - - -def gen(): - yield 'pforpersia,champaignurbana'.encode('utf-8') - yield 'there'.encode('utf-8') - - -def txn_replay(session_filename, txn, proxy, result_queue, request_session): - """ Replays a single transaction - :param request_session: has to be a valid requests session""" - req = txn.getRequest() - resp = txn.getResponse() - - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier - if 'body' in txn_req_headers_dict: - del txn_req_headers_dict['body'] - - #print("Replaying session") - try: - # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), - # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), - # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me - method = extractHeader.extract_txn_req_method(txn_req_headers) - response = None - body = None - content = None - if 'Transfer-Encoding' in txn_req_headers_dict: - # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header - # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this - # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? - del txn_req_headers_dict['Host'] - if 'Content-Length' in txn_req_headers_dict: - #print("ewww !") - del txn_req_headers_dict['Content-Length'] - body = gen() - if 'Content-Length' in txn_req_headers_dict: - nBytes = int(txn_req_headers_dict['Content-Length']) - body = createDummyBodywithLength(nBytes) - #print("request session is",id(request_session)) - if method == 'GET': - r1 = request_session.request('GET', 'http://'+extractHeader.extract_host(txn_req_headers)+extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, data=body) - responseHeaders = r1.headers - responseContent = r1.content # byte array - - #print("len: {0} received {1}".format(responseHeaders['Content-Length'], responseContent)) - - elif method == 'POST': - r1 = request_session.request('POST', 'http://'+extractHeader.extract_host(txn_req_headers)+extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, data=body) - responseHeaders = r1.headers - responseContent = r1.content - - #print("len: {0} received {1}".format(responseHeaders['Content-Length'], responseContent)) - elif method == 'HEAD': - r1 = request_session.request('HEAD', 'http://'+extractHeader.extract_host(txn_req_headers)+extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, data=body) - responseHeaders = r1.headers - responseContent = r1.content - else: # EXPERIMENTAL - r1 = request_session.request(method, 'http://'+extractHeader.extract_host(txn_req_headers)+extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, data=body) - responseHeaders = r1.headers - responseContent = r1.content - - #gzip_file = gzip.GzipFile(fileobj=responseContent) - #shutil.copyfileobj(gzip_file, f) - - expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) - # print("------------EXPECTED-----------") - # print(expected) - # print("------------RESP--------------") - # print(responseHeaders) - # print() - - if mainProcess.verbose: - expected_output_split = resp.getHeaders().split('\r\n')[0].split(' ', 2) - expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) - r = result.Result(session_filename, expected_output[0], r1.status_code, responseContent) - b_res, res = r.getResult(responseHeaders, expected, colorize=True) - print(res) - - if not b_res: - print("Received response") - print(responseHeaders) - print("Expected response") - print(expected) - # result_queue.put(r) - except UnicodeEncodeError as e: - # these unicode errors are due to the interaction between Requests and our wiretrace data. - # TODO fix - print("UnicodeEncodeError exception") - - except requests.exceptions.ContentDecodingError as e: - print("ContentDecodingError", e) - except: - e = sys.exc_info() - print("ERROR in NonSSLReplay: ", e, response, session_filename) - - -def session_replay(input, proxy, result_queue): - global bSTOP - ''' Replay all transactions in session - - This entire session will be replayed in one requests.Session (so one socket / TCP connection)''' - # if timing_control: - # time.sleep(float(session._timestamp)) # allow other threads to run - while bSTOP == False: - for session in iter(input.get, 'STOP'): - # print(bSTOP) - if session == 'STOP': - print("Queue is empty") - bSTOP = True - break - with requests.Session() as request_session: - request_session.proxies = proxy - for txn in session.getTransactionIter(): - try: - txn_replay(session._filename, txn, proxy, result_queue, request_session) - except: - e = sys.exc_info() - print("ERROR in replaying: ", e, txn.getRequest().getHeaders()) - bSTOP = True - #print("Queue is empty") - input.put('STOP') - break - - -def client_replay(input, proxy, result_queue, nThread): - Threads = [] - for i in range(nThread): - t = Thread(target=session_replay, args=[input, proxy, result_queue]) - t.start() - Threads.append(t) - - for t1 in Threads: - t1.join() diff --git a/tests/tools/traffic-replay/RandomReplay.py b/tests/tools/traffic-replay/RandomReplay.py deleted file mode 100644 index f6bf8691ff4..00000000000 --- a/tests/tools/traffic-replay/RandomReplay.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket -import requests -import os -from threading import Thread -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -from collections import deque -import collections -import lib.result as result -import extractHeader -import mainProcess -import json -import gzip -import NonSSL -import SSLReplay -import h2Replay -import itertools -import random -bSTOP = False - - -def session_replay(input, proxy, result_queue): - global bSTOP - ''' Replay all transactions in session - - This entire session will be replayed in one requests.Session (so one socket / TCP connection)''' - # if timing_control: - # time.sleep(float(session._timestamp)) # allow other threads to run - while bSTOP == False: - for session in iter(input.get, 'STOP'): - # print(bSTOP) - if session == 'STOP': - print("Queue is empty") - bSTOP = True - break - with requests.Session() as request_session: - request_session.proxies = proxy - for txn in session.getTransactionIter(): - type = random.randint(1, 1000) - try: - if type % 3 == 0: - NonSSL.txn_replay(session._filename, txn, proxy, result_queue, request_session) - elif type % 3 == 1: - SSLReplay.txn_replay(session._filename, txn, proxy, result_queue, request_session) - elif type % 3 == 2: - h2Replay.txn_replay(session._filename, txn, proxy, result_queue, request_session) - except: - e = sys.exc_info() - print("ERROR in replaying: ", e, txn.getRequest().getHeaders()) - bSTOP = True - #print("Queue is empty") - input.put('STOP') - break - - -def client_replay(input, proxy, result_queue, nThread): - Threads = [] - for i in range(nThread): - - t2 = Thread(target=SSLReplay.session_replay, args=[input, proxy, result_queue]) - t = Thread(target=NonSSL.session_replay, args=[input, proxy, result_queue]) - t1 = Thread(target=h2Replay.session_replay, args=[input, proxy, result_queue]) - t2.start() - t.start() - t1.start() - Threads.append(t) - Threads.append(t2) - Threads.append(t1) - - for t1 in Threads: - t1.join() diff --git a/tests/tools/traffic-replay/SSLReplay.py b/tests/tools/traffic-replay/SSLReplay.py deleted file mode 100644 index c75b6a5b014..00000000000 --- a/tests/tools/traffic-replay/SSLReplay.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import http.client -import socket -import ssl -import pprint -# import gevent -import requests -import os -#import threading -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import extractHeader -# from gevent import monkey, sleep -from threading import Thread -import mainProcess -import json -import extractHeader -import time -import Config -bSTOP = False - - -class ProxyHTTPSConnection(http.client.HTTPSConnection): - "This class allows communication via SSL." - - default_port = http.client.HTTPS_PORT - - # XXX Should key_file and cert_file be deprecated in favour of context? - - def __init__(self, host, port=None, key_file=None, cert_file=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, *, context=None, - check_hostname=None, server_name=None): - # http.client.HTTPSConnection.__init__(self) - super().__init__(host, port, key_file, cert_file, timeout, source_address, context=context, check_hostname=check_hostname) - ''' - self.key_file = key_file - self.cert_file = cert_file - if context is None: - context = ssl._create_default_https_context() - will_verify = context.verify_mode != ssl.CERT_NONE - if check_hostname is None: - check_hostname = context.check_hostname - if check_hostname and not will_verify: - raise ValueError("check_hostname needs a SSL context with " - "either CERT_OPTIONAL or CERT_REQUIRED") - if key_file or cert_file: - context.load_cert_chain(cert_file, key_file) - self._context = context - self._check_hostname = check_hostname - ''' - self.server_name = server_name - - def connect(self): - "Connect to a host on a given (SSL) port." - http.client.HTTPConnection.connect(self) - - if self._tunnel_host: - server_hostname = self._tunnel_host - else: - server_hostname = self.server_name - self.sock = self._context.wrap_socket(self.sock, - do_handshake_on_connect=True, - server_side=False, - server_hostname=server_hostname) - if not self._context.check_hostname and self._check_hostname: - try: - ssl.match_hostname(self.sock.getpeercert(), server_hostname) - except Exception: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - raise - - -def txn_replay(session_filename, txn, proxy, result_queue, request_session): - """ Replays a single transaction - :param request_session: has to be a valid requests session""" - req = txn.getRequest() - resp = txn.getResponse() - responseDict = {} - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier - if 'body' in txn_req_headers_dict: - del txn_req_headers_dict['body'] - - #print("Replaying session") - try: - # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), - # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), - # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me - method = extractHeader.extract_txn_req_method(txn_req_headers) - response = None - body = None - content = None - if 'Transfer-Encoding' in txn_req_headers_dict: - # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header - # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this - # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? - del txn_req_headers_dict['Host'] - if 'Content-Length' in txn_req_headers_dict: - #print("ewww !") - del txn_req_headers_dict['Content-Length'] - body = gen() - if 'Content-Length' in txn_req_headers_dict: - nBytes = int(txn_req_headers_dict['Content-Length']) - body = createDummyBodywithLength(nBytes) - #print("request session is",id(request_session)) - - # NOTE: request_session here is actually python's HTTPSConnection, which is different from that in NonSSL, which uses the requests library -_- - if method == 'GET': - request_session.request('GET', extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=body) - r1 = request_session.getresponse() - responseContent = r1.read() # byte array - - elif method == 'POST': - request_session.request('POST', extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=body) - r1 = request_session.getresponse() - responseContent = r1.read() - - elif method == 'HEAD': - request_session.request('HEAD', extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=body) - r1 = request_session.getresponse() - responseContent = r1.read() - else: # EXPERIMENTAL - request_session.request(method, extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=body) - r1 = request_session.getresponse() - responseContent = r1.read() - - responseHeaders = extractHeader.responseHeaderTuple_to_dict(r1.getheaders()) - expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) - # print("------------EXPECTED-----------") - # print(expected) - # print("------------RESP--------------") - # print(responseHeaders) - # print() - if mainProcess.verbose: - expected_output_split = resp.getHeaders().split('\r\n')[0].split(' ', 2) - expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) - r = result.Result(session_filename, expected_output[0], r1.status, responseContent) - b_res, res = r.getResult(responseHeaders, expected, colorize=True) - print(res) - - if not res: - print("Received response") - print(responseHeaders) - print("Expected response") - print(expected) - # result_queue.put(r) - except UnicodeEncodeError as e: - # these unicode errors are due to the interaction between Requests and our wiretrace data. - # TODO fix - print("UnicodeEncodeError exception") - - except requests.exceptions.ContentDecodingError as e: - print("ContentDecodingError", e) - except: - e = sys.exc_info() - print("ERROR in SSLReplay: ", e, response, session_filename) - - -def client_replay(input, proxy, result_queue, nThread): - Threads = [] - for i in range(nThread): - t = Thread(target=session_replay, args=[input, proxy, result_queue]) - t.start() - Threads.append(t) - - for t1 in Threads: - t1.join() - - -def session_replay(input, proxy, result_queue): - ''' Replay all transactions in session - - This entire session will be replayed in one requests.Session (so one socket / TCP connection)''' - # if timing_control: - # time.sleep(float(session._timestamp)) # allow other threads to run - global bSTOP - sslSocks = [] - while bSTOP == False: - for session in iter(input.get, 'STOP'): - txn = session.returnFirstTransaction() - req = txn.getRequest() - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - sc = ssl.SSLContext(protocol=ssl.PROTOCOL_SSLv23) - sc.load_cert_chain(Config.ca_certs, keyfile=Config.keyfile) - conn = ProxyHTTPSConnection(Config.proxy_host, Config.proxy_ssl_port, cert_file=Config.ca_certs, - key_file=Config.keyfile, context=sc, server_name=txn_req_headers_dict['Host']) - for txn in session.getTransactionIter(): - try: - # print(txn._uuid) - txn_replay(session._filename, txn, proxy, result_queue, conn) - except: - e = sys.exc_info() - print("ERROR in replaying: ", e, txn.getRequest().getHeaders()) - #sslSocket.bStop = False - - bSTOP = True - print("stopping now") - input.put('STOP') - break - - # time.sleep(0.5) - for sslSock in sslSocks: - sslSock.ssl_sock.close() diff --git a/tests/tools/traffic-replay/Scheduler.py b/tests/tools/traffic-replay/Scheduler.py deleted file mode 100644 index a1a4353d49a..00000000000 --- a/tests/tools/traffic-replay/Scheduler.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time -import random -import json -from multiprocessing import Process, Queue, current_process -import sessionvalidation.sessionvalidation as sv -import WorkerTask -import time - - -def LaunchWorkers(path, nProcess, proxy, replay_type, nThread): - ms1 = time.time() - s = sv.SessionValidator(path, allow_custom=True) - sessions = s.getSessionList() - sessions.sort(key=lambda session: session._timestamp) - Processes = [] - Qsize = 25000 # int (1.5 * len(sessions)/(nProcess)) - QList = [Queue(Qsize) for i in range(nProcess)] - print("Dropped {0} sessions for being malformed. Number of correct sessions {1}".format( - len(s.getBadSessionList()), len(sessions))) - print(range(nProcess)) - OutputQ = Queue() - #======================================== Pre-load queues - for session in sessions: - if replay_type == 'mixed': - if nProcess < 2: - raise ValueError("For mixed replay type, there should be at least 2 processes.") - # odd Qs for SSL sessions, even Qs for nonSSL sessions - num = random.randint(0, nProcess - 1) - - # get the first transaction in each session, which is indictive if session is over SSL or not - if "https" in session.returnFirstTransaction().getRequest().getHeaders(): - # spin until we get an odd number - while num & 1 == 0: - num = random.randint(0, nProcess - 1) - else: - # nonSSL sessions get put here into even Qs - while num & 1 == 1: - num = random.randint(0, nProcess - 1) - - QList[num].put(session) - else: - # if nProcess == 1: - # QList[0].put(session) - # else: - QList[random.randint(0, nProcess - 1)].put(session) - # if QList[0].qsize() > 10 : - # break - #=============================================== Launch Processes - # for i in range(nProcess): - # QList[i].put('STOP') - for i in range(nProcess): - QList[i].put('STOP') - - if replay_type == 'mixed': - if i & 1: # odd/SSL - p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, 'ssl', nThread]) - else: # even/nonSSL - p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, 'nossl', nThread]) - else: - p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, replay_type, nThread]) - - p.daemon = False - Processes.append(p) - p.start() - - for p in Processes: - p.join() - ms2 = time.time() - print("OK enough, it is time to exit, running time in seconds", (ms2 - ms1)) diff --git a/tests/tools/traffic-replay/WorkerTask.py b/tests/tools/traffic-replay/WorkerTask.py deleted file mode 100644 index 839e696665a..00000000000 --- a/tests/tools/traffic-replay/WorkerTask.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket -import requests -import os -#import threading -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import extractHeader -import NonSSL -import SSLReplay -import h2Replay -import RandomReplay - - -def worker(input, output, proxy, replay_type, nThread): - #progress_bar = Bar(" Replaying sessions {0}".format(current_process().name), max=input.qsize()) - #print("playing {0}=>{1}:{2}".format(current_process().name,session._timestamp,proxy)) - if replay_type == 'nossl': - NonSSL.client_replay(input, proxy, output, nThread) - elif replay_type == 'ssl': - SSLReplay.client_replay(input, proxy, output, nThread) - elif replay_type == 'h2': - h2Replay.client_replay(input, proxy, output, nThread) - elif replay_type == 'random': - RandomReplay.client_replay(input, proxy, output, nThread) - - # progress_bar.next() - # progress_bar.finish() - print("process{0} has exited".format(current_process().name)) diff --git a/tests/tools/traffic-replay/__main__.py b/tests/tools/traffic-replay/__main__.py deleted file mode 100644 index 4b64e5180ae..00000000000 --- a/tests/tools/traffic-replay/__main__.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import, division, print_function -import mainProcess -import argparse -import Config - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser.add_argument("-type", action='store', dest='replay_type', - help="Replay type: ssl/random/h2/nossl/mixed (at least 2 processes needed for mixed)") - parser.add_argument("-log_dir", type=str, help="directory of JSON replay files") - parser.add_argument("-v", dest="verbose", help="verify response status code", action="store_true") - parser.add_argument("-host", help="proxy/host to send the requests to", default=Config.proxy_host) - parser.add_argument("-port", type=int, help=" The non secure port of ATS to send the request to", - default=Config.proxy_nonssl_port) - parser.add_argument("-s_port", type=int, help="secure port", default=Config.proxy_ssl_port) - parser.add_argument("-ca_cert", help="Certificate to present", default=Config.ca_certs) - parser.add_argument("-colorize", type=str, help="specify whether to use colorize the output", default='True') - - args = parser.parse_args() - - # Let 'er loose - #main(args.log_dir, args.hostname, int(args.port), args.threads, args.timing, args.verbose) - Config.colorize = True if args.colorize == 'True' else False - mainProcess.main(args.log_dir, args.replay_type, args.verbose, pHost=args.host, pNSSLport=args.port, pSSL=args.s_port) diff --git a/tests/tools/traffic-replay/extractHeader.py b/tests/tools/traffic-replay/extractHeader.py deleted file mode 100644 index a8151833b3d..00000000000 --- a/tests/tools/traffic-replay/extractHeader.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sessionvalidation.sessionvalidation as sv - - -def extract_txn_req_method(headers): - ''' Extracts the HTTP request method from the header in a string format ''' - line = (headers.split('\r\n'))[0] - return (line.split(' '))[0] - - -def extract_host(headers): - ''' Returns the host header from the given headers ''' - lines = headers.split('\r\n') - for line in lines: - if 'Host:' in line: - return line.split(' ')[1] - return "notfound" - -def responseHeaderTuple_to_dict(header): - header_dict = {} - - for key, val in header: - if key.lower() in header_dict: - header_dict[key.lower()] += ", {0}".format(val) - else: - header_dict[key.lower()] = val - - return header_dict - -def responseHeader_to_dict(header): - headerFields = header.split('\r\n', 1)[1] - fields = headerFields.split('\r\n') - header = [x for x in fields if (x != u'')] - headers = {} - for line in header: - split_here = line.find(":") - # append multiple headers into a single string - if line[:split_here].lower() in headers: - headers[line[:split_here].lower()] += ", {0}".format(line[(split_here + 1):].strip()) - else: - headers[line[:split_here].lower()] = line[(split_here + 1):].strip() - - return headers - - -def header_to_dict(header): - ''' Convert a HTTP header in string format to a python dictionary - Returns a dictionary of header values - ''' - header = header.split('\r\n') - header = [x for x in header if (x != u'')] - headers = {} - for line in header: - should_skip = False - - # we have to ignore the intital request line with the HTTP method in it - for method in sv.allowed_HTTP_request_methods: - if method in line: - should_skip = True - - if should_skip: # ignore initial request line - continue - - split_here = line.find(":") - headers[line[:split_here]] = line[(split_here + 1):].strip() - - return headers - - -def extract_GET_path(headers): - ''' Extracts the HTTP request URL from the header in a string format ''' - line = (headers.split('\r\n'))[0] - return (line.split(' '))[1] diff --git a/tests/tools/traffic-replay/h2Replay.py b/tests/tools/traffic-replay/h2Replay.py deleted file mode 100644 index 2026c74bf88..00000000000 --- a/tests/tools/traffic-replay/h2Replay.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from threading import Thread -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import extractHeader -import mainProcess -import json -from hyper import HTTP20Connection -from hyper.tls import wrap_socket, H2_NPN_PROTOCOLS, H2C_PROTOCOL -from hyper.common.bufsocket import BufferedSocket -import hyper -import socket -import logging -import h2 -from h2.connection import H2Configuration -import threading -import Config - -log = logging.getLogger(__name__) -bSTOP = False -hyper.tls._context = hyper.tls.init_context() -hyper.tls._context.check_hostname = False -hyper.tls._context.verify_mode = hyper.compat.ssl.CERT_NONE - - -class _LockedObject(object): - """ - A wrapper class that hides a specific object behind a lock. - - The goal here is to provide a simple way to protect access to an object - that cannot safely be simultaneously accessed from multiple threads. The - intended use of this class is simple: take hold of it with a context - manager, which returns the protected object. - """ - - def __init__(self, obj): - self.lock = threading.RLock() - self._obj = obj - - def __enter__(self): - self.lock.acquire() - return self._obj - - def __exit__(self, _exc_type, _exc_val, _exc_tb): - self.lock.release() - - -class h2ATS(HTTP20Connection): - - def __init_state(self): - """ - Initializes the 'mutable state' portions of the HTTP/2 connection - object. - - This method exists to enable HTTP20Connection objects to be reused if - they're closed, by resetting the connection object to its basic state - whenever it ends up closed. Any situation that needs to recreate the - connection can call this method and it will be done. - - This is one of the only methods in hyper that is truly private, as - users should be strongly discouraged from messing about with connection - objects themselves. - """ - - config1 = H2Configuration( - client_side=True, - header_encoding='utf-8', - validate_outbound_headers=False, - validate_inbound_headers=False, - - ) - self._conn = _LockedObject(h2.connection.H2Connection(config=config1)) - - # Streams are stored in a dictionary keyed off their stream IDs. We - # also save the most recent one for easy access without having to walk - # the dictionary. - # - # We add a set of all streams that we or the remote party forcefully - # closed with RST_STREAM, to avoid encountering issues where frames - # were already in flight before the RST was processed. - # - # Finally, we add a set of streams that recently received data. When - # using multiple threads, this avoids reading on threads that have just - # acquired the I/O lock whose streams have already had their data read - # for them by prior threads. - self.streams = {} - self.recent_stream = None - self.next_stream_id = 1 - self.reset_streams = set() - self.recent_recv_streams = set() - - # The socket used to send data. - self._sock = None - - # Instantiate a window manager. - #self.window_manager = self.__wm_class(65535) - - return - - def __init__(self, host, **kwargs): - HTTP20Connection.__init__(self, host, **kwargs) - self.__init_state() - - def connect(self): - """ - Connect to the server specified when the object was created. This is a - no-op if we're already connected. - - Concurrency - ----------- - - This method is thread-safe. It may be called from multiple threads, and - is a noop for all threads apart from the first. - - :returns: Nothing. - - """ - #print("connecting to ATS") - with self._lock: - if self._sock is not None: - return - sni = self.host - if not self.proxy_host: - host = self.host - port = self.port - else: - host = self.proxy_host - port = self.proxy_port - - sock = socket.create_connection((host, port)) - - if self.secure: - #assert not self.proxy_host, "Proxy with HTTPS not supported." - sock, proto = wrap_socket(sock, sni, self.ssl_context, - force_proto=self.force_proto) - else: - proto = H2C_PROTOCOL - - log.debug("Selected NPN protocol: %s", proto) - assert proto in H2_NPN_PROTOCOLS or proto == H2C_PROTOCOL - - self._sock = BufferedSocket(sock, self.network_buffer_size) - - self._send_preamble() - - -def createDummyBodywithLength(numberOfbytes): - if numberOfbytes == 0: - return None - body = 'a' - while numberOfbytes != 1: - body += 'b' - numberOfbytes -= 1 - return body - - -def handleResponse(response, *args, **kwargs): - print(response.status_code) - # resp=args[0] - #expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2) - #expected_output = (int(expected_output_split[1]), str( expected_output_split[2])) - #r = result.Result(session_filename, expected_output[0], response.status_code) - # print(r.getResultString(colorize=True)) -# make sure len of the message body is greater than length - - -def gen(): - yield 'pforpersia,champaignurbana'.encode('utf-8') - yield 'there'.encode('utf-8') - - -def txn_replay(session_filename, txn, proxy, result_queue, h2conn, request_IDs): - """ Replays a single transaction - :param request_session: has to be a valid requests session""" - req = txn.getRequest() - resp = txn.getResponse() - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier - if 'body' in txn_req_headers_dict: - del txn_req_headers_dict['body'] - responseID = -1 - #print("Replaying session") - try: - # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), - # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), - # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me - method = extractHeader.extract_txn_req_method(txn_req_headers) - response = None - mbody = None - #txn_req_headers_dict['Host'] = "localhost" - if 'Transfer-Encoding' in txn_req_headers_dict: - # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header - # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this - # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? - #del txn_req_headers_dict['Host'] - if 'Content-Length' in txn_req_headers_dict: - #print("ewww !") - del txn_req_headers_dict['Content-Length'] - mbody = gen() - if 'Content-Length' in txn_req_headers_dict: - nBytes = int(txn_req_headers_dict['Content-Length']) - mbody = createDummyBodywithLength(nBytes) - if 'Connection' in txn_req_headers_dict: - del txn_req_headers_dict['Connection'] - #str2 = extractHeader.extract_host(txn_req_headers)+ extractHeader.extract_GET_path(txn_req_headers) - # print(str2) - if method == 'GET': - responseID = h2conn.request('GET', url=extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=mbody) - # print("get response", responseID) - return responseID - # request_IDs.append(responseID) - #response = h2conn.get_response(id) - # print(response.headers) - # if 'Content-Length' in response.headers: - # content = response.read() - #print("len: {0} received {1}".format(response.headers['Content-Length'],content)) - - elif method == 'POST': - responseID = h2conn.request('POST', url=extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=mbody) - print("get response", responseID) - return responseID - - elif method == 'HEAD': - responseID = h2conn.request('HEAD', url=extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict) - print("get response", responseID) - return responseID - - except UnicodeEncodeError as e: - # these unicode errors are due to the interaction between Requests and our wiretrace data. - # TODO fix - print("UnicodeEncodeError exception") - - except: - e = sys.exc_info() - print("ERROR in requests: ", e, response, session_filename) - - -def session_replay(input, proxy, result_queue): - global bSTOP - ''' Replay all transactions in session - - This entire session will be replayed in one requests.Session (so one socket / TCP connection)''' - # if timing_control: - # time.sleep(float(session._timestamp)) # allow other threads to run - while bSTOP == False: - for session in iter(input.get, 'STOP'): - print(bSTOP) - if session == 'STOP': - print("Queue is empty") - bSTOP = True - break - txn = session.returnFirstTransaction() - req = txn.getRequest() - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - with h2ATS(txn_req_headers_dict['Host'], secure=True, proxy_host=Config.proxy_host, proxy_port=Config.proxy_ssl_port) as h2conn: - request_IDs = [] - respList = [] - for txn in session.getTransactionIter(): - try: - ret = txn_replay(session._filename, txn, proxy, result_queue, h2conn, request_IDs) - respList.append(txn.getResponse()) - request_IDs.append(ret) - #print("txn return value is ",ret) - except: - e = sys.exc_info() - print("ERROR in replaying: ", e, txn.getRequest().getHeaders()) - for id in request_IDs: - expectedH = respList.pop(0) - # print("extracting",id) - response = h2conn.get_response(id) - #print("code {0}:{1}".format(response.status,response.headers)) - response_dict = {} - if mainProcess.verbose: - for field, value in response.headers.items(): - response_dict[field.decode('utf-8')] = value.decode('utf-8') - - expected_output_split = expectedH.getHeaders().split('\r\n')[0].split(' ', 2) - expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) - r = result.Result("", expected_output[0], response.status, response.read()) - expected_Dict = extractHeader.responseHeader_to_dict(expectedH.getHeaders()) - b_res, res = r.getResult(response_dict, expected_Dict, colorize=Config.colorize) - print(res) - - if not b_res: - print("Received response") - print(response_dict) - print("Expected response") - print(expected_Dict) - - bSTOP = True - #print("Queue is empty") - input.put('STOP') - break - - -def client_replay(input, proxy, result_queue, nThread): - Threads = [] - for i in range(nThread): - t = Thread(target=session_replay, args=[input, proxy, result_queue]) - t.start() - Threads.append(t) - - for t1 in Threads: - t1.join() diff --git a/tests/tools/traffic-replay/mainProcess.py b/tests/tools/traffic-replay/mainProcess.py deleted file mode 100644 index bde8de4a5fb..00000000000 --- a/tests/tools/traffic-replay/mainProcess.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import sys -import json -import socket -import os -import threading -import time -import argparse -import subprocess -import shlex -from multiprocessing import Pool, Process -from collections import deque -#from progress.bar import Bar - -sys.path.append( - os.path.normpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..' - ) - ) -) - -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import WorkerTask -import Scheduler -import Config -verbose = False - - -def check_for_ats(hostname, port): - ''' Checks to see if ATS is running on `hostname` and `port` - If not running, this function will terminate the script - ''' - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((hostname, port)) - if result != 0: - # hostname:port is not being listened to - print('==========') - print('Error: Apache Traffic Server is not running on {0}:{1}'.format(hostname, port)) - print('Aborting') - print('==========') - sys.exit() -# Note: this function can't handle multi-line (ie wrapped line) headers -# Hopefully this isn't an issue because multi-line headers are deprecated now - - -def main(path, replay_type, Bverbose, pHost=Config.proxy_host, pNSSLport=Config.proxy_nonssl_port, pSSL=Config.proxy_ssl_port): - global verbose - verbose = Bverbose - check_for_ats(pHost, pNSSLport) - Config.proxy_host = pHost - Config.proxy_nonssl_port = pNSSLport - Config.proxy_ssl_port = pSSL - proxy = {"http": "http://{0}:{1}".format(Config.proxy_host, Config.proxy_nonssl_port)} - Scheduler.LaunchWorkers(path, Config.nProcess, proxy, replay_type, Config.nThread) diff --git a/tests/unit_tests/Makefile.am b/tests/unit_tests/Makefile.am index 84db12b2842..791c6005d2c 100644 --- a/tests/unit_tests/Makefile.am +++ b/tests/unit_tests/Makefile.am @@ -27,3 +27,4 @@ unit_tests_SOURCES = main.cpp clang-tidy-local: $(DIST_SOURCES) $(CXX_Clang_Tidy) + \ No newline at end of file From 8df21423ec6528801da4e498dab75336aaf0be32 Mon Sep 17 00:00:00 2001 From: Jesse Zhang Date: Fri, 25 Jan 2019 11:16:21 -0600 Subject: [PATCH 196/526] Test modifications for new microserver compatibility --- .../cookie_remap/collapseslashes.test.py | 7 ++++++ .../cookie_remap/gold/collapseslashes.gold | 2 +- .../cookie_remap/gold/doesntexistcookie.gold | 2 +- .../cookie_remap/gold/existscookie.gold | 2 +- .../cookie_remap/gold/matchcookie.gold | 2 +- .../cookie_remap/gold/matchcookie2.gold | 2 +- .../pluginTest/cookie_remap/gold/matrix.gold | 14 ++++------- .../cookie_remap/gold/regexdoesntmatch.gold | 2 +- .../cookie_remap/gold/regexmatches.gold | 2 +- .../cookie_remap/gold/substitute.gold | 10 +++----- .../cookie_remap/gold/wontmatchcookie.gold | 2 +- .../cookie_remap/gold/wontmatchcookie2.gold | 2 +- .../cookie_remap/matrixparams.test.py | 25 +++++++++++++++++++ .../cookie_remap/regexcookie.test.py | 2 +- .../cookie_remap/substitute.test.py | 14 ++++++++--- tests/gold_tests/tls/tls_verify_base.test.py | 10 ++++---- 16 files changed, 67 insertions(+), 33 deletions(-) diff --git a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py index 48198996aa1..7cc647f0d0b 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py @@ -34,6 +34,13 @@ # and verify it collapsed the double // server = Test.MakeOriginServer("server", ip='127.0.0.10') +request_header = {"headers": "GET /i/like/cheetos?.done=http://finance.yahoo.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + # Setup the remap configuration config_path = os.path.join(Test.TestDirectory, "configs/collapseconfig.txt") with open(config_path, 'r') as config_file: diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold index 68802f1e736..254fdc612ca 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold @@ -1,3 +1,3 @@ `` -Request=GET /i/like/cheetos?.done=http://finance.yahoo.com HTTP/1.1 lookup_key={PATH} +Serving GET /i/like/cheetos?.done=http://finance.yahoo.com... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold index d137bee4ec8..204c488f4e9 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold @@ -1,3 +1,3 @@ `` -Request=GET /cookiedoesntexist HTTP/1.1 lookup_key={PATH} +Serving GET /cookiedoesntexist... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold index 9bae354b2c6..29680e34bc2 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold @@ -1,3 +1,3 @@ `` -Request=GET /cookieexists HTTP/1.1 lookup_key={PATH} +Serving GET /cookieexists... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold index fe1811dbd5f..53c5f864a69 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold @@ -1,3 +1,3 @@ `` -Request=GET /cookiematches HTTP/1.1 lookup_key={PATH} +Serving GET /cookiematches... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold index 969256d8d40..2fed63291e6 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold @@ -1,3 +1,3 @@ `` -Request=GET /cookiematches?a=1&b=2&c=3 HTTP/1.1 lookup_key={PATH} +Serving GET /cookiematches?a=1&b=2&c=3... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold index dfc27353d5e..3989deaa61e 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold @@ -1,15 +1,11 @@ `` -Request=GET /eighth/magic;matrix=1/eighth HTTP/1.1 lookup_key={PATH} +Serving GET /eighth/magic;matrix=1/eighth... Finished `` +Serving GET /eighth/magic;matrix=1/eighth?hello=10... Finished `` -Request=GET /eighth/magic;matrix=1/eighth?hello=10 HTTP/1.1 lookup_key={PATH} +Serving GET /tenth/magic/tenth;matrix=2... Finished `` +Serving GET /tenth/magic/tenth;matrix=2?query=10... Finished `` -Request=GET /tenth/magic/tenth;matrix=2 HTTP/1.1 lookup_key={PATH} -`` -`` -Request=GET /tenth/magic/tenth;matrix=2?query=10 HTTP/1.1 lookup_key={PATH} -`` -`` -Request=GET /eleventh/magic;matrix=4/eleventh;matrix=2?query=true HTTP/1.1 lookup_key={PATH} +Serving GET /eleventh/magic;matrix=4/eleventh;matrix=2?query=true... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold index 206e19b875b..6e357f14b25 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold @@ -1,3 +1,3 @@ `` -Request=GET /regexdoesntmatch HTTP/1.1 lookup_key={PATH} +Serving GET /regexdoesntmatch... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold index 5f484fdd5e0..0f3b6f30f53 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold @@ -1,3 +1,3 @@ `` -Request=GET /regexmatches?cookies=oreos-chipsahoy-icecream HTTP/1.1 lookup_key={PATH} +Serving GET /regexmatches?cookies=oreos-chipsahoy-icecream... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold index b490d41fe95..76ab353bda1 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold @@ -1,9 +1,7 @@ `` -Request=GET /photos/search?query=magic HTTP/1.1 lookup_key={PATH} +Serving GET /photos/search?query=magic... Finished `` +Serving GET /photos/search?query=/theunmatchedpath... Finished `` -Request=GET /photos/search?query=/theunmatchedpath HTTP/1.1 lookup_key={PATH} -`` -`` -equest=GET /photos/search/magic/foobar HTTP/1.1 lookup_key={PATH} -`` +Serving GET /photos/search/magic/foobar... Finished +`` \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold index 29bb5e517fd..2257bd61371 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold @@ -1,3 +1,3 @@ `` -Request=GET /cookiedoesntmatch HTTP/1.1 lookup_key={PATH} +Serving GET /cookiedoesntmatch... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold index 6d5dbb16c23..83470c5b00f 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold @@ -1,3 +1,3 @@ `` -Request=GET /cookiedoesntmatch?a=1&b=2&c=3 HTTP/1.1 lookup_key={PATH} +Serving GET /cookiedoesntmatch?a=1&b=2&c=3... Finished `` diff --git a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py index 9b3c7f32eb4..0d8ca8dec82 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py @@ -35,6 +35,31 @@ # That's why I am not adding any canned request/response server = Test.MakeOriginServer("server", ip='127.0.0.10') +request_header = {"headers": "GET /eighth/magic;matrix=1/eighth HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header, response_header) + +request_header_2 = {"headers": "GET /eighth/magic;matrix=1/eighth?hello=10 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_2, response_header_2) + +request_header_3 = {"headers": "GET /tenth/magic/tenth;matrix=2 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_3 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_3, response_header_3) + +request_header_4 = {"headers": "GET /tenth/magic/tenth;matrix=2?query=10 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_4 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_4, response_header_4) + +request_header_5 = {"headers": "GET /eleventh/magic;matrix=4/eleventh;matrix=2?query=true HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_5 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_5, response_header_5) + # Setup the remap configuration config_path = os.path.join(Test.TestDirectory, "configs/matrixconfig.txt") with open(config_path, 'r') as config_file: diff --git a/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py index 3e2c8175a7c..bf0c594a614 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py @@ -34,7 +34,7 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /regexmatches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /regexmatches?cookies=oreos-chipsahoy-icecream HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} diff --git a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py index 9cfb01d8a69..7a5dfe5ac4b 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py @@ -32,13 +32,21 @@ server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -# expected response from the origin server +request_header = {"headers": "GET /photos/search?query=magic HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -# add response to the server dictionary server.addResponse("sessionfile.log", request_header, response_header) +request_header_2 = {"headers": "GET /photos/search?query=/theunmatchedpath HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_2, response_header_2) + +request_header_3 = {"headers": "GET /photos/search/magic/foobar HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_3 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_3, response_header_3) + # Setup the remap configuration config_path = os.path.join(Test.TestDirectory, "configs/substituteconfig.txt") with open(config_path, 'r') as config_file: diff --git a/tests/gold_tests/tls/tls_verify_base.test.py b/tests/gold_tests/tls/tls_verify_base.test.py index 8279b22a7d3..1b896ff6aac 100644 --- a/tests/gold_tests/tls/tls_verify_base.test.py +++ b/tests/gold_tests/tls/tls_verify_base.test.py @@ -54,15 +54,15 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' From 4457e907e9c4b095943cf83ba0eed93ee6cdcb08 Mon Sep 17 00:00:00 2001 From: Pushkar Pradhan Date: Fri, 25 Jan 2019 07:58:41 -0800 Subject: [PATCH 197/526] Remove cookie_remap plugin's unit tests until I fix the broken build --- plugins/experimental/cookie_remap/Makefile.inc | 9 --------- 1 file changed, 9 deletions(-) diff --git a/plugins/experimental/cookie_remap/Makefile.inc b/plugins/experimental/cookie_remap/Makefile.inc index 5b44c4f4a4e..11de3ac0104 100644 --- a/plugins/experimental/cookie_remap/Makefile.inc +++ b/plugins/experimental/cookie_remap/Makefile.inc @@ -27,13 +27,4 @@ experimental_cookie_remap_cookie_remap_la_LDFLAGS = \ AM_CPPFLAGS += @YAMLCPP_INCLUDES@ -check_PROGRAMS += \ - cookie_remap/test_cookiejar - -cookie_remap_test_cookiejar_CPPFLAGS = $(AM_CPPFLAGS) -Iexperimental/cookie_remap -I$(abs_top_srcdir)/tests/include -cookie_remap_test_cookiejar_SOURCES = \ - experimental/cookie_remap/unit_tests/cookiejar/unit_test.cc \ - experimental/cookie_remap/strip.c \ - experimental/cookie_remap/cookiejar.cc - # vim: ft=make ts=8 sw=8 et: From 0188338cf9d1e3d14e039eb0200dabd594ead3bf Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 25 Jan 2019 15:01:01 -0600 Subject: [PATCH 198/526] Remove start_HttpProxyServerBackDoor. --- proxy/ProxyClientSession.h | 2 +- proxy/http/Http1ClientSession.cc | 5 +---- proxy/http/Http1ClientSession.h | 2 +- proxy/http/HttpProxyServerMain.cc | 16 ---------------- proxy/http/HttpProxyServerMain.h | 2 -- proxy/http/HttpSessionAccept.cc | 16 +++++----------- proxy/http/HttpSessionAccept.h | 13 +------------ proxy/http2/Http2ClientSession.cc | 5 +---- proxy/http2/Http2ClientSession.h | 2 +- proxy/http2/Http2SessionAccept.cc | 2 +- 10 files changed, 12 insertions(+), 53 deletions(-) diff --git a/proxy/ProxyClientSession.h b/proxy/ProxyClientSession.h index 7e1d03974ff..33b91486011 100644 --- a/proxy/ProxyClientSession.h +++ b/proxy/ProxyClientSession.h @@ -81,7 +81,7 @@ class ProxyClientSession : public VConnection virtual void free(); virtual void start() = 0; - virtual void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) = 0; + virtual void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) = 0; virtual NetVConnection *get_netvc() const = 0; diff --git a/proxy/http/Http1ClientSession.cc b/proxy/http/Http1ClientSession.cc index c4c3572c562..9f1a00224e2 100644 --- a/proxy/http/Http1ClientSession.cc +++ b/proxy/http/Http1ClientSession.cc @@ -145,7 +145,7 @@ Http1ClientSession::free() } void -Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) +Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) { ink_assert(new_vc != nullptr); ink_assert(client_vc == nullptr); @@ -159,9 +159,6 @@ Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB MUTEX_TRY_LOCK(lock, mutex, this_ethread()); ink_assert(lock.is_locked()); - // Disable hooks for backdoor connections. - this->hooks_on = !backdoor; - // Unique client session identifier. con_id = ProxyClientSession::next_connection_id(); diff --git a/proxy/http/Http1ClientSession.h b/proxy/http/Http1ClientSession.h index b81d41be5f9..ce8346665ce 100644 --- a/proxy/http/Http1ClientSession.h +++ b/proxy/http/Http1ClientSession.h @@ -65,7 +65,7 @@ class Http1ClientSession : public ProxyClientSession this->release(&trans); } - void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) override; + void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; // Implement VConnection interface. VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index fc68a9a67bb..09530976492 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -368,22 +368,6 @@ start_HttpProxyServer() } } -void -start_HttpProxyServerBackDoor(int port, int accept_threads) -{ - NetProcessor::AcceptOptions opt; - HttpSessionAccept::Options ha_opt; - - opt.local_port = port; - opt.accept_threads = accept_threads; - opt.localhost_only = true; - ha_opt.backdoor = true; - opt.backdoor = true; - - // The backdoor only binds the loopback interface - netProcessor.main_accept(new HttpSessionAccept(ha_opt), NO_FD, opt); -} - void stop_HttpProxyServer() { diff --git a/proxy/http/HttpProxyServerMain.h b/proxy/http/HttpProxyServerMain.h index 33f19ec73ee..96c461abecc 100644 --- a/proxy/http/HttpProxyServerMain.h +++ b/proxy/http/HttpProxyServerMain.h @@ -47,8 +47,6 @@ void start_HttpProxyServer(); void stop_HttpProxyServer(); -void start_HttpProxyServerBackDoor(int port, int accept_threads = 0); - NetProcessor::AcceptOptions make_net_accept_options(const HttpProxyPort *port, unsigned nthreads); extern std::mutex proxyServerMutex; diff --git a/proxy/http/HttpSessionAccept.cc b/proxy/http/HttpSessionAccept.cc index ac24ef86cb4..a13f8f30f1b 100644 --- a/proxy/http/HttpSessionAccept.cc +++ b/proxy/http/HttpSessionAccept.cc @@ -33,16 +33,10 @@ HttpSessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferReade IpAllow::ACL acl; ip_port_text_buffer ipb; - // The backdoor port is now only bound to "localhost", so no - // reason to check for if it's incoming from "localhost" or not. - if (backdoor) { - acl = IpAllow::makeAllowAllACL(); - } else { - acl = IpAllow::match(client_ip, IpAllow::SRC_ADDR); - if (!acl.isValid()) { // if there's no ACL, it's a hard deny. - Warning("client '%s' prohibited by ip-allow policy", ats_ip_ntop(client_ip, ipb, sizeof(ipb))); - return false; - } + acl = IpAllow::match(client_ip, IpAllow::SRC_ADDR); + if (!acl.isValid()) { // if there's no ACL, it's a hard deny. + Warning("client '%s' prohibited by ip-allow policy", ats_ip_ntop(client_ip, ipb, sizeof(ipb))); + return false; } // Set the transport type if not already set @@ -66,7 +60,7 @@ HttpSessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferReade new_session->host_res_style = ats_host_res_from(client_ip->sa_family, host_res_preference); new_session->acl = std::move(acl); - new_session->new_connection(netvc, iobuf, reader, backdoor); + new_session->new_connection(netvc, iobuf, reader); return true; } diff --git a/proxy/http/HttpSessionAccept.h b/proxy/http/HttpSessionAccept.h index 49d8350e849..0f9978bf0a6 100644 --- a/proxy/http/HttpSessionAccept.h +++ b/proxy/http/HttpSessionAccept.h @@ -81,10 +81,6 @@ class HttpSessionAcceptOptions bool f_transparent_passthrough; /// Set transparent passthrough. self &setTransparentPassthrough(bool); - /// Accepting backdoor connections. - bool backdoor; - /// Set backdoor accept. - self &setBackdoor(bool); /// Host address resolution preference order. HostResPreferenceOrder host_res_preference; /// Set the host query preference. @@ -96,7 +92,7 @@ class HttpSessionAcceptOptions }; inline HttpSessionAcceptOptions::HttpSessionAcceptOptions() - : transport_type(0), outbound_port(0), f_outbound_transparent(false), f_transparent_passthrough(false), backdoor(false) + : transport_type(0), outbound_port(0), f_outbound_transparent(false), f_transparent_passthrough(false) { memcpy(host_res_preference, host_res_default_preference_order, sizeof(host_res_preference)); } @@ -149,13 +145,6 @@ HttpSessionAcceptOptions::setTransparentPassthrough(bool flag) return *this; } -inline HttpSessionAcceptOptions & -HttpSessionAcceptOptions::setBackdoor(bool flag) -{ - backdoor = flag; - return *this; -} - inline HttpSessionAcceptOptions & HttpSessionAcceptOptions::setHostResPreference(HostResPreferenceOrder const order) { diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index ed830a9d44d..99921dd03bd 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -151,15 +151,12 @@ Http2ClientSession::start() } void -Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) +Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) { ink_assert(new_vc->mutex->thread_holding == this_ethread()); HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, new_vc->mutex->thread_holding); HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT, new_vc->mutex->thread_holding); - // HTTP/2 for the backdoor connections? Let's not deal woth that yet. - ink_release_assert(backdoor == false); - // Unique client session identifier. this->con_id = ProxyClientSession::next_connection_id(); this->client_vc = new_vc; diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 51f709e4e71..fe8a79500f5 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -164,7 +164,7 @@ class Http2ClientSession : public ProxyClientSession void start() override; void destroy() override; void free() override; - void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) override; + void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; bool ready_to_free() const diff --git a/proxy/http2/Http2SessionAccept.cc b/proxy/http2/Http2SessionAccept.cc index 165eb5e9baa..f3180798f00 100644 --- a/proxy/http2/Http2SessionAccept.cc +++ b/proxy/http2/Http2SessionAccept.cc @@ -59,7 +59,7 @@ Http2SessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferRead new_session->outbound_ip4 = options.outbound_ip4; new_session->outbound_ip6 = options.outbound_ip6; new_session->outbound_port = options.outbound_port; - new_session->new_connection(netvc, iobuf, reader, false /* backdoor */); + new_session->new_connection(netvc, iobuf, reader); return true; } From ad3b95123d43b7ebf8ef24fb3cb6e4df444b865c Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 25 Jan 2019 20:52:54 -0600 Subject: [PATCH 199/526] Cleanup: remove lib/cppapi from clang-format, it's been moved. --- Makefile.am | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index e04aa9aa65a..15a411a7709 100644 --- a/Makefile.am +++ b/Makefile.am @@ -135,7 +135,6 @@ clang-format-iocore: clang-format-lib: @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/include - @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/cppapi @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/records clang-format-mgmt: From 41a2a67186f690b30acac93adcad8f306c6ed91a Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 25 Jan 2019 20:32:36 -0600 Subject: [PATCH 200/526] Doc: Fix a number of minor errors, add some documentation for the thread affinity plugin API. --- doc/Makefile.am | 4 +--- doc/admin-guide/files/ssl_server_name.yaml.en.rst | 2 +- doc/admin-guide/plugins/cookie_remap.en.rst | 5 +---- doc/admin-guide/plugins/index.en.rst | 1 + doc/admin-guide/plugins/slice.en.rst | 2 +- doc/appendices/glossary.en.rst | 7 +++++++ .../api/functions/TSContThreadAffinityClear.en.rst | 2 +- .../api/functions/TSContThreadAffinityGet.en.rst | 2 +- .../api/functions/TSContThreadAffinitySet.en.rst | 2 +- .../api/functions/TSThreadSelf.en.rst | 3 +++ doc/developer-guide/api/functions/TSTypes.en.rst | 14 ++++++++++++++ 11 files changed, 32 insertions(+), 12 deletions(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index caa7db7a8db..f6c174fe6d9 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -48,12 +48,10 @@ doxygen: Doxyfile PAPER = letter BUILDDIR = docbuild -# Internal variables. # [amc] LaTex apparently doesn't work as of Sphinx 1.6.1 # see https://media.readthedocs.org/pdf/sphinx/1.6.3/sphinx.pdf # section 24.3.2 around page 247, third item for 'NotImplementedError', so this is kind of useless. -PAPEROPT_a4 = -t latex_a4 -PAPEROPT_letter = -t latex_letter + ALLSPHINXOPTS = $(SPHINXOPTS) # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(SPHINXOPTS) diff --git a/doc/admin-guide/files/ssl_server_name.yaml.en.rst b/doc/admin-guide/files/ssl_server_name.yaml.en.rst index ac8068b49dc..6f2f1a02708 100644 --- a/doc/admin-guide/files/ssl_server_name.yaml.en.rst +++ b/doc/admin-guide/files/ssl_server_name.yaml.en.rst @@ -74,7 +74,7 @@ verify_client One of the values :code:`NONE`, :code:`MODERATE`, or : By default this is :ts:cv:`proxy.config.ssl.client.certification_level`. valid_tls_versions_in This specifies the list of TLS protocols that will be offered to user agents during - the TLS negotiaton. This replaces the global settings in :ts:cv:`proxy.config.ssl.TSLv1`, + the TLS negotiaton. This replaces the global settings in :ts:cv:`proxy.config.ssl.TLSv1`, :ts:cv:`proxy.config.ssl.TLSv1_1`, :ts:cv:`proxy.config.ssl.TLSv1_2`, and :ts:cv:`proxy.config.ssl.TLSv1_3`. The potential values are TLSv1, TLSv1_1, TLSv1_2, and TLSv1_3. You must list all protocols that |TS| should offer to the client when using diff --git a/doc/admin-guide/plugins/cookie_remap.en.rst b/doc/admin-guide/plugins/cookie_remap.en.rst index f79074b416e..24b5d94783b 100644 --- a/doc/admin-guide/plugins/cookie_remap.en.rst +++ b/doc/admin-guide/plugins/cookie_remap.en.rst @@ -19,11 +19,8 @@ .. _admin-plugins-cookie_remap: -============================================================ Cookie Based Routing Inside TrafficServer Using cookie_remap -============================================================ - ----- +************************************************************ * `Cookie Based Routing Inside TrafficServer Using cookie_remap <#cookie-based-routing-inside-trafficserver-using-cookie_remap>`_ diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index 4f750a31a65..de8e8841eb6 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -52,6 +52,7 @@ Plugins that are considered stable are installed by default in |TS| releases. Cache Promotion Policies Combo Handler Configuration Remap + Cookie Remap ESI Escalate Compress diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst index 34a219e6b4d..86ad1a622d0 100644 --- a/doc/admin-guide/plugins/slice.en.rst +++ b/doc/admin-guide/plugins/slice.en.rst @@ -37,7 +37,7 @@ plugin allows the following: devices and within cache groups. Configuration -============ +============= This plugin is intended for use as a remap plugin and is configured in :file:`remap.config`. diff --git a/doc/appendices/glossary.en.rst b/doc/appendices/glossary.en.rst index 282d00b677c..863853ce575 100644 --- a/doc/appendices/glossary.en.rst +++ b/doc/appendices/glossary.en.rst @@ -35,6 +35,13 @@ Glossary invoked to continue the suspended processing. This can be considered similar to co-routines. + event loop + Code that executes callbacks in continuations from a queue of events. + + event thread + A thread created by |TS| that has an :term:`event loop`. Event loops drive activity in |TS| + and are responsible for all network I/O handling, hook processing, and scheduled events. + session A single connection from a client to Traffic Server, covering all requests and responses on that connection. A session starts when the diff --git a/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst index 984653511d6..ef17ae0cddc 100644 --- a/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst +++ b/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst @@ -19,7 +19,7 @@ .. default-domain:: c TSContThreadAffinityClear -************** +************************* Synopsis ======== diff --git a/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst index c31363c67e1..2bc4955ad19 100644 --- a/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst +++ b/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst @@ -19,7 +19,7 @@ .. default-domain:: c TSContThreadAffinityGet -************** +*********************** Synopsis ======== diff --git a/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst index cae29032bce..5b355130af8 100644 --- a/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst +++ b/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst @@ -19,7 +19,7 @@ .. default-domain:: c TSContThreadAffinitySet -************** +*********************** Synopsis ======== diff --git a/doc/developer-guide/api/functions/TSThreadSelf.en.rst b/doc/developer-guide/api/functions/TSThreadSelf.en.rst index 7c213c2318a..b0d059013de 100644 --- a/doc/developer-guide/api/functions/TSThreadSelf.en.rst +++ b/doc/developer-guide/api/functions/TSThreadSelf.en.rst @@ -30,3 +30,6 @@ Synopsis Description =========== + +Return an instance of :type:`TSThread` that identifies the current thread. This must be called from +within a |TS| create thread context. That is any thread created by the |TS| core or via the |TS| API. diff --git a/doc/developer-guide/api/functions/TSTypes.en.rst b/doc/developer-guide/api/functions/TSTypes.en.rst index 6b18d9a71c9..8abbea62cbd 100644 --- a/doc/developer-guide/api/functions/TSTypes.en.rst +++ b/doc/developer-guide/api/functions/TSTypes.en.rst @@ -150,6 +150,20 @@ more widely. Those are described on this page. .. type:: TSThread + This represents an internal |TS| thread, created by the |TS| core. It is an opaque type which + can be used only to check for equality / inequality, and passed to API functions. An instance + that refers to the current thread can be obtained with :func:`TSThreadSelf`. + +.. type:: TSEventThread + + This type represents an :term:`event thread`. It is an opaque which is used to specify a + particular event processing thread in |TS|. If plugin code is executing in an event thread + (which will be true if called from a hook or a scheduled event) then the current event thread + can be obtained via :func:`TSEventThreadSelf`. + + A :code:`TSEventThread` is also a :type:`TSThread` and can be passed as an argument to any + parameter of type :type:`TSThread`. + .. type:: TSThreadFunc .. type:: TSUuidVersion From e25900853d00ba6d79875eb4733beb2a989cea6f Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 25 Jan 2019 21:05:48 -0600 Subject: [PATCH 201/526] Cleanup: Remove "hooks_on" and associated machinery. This is initialized to 'true' and can never change. --- proxy/ProxyClientSession.cc | 2 +- proxy/ProxyClientSession.h | 7 ------- proxy/ProxyClientTransaction.h | 5 ----- proxy/http/HttpSM.cc | 6 ------ proxy/http/HttpTransact.h | 1 - proxy/http/HttpUpdateSM.cc | 1 - 6 files changed, 1 insertion(+), 21 deletions(-) diff --git a/proxy/ProxyClientSession.cc b/proxy/ProxyClientSession.cc index 23077797564..6e4857bf9aa 100644 --- a/proxy/ProxyClientSession.cc +++ b/proxy/ProxyClientSession.cc @@ -170,7 +170,7 @@ ProxyClientSession::do_api_callout(TSHttpHookID id) this->api_scope = API_HOOK_SCOPE_GLOBAL; this->api_current = nullptr; - if (this->hooks_on && this->has_hooks()) { + if (this->has_hooks()) { SET_HANDLER(&ProxyClientSession::state_api_callout); this->state_api_callout(EVENT_NONE, nullptr); } else { diff --git a/proxy/ProxyClientSession.h b/proxy/ProxyClientSession.h index 33b91486011..241da9b205f 100644 --- a/proxy/ProxyClientSession.h +++ b/proxy/ProxyClientSession.h @@ -134,12 +134,6 @@ class ProxyClientSession : public VConnection return this->debug_on; } - bool - hooks_enabled() const - { - return this->hooks_on; - } - bool has_hooks() const { @@ -311,7 +305,6 @@ class ProxyClientSession : public VConnection // Session specific debug flag. bool debug_on = false; - bool hooks_on = true; bool in_destroy = false; int64_t con_id = 0; diff --git a/proxy/ProxyClientTransaction.h b/proxy/ProxyClientTransaction.h index 4dc46aad445..14de5559e03 100644 --- a/proxy/ProxyClientTransaction.h +++ b/proxy/ProxyClientTransaction.h @@ -98,11 +98,6 @@ class ProxyClientTransaction : public VConnection { return parent ? parent->debug() : false; } - bool - hooks_enabled() const - { - return parent ? parent->hooks_enabled() : false; - } APIHook * ssn_hook_get(TSHttpHookID id) const diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 8cdb22585ac..cf94a9f7b08 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -503,7 +503,6 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader ats_ip_copy(&t_state.client_info.dst_addr, netvc->get_local_addr()); t_state.client_info.dst_addr.port() = netvc->get_local_port(); t_state.client_info.is_transparent = netvc->get_is_transparent(); - t_state.backdoor_request = !client_vc->hooks_enabled(); t_state.client_info.port_attribute = static_cast(netvc->attributes); // Record api hook set state @@ -5106,11 +5105,6 @@ HttpSM::do_http_server_open(bool raw) void HttpSM::do_api_callout_internal() { - if (t_state.backdoor_request) { - handle_api_return(); - return; - } - switch (t_state.api_next_action) { case HttpTransact::SM_ACTION_API_SM_START: cur_hook_id = TS_HTTP_TXN_START_HOOK; diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 082e9cdd054..8183dddc194 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -714,7 +714,6 @@ class HttpTransact bool cdn_remap_complete = false; bool first_dns_lookup = true; - bool backdoor_request = false; // internal HttpRequestData request_data; ParentConfigParams *parent_params = nullptr; ParentResult parent_result; diff --git a/proxy/http/HttpUpdateSM.cc b/proxy/http/HttpUpdateSM.cc index a6ae057dc45..efe7acbdcfd 100644 --- a/proxy/http/HttpUpdateSM.cc +++ b/proxy/http/HttpUpdateSM.cc @@ -72,7 +72,6 @@ HttpUpdateSM::start_scheduled_update(Continuation *cont, HTTPHdr *request) // Fix ME: What should these be set to since there is not a // real client ats_ip4_set(&t_state.client_info.src_addr, htonl(INADDR_LOOPBACK), 0); - t_state.backdoor_request = false; t_state.client_info.port_attribute = HttpProxyPort::TRANSPORT_DEFAULT; t_state.req_flavor = HttpTransact::REQ_FLAVOR_SCHEDULED_UPDATE; From 140e3a3a49a85911de5d6b24fc2d39c5e77367ff Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 23 Jan 2019 14:31:25 -0700 Subject: [PATCH 202/526] Various fixes and improvements to background_fetch These changes include: 1) Adds a new option, --allow-304 / -a, which allows 304 responses to still kick off a background fetch. This is important for request that comes in with e.g. Range: bytes=0-12 If-Modified-Since: Tue, 22 Jan 2019 19:36:03 GMT 2) Similar, to make the conditional request succeed properly with conditional requests, we have to filter out more headers. In addition to the Range: header, we now also filter out any values of If-Match If-Modified-Since If-None-Match If-Range If-Unmodified-Since 3) To make all of this work with both global and per-remap setups, we generalize the option parsing to work equally for both types. To be backward compatible, a single option without a leading '-' is assumed to be the name of the configuration file, but the correct way to use this in a remap plugin is @pparam=--config=/some/config/file.conf 4) In addition, while reading up on this, we should allow the background fetch to be kicked off with an If-Range header. --- plugins/background_fetch/background_fetch.cc | 122 ++++++++++++------- plugins/background_fetch/configs.cc | 51 +++++++- plugins/background_fetch/configs.h | 21 +++- 3 files changed, 148 insertions(+), 46 deletions(-) diff --git a/plugins/background_fetch/background_fetch.cc b/plugins/background_fetch/background_fetch.cc index 394a3dca406..f6b99c9c208 100644 --- a/plugins/background_fetch/background_fetch.cc +++ b/plugins/background_fetch/background_fetch.cc @@ -25,11 +25,12 @@ #include #include #include -#include #include #include #include +#include +#include #include "ts/ts.h" #include "ts/remap.h" @@ -38,12 +39,22 @@ #include "configs.h" // Global config, if we don't have a remap specific config. -static BgFetchConfig *gConfig; +static BgFetchConfig *gConfig = nullptr; + +// This is the list of all headers that must be removed when we make the actual background +// fetch request. +static const std::array FILTER_HEADERS{ + {{TS_MIME_FIELD_RANGE, static_cast(TS_MIME_LEN_RANGE)}, + {TS_MIME_FIELD_IF_MATCH, static_cast(TS_MIME_LEN_IF_MATCH)}, + {TS_MIME_FIELD_IF_MODIFIED_SINCE, static_cast(TS_MIME_LEN_IF_MODIFIED_SINCE)}, + {TS_MIME_FIELD_IF_NONE_MATCH, static_cast(TS_MIME_LEN_IF_NONE_MATCH)}, + {TS_MIME_FIELD_IF_RANGE, static_cast(TS_MIME_LEN_IF_RANGE)}, + {TS_MIME_FIELD_IF_UNMODIFIED_SINCE, static_cast(TS_MIME_LEN_IF_UNMODIFIED_SINCE)}}}; /////////////////////////////////////////////////////////////////////////// -// Hold the global ackground fetch state. This is currently shared across all +// Hold the global background fetch state. This is currently shared across all // configurations, as a singleton. ToDo: Would it ever make sense to do this -// per remap rule? Probably not. +// per remap rule? Maybe for per-remap logging ?? typedef std::unordered_map OutstandingRequests; class BgFetchState @@ -61,11 +72,16 @@ class BgFetchState } ~BgFetchState() { TSMutexDestroy(_lock); } + void - createLog(const char *log_name) + createLog(const std::string &log_name) { - TSDebug(PLUGIN_NAME, "Creating log name %s", log_name); - TSAssert(TS_SUCCESS == TSTextLogObjectCreate(log_name, TS_LOG_MODE_ADD_TIMESTAMP, &_log)); + if (!_log) { + TSDebug(PLUGIN_NAME, "Creating log name %s", log_name.c_str()); + TSAssert(TS_SUCCESS == TSTextLogObjectCreate(log_name.c_str(), TS_LOG_MODE_ADD_TIMESTAMP, &_log)); + } else { + TSError("[%s] A log file was already create, ignoring creation of %s", PLUGIN_NAME, log_name.c_str()); + } } TSTextLogObject @@ -261,10 +277,13 @@ BgFetchData::initialize(TSMBuffer request, TSMLoc req_hdr, TSHttpTxn txnp) TSDebug(PLUGIN_NAME, "Set header Host: %.*s", len, hostp); } - // Next, remove any Range: headers from our request. - if (remove_header(mbuf, hdr_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE) > 0) { - TSDebug(PLUGIN_NAME, "Removed the Range: header from request"); + // Next, remove the Range headers and IMS (conditional) headers from the request + for (auto const &header : FILTER_HEADERS) { + if (remove_header(mbuf, hdr_loc, header.data(), header.size()) > 0) { + TSDebug(PLUGIN_NAME, "Removed the %s header from request", header.data()); + } } + // Everything went as planned, so we can return true ret = true; } @@ -507,9 +526,10 @@ cont_handle_response(TSCont contp, TSEvent event, void *edata) // ToDo: Check the MIME type first, to see if it's a type we care about. // ToDo: Such MIME types should probably be per remap rule. - // Only deal with 206 responses from Origin - TSDebug(PLUGIN_NAME, "Testing: response is 206?"); - if (TS_HTTP_STATUS_PARTIAL_CONTENT == TSHttpHdrStatusGet(response, resp_hdr)) { + // Only deal with 206 and possibly 304 responses from Origin + TSHttpStatus status = TSHttpHdrStatusGet(response, resp_hdr); + TSDebug(PLUGIN_NAME, "Testing: response status code: %d?", status); + if (TS_HTTP_STATUS_PARTIAL_CONTENT == status || (config->allow304() && TS_HTTP_STATUS_NOT_MODIFIED == status)) { // Everything looks good so far, add a TXN hook for SEND_RESPONSE_HDR TSCont contp = TSContCreate(cont_check_cacheable, nullptr); @@ -538,9 +558,6 @@ void TSPluginInit(int argc, const char *argv[]) { TSPluginRegistrationInfo info; - static const struct option longopt[] = {{const_cast("log"), required_argument, nullptr, 'l'}, - {const_cast("config"), required_argument, nullptr, 'c'}, - {nullptr, no_argument, nullptr, '\0'}}; info.plugin_name = (char *)PLUGIN_NAME; info.vendor_name = (char *)"Apache Software Foundation"; @@ -554,26 +571,18 @@ TSPluginInit(int argc, const char *argv[]) gConfig = new BgFetchConfig(cont); - while (true) { - int opt = getopt_long(argc, (char *const *)argv, "lc", longopt, nullptr); - - switch (opt) { - case 'l': - BgFetchState::getInstance().createLog(optarg); - break; - case 'c': - TSDebug(PLUGIN_NAME, "config file '%s'", optarg); - gConfig->readConfig(optarg); - break; - } - - if (opt == -1) { - break; + if (gConfig->parseOptions(argc, argv)) { + // Create the global log file. Note that calling this multiple times currently has no + // effect, only one log file is ever created. The BgFetchState is a singleton. + if (!gConfig->logFile().empty()) { + BgFetchState::getInstance().createLog(gConfig->logFile()); } + TSDebug(PLUGIN_NAME, "Initialized"); + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, cont); + } else { + // ToDo: Hmmm, no way to fail a global plugin here? + TSDebug(PLUGIN_NAME, "Failed to initialize as global plugin"); } - - TSDebug(PLUGIN_NAME, "Initialized"); - TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, cont); } /////////////////////////////////////////////////////////////////////////// @@ -608,16 +617,40 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int / { TSCont cont = TSContCreate(cont_handle_response, nullptr); BgFetchConfig *config = new BgFetchConfig(cont); - - // Parse the optional rules, wihch becomes a linked list of BgFetchRule's. - if (argc > 2) { - TSDebug(PLUGIN_NAME, "config file %s", argv[2]); - config->readConfig(argv[2]); + bool success = true; + + // The first two arguments are the "from" and "to" URL string. We need to + // skip them, but we also require that there be an option to masquerade as + // argv[0], so we increment the argument indexes by 1 rather than by 2. + argc--; + argv++; + + // This is for backwards compatibility, ugly! ToDo: Remove for ATS v9.0.0 IMO. + if (argc > 1 && *argv[1] != '-') { + TSDebug(PLUGIN_NAME, "config file %s", argv[1]); + if (!config->readConfig(argv[1])) { + success = false; + } + } else { + if (config->parseOptions(argc, const_cast(argv))) { + // Create the global log file. Remember, the BgFetchState is a singleton. + if (config->logFile().size()) { + BgFetchState::getInstance().createLog(config->logFile()); + } + } else { + success = false; + } } - *ih = (void *)config; + if (success) { + *ih = config; - return TS_SUCCESS; + return TS_SUCCESS; + } + + // Something went wrong with the configuration setup. + delete config; + return TS_ERROR; } void @@ -625,6 +658,7 @@ TSRemapDeleteInstance(void *ih) { BgFetchConfig *config = static_cast(ih); + TSContDestroy(config->getCont()); delete config; } @@ -644,11 +678,15 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo * /* rri */) if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &bufp, &req_hdrs)) { TSMLoc field_loc = TSMimeHdrFieldFind(bufp, req_hdrs, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE); + if (!field_loc) { // Less common case, but also allow If-Range header to triger, but only if Range not present + field_loc = TSMimeHdrFieldFind(bufp, req_hdrs, TS_MIME_FIELD_IF_RANGE, TS_MIME_LEN_IF_RANGE); + } + if (field_loc) { BgFetchConfig *config = static_cast(ih); TSHttpTxnHookAdd(txnp, TS_HTTP_READ_RESPONSE_HDR_HOOK, config->getCont()); - TSDebug(PLUGIN_NAME, "background fetch TSRemapDoRemap"); + TSDebug(PLUGIN_NAME, "TSRemapDoRemap() added hook, request was Range / If-Range"); TSHandleMLocRelease(bufp, req_hdrs, field_loc); } TSHandleMLocRelease(bufp, TS_NULL_MLOC, req_hdrs); diff --git a/plugins/background_fetch/configs.cc b/plugins/background_fetch/configs.cc index 032216523a2..96fd2518e22 100644 --- a/plugins/background_fetch/configs.cc +++ b/plugins/background_fetch/configs.cc @@ -22,10 +22,56 @@ limitations under the License. */ -#include "configs.h" +#include #include #include +#include "configs.h" + +// Parse the command line options. This got a little wonky, since we decided to have different +// syntax for remap vs global plugin initialization, and the global BG fetch state :-/. Clean up +// later... +bool +BgFetchConfig::parseOptions(int argc, const char *argv[]) +{ + static const struct option longopt[] = {{const_cast("log"), required_argument, nullptr, 'l'}, + {const_cast("config"), required_argument, nullptr, 'c'}, + {const_cast("allow-304"), no_argument, nullptr, 'a'}, + {nullptr, no_argument, nullptr, '\0'}}; + + while (true) { + int opt = getopt_long(argc, (char *const *)argv, "lc", longopt, nullptr); + + if (opt == -1) { + break; + } + + switch (opt) { + case 'l': + TSDebug(PLUGIN_NAME, "option: log file specified: %s", optarg); + _log_file = optarg; + break; + case 'c': + TSDebug(PLUGIN_NAME, "option: config file '%s'", optarg); + if (!readConfig(optarg)) { + // Error messages are written in the parser + return false; + } + break; + case 'a': + TSDebug(PLUGIN_NAME, "option: --allow-304 set"); + _allow_304 = true; + break; + default: + TSError("[%s] invalid plugin option: %c", PLUGIN_NAME, opt); + return false; + break; + } + } + + return true; +} + // Read a config file, populare the linked list (chain the BgFetchRule's) bool BgFetchConfig::readConfig(const char *config_file) @@ -45,11 +91,12 @@ BgFetchConfig::readConfig(const char *config_file) } else { snprintf(file_path, sizeof(file_path), "%s/%s", TSConfigDirGet(), config_file); } - TSDebug(PLUGIN_NAME, "Chosen config file is at: %s", file_path); + TSDebug(PLUGIN_NAME, "chosen config file is at: %s", file_path); file = TSfopen(file_path, "r"); if (nullptr == file) { TSError("[%s] invalid config file: %s", PLUGIN_NAME, file_path); + TSDebug(PLUGIN_NAME, "invalid config file: %s", file_path); return false; } diff --git a/plugins/background_fetch/configs.h b/plugins/background_fetch/configs.h index 569ebde10be..ba6dd35fb5c 100644 --- a/plugins/background_fetch/configs.h +++ b/plugins/background_fetch/configs.h @@ -26,6 +26,7 @@ #include #include +#include #include "rules.h" @@ -48,6 +49,8 @@ class BgFetchConfig } } + bool parseOptions(int argc, const char *argv[]); + BgFetchRule * getRules() const { @@ -60,12 +63,26 @@ class BgFetchConfig return _cont; } + const std::string & + logFile() const + { + return _log_file; + } + + bool + allow304() const + { + return _allow_304; + } + // This parses and populates the BgFetchRule linked list (_rules). bool readConfig(const char *file_name); bool bgFetchAllowed(TSHttpTxn txnp) const; private: - TSCont _cont; - BgFetchRule *_rules{nullptr}; + TSCont _cont = nullptr; + BgFetchRule *_rules = nullptr; + bool _allow_304 = false; + std::string _log_file; }; From bb6db01b13df72bed6bd5ccd311881aefc524619 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 23 Jan 2019 16:57:41 -0700 Subject: [PATCH 203/526] Turns off HostDB disk sync by default We've been running with this set to 0 for quite a while, and I still feel that the whole disk synching mechanism of HostDB is not particularly valid. This is a step towards simplifying HostDB, such that moving to a non-persistent / in memory "DB" is more straight forward. --- doc/admin-guide/files/records.config.en.rst | 5 +++-- iocore/hostdb/HostDB.cc | 2 +- mgmt/RecordsConfig.cc | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index b1a86b35915..7a3236c83d0 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -2715,9 +2715,10 @@ HostDB The filename to persist hostdb to on disk. -.. ts:cv:: CONFIG proxy.config.cache.hostdb.sync_frequency INT 120 +.. ts:cv:: CONFIG proxy.config.cache.hostdb.sync_frequency INT 0 - Set the frequency (in seconds) to sync hostdb to disk. + Set the frequency (in seconds) to sync hostdb to disk. If set to zero (default as of v9.0.0), we won't + sync to disk ever. Note: hostdb is syncd to disk on a per-partition basis (of which there are 64). This means that the minimum time to sync all data to disk is :ts:cv:`proxy.config.cache.hostdb.sync_frequency` * 64 diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index de09e9187ef..b5f16befe28 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -59,7 +59,7 @@ static ink_time_t hostdb_hostfile_update_timestamp = 0; static char hostdb_filename[PATH_NAME_MAX] = DEFAULT_HOST_DB_FILENAME; int hostdb_max_count = DEFAULT_HOST_DB_SIZE; char hostdb_hostfile_path[PATH_NAME_MAX] = ""; -int hostdb_sync_frequency = 120; +int hostdb_sync_frequency = 0; int hostdb_disable_reverse_lookup = 0; ClassAllocator hostDBContAllocator("hostDBContAllocator"); diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index b8e5e53c77a..5da12f1a324 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -961,7 +961,7 @@ static const RecordElement RecordsConfig[] = {RECT_CONFIG, "proxy.config.hostdb.timed_round_robin", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , // # how often should the hostdb be synced (seconds) - {RECT_CONFIG, "proxy.config.cache.hostdb.sync_frequency", RECD_INT, "120", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.cache.hostdb.sync_frequency", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.hostdb.host_file.path", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , From 1b5c6fc454fe9b1a939313cecaf17ebb2f80fcb3 Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Mon, 28 Jan 2019 11:07:16 -0600 Subject: [PATCH 204/526] fix a segfault on mac --- iocore/eventsystem/I_Continuation.h | 2 +- iocore/eventsystem/P_Thread.h | 2 +- iocore/eventsystem/P_UnixEThread.h | 2 +- src/traffic_server/InkAPI.cc | 20 ++++++++++++++++++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/iocore/eventsystem/I_Continuation.h b/iocore/eventsystem/I_Continuation.h index abb1d7dc269..a7cceb6b5b6 100644 --- a/iocore/eventsystem/I_Continuation.h +++ b/iocore/eventsystem/I_Continuation.h @@ -145,7 +145,7 @@ class Continuation : private force_VFPT_to_top */ ContFlags control_flags; - EThread *thread_affinity = this_event_thread(); + EThread *thread_affinity = nullptr; bool setThreadAffinity(EThread *ethread) diff --git a/iocore/eventsystem/P_Thread.h b/iocore/eventsystem/P_Thread.h index 6e2a90771a0..f3030045b0c 100644 --- a/iocore/eventsystem/P_Thread.h +++ b/iocore/eventsystem/P_Thread.h @@ -45,5 +45,5 @@ Thread::set_specific() TS_INLINE Thread * this_thread() { - return (Thread *)ink_thread_getspecific(Thread::thread_data_key); + return static_cast(ink_thread_getspecific(Thread::thread_data_key)); } diff --git a/iocore/eventsystem/P_UnixEThread.h b/iocore/eventsystem/P_UnixEThread.h index 042757bfb33..cccda6bc6b6 100644 --- a/iocore/eventsystem/P_UnixEThread.h +++ b/iocore/eventsystem/P_UnixEThread.h @@ -176,7 +176,7 @@ EThread::schedule_spawn(Continuation *c, int ev, void *cookie) TS_INLINE EThread * this_ethread() { - return (EThread *)this_thread(); + return dynamic_cast(this_thread()); } TS_INLINE EThread * diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 55a8c132185..b7c540f0bc8 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -4413,7 +4413,8 @@ TSContSchedule(TSCont contp, TSHRTime timeout) EThread *eth = i->getThreadAffinity(); if (eth == nullptr) { - return nullptr; + eth = this_event_thread(); + i->setThreadAffinity(eth); } TSAction action; @@ -4441,6 +4442,10 @@ TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) ink_assert(!"not reached"); } + if (i->getThreadAffinity() == nullptr) { + i->setThreadAffinity(this_event_thread()); + } + EventType etype; switch (tp) { @@ -4495,6 +4500,9 @@ TSContScheduleOnThread(TSCont contp, TSHRTime timeout, TSEventThread ethread) } EThread *eth = reinterpret_cast(ethread); + if (i->getThreadAffinity() == nullptr) { + i->setThreadAffinity(eth); + } TSAction action; if (timeout == 0) { @@ -4523,7 +4531,8 @@ TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */) EThread *eth = i->getThreadAffinity(); if (eth == nullptr) { - return nullptr; + eth = this_event_thread(); + i->setThreadAffinity(eth); } TSAction action = reinterpret_cast(eth->schedule_every(i, HRTIME_MSECONDS(every))); @@ -4546,6 +4555,10 @@ TSContScheduleEveryOnPool(TSCont contp, TSHRTime every, TSThreadPool tp) ink_assert(!"not reached"); } + if (i->getThreadAffinity() == nullptr) { + i->setThreadAffinity(this_event_thread()); + } + EventType etype; switch (tp) { @@ -4583,6 +4596,9 @@ TSContScheduleEveryOnThread(TSCont contp, TSHRTime every /* millisecs */, TSEven } EThread *eth = reinterpret_cast(ethread); + if (i->getThreadAffinity() == nullptr) { + i->setThreadAffinity(eth); + } TSAction action = reinterpret_cast(eth->schedule_every(i, HRTIME_MSECONDS(every))); From 54fb242cb6a01ea21d3c0393d3296d2c0b8a12cf Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 28 Jan 2019 13:10:22 -0600 Subject: [PATCH 205/526] Clean up StrHeapDesc constructor. --- proxy/hdrs/HdrHeap.cc | 7 ------- proxy/hdrs/HdrHeap.h | 9 +++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index 27d4b3a7b5e..be065a2cf72 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -1170,13 +1170,6 @@ HdrStrHeap::expand(char *ptr, int old_size, int new_size) } } -StrHeapDesc::StrHeapDesc() -{ - m_heap_start = nullptr; - m_heap_len = 0; - m_locked = false; -} - struct StrTest { char *ptr; int len; diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index c1fe0b2bbe6..9737782f0bb 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -158,11 +158,12 @@ class HdrStrHeap : public RefCountObj }; struct StrHeapDesc { - StrHeapDesc(); + StrHeapDesc() = default; + Ptr m_ref_count_ptr; - char *m_heap_start; - int32_t m_heap_len; - bool m_locked; + char *m_heap_start = nullptr; + int32_t m_heap_len = 0; + bool m_locked = false; bool contains(const char *str) const From b9f29f0c096e95a621f022c6c30694fd949bd811 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 28 Jan 2019 16:31:10 -0600 Subject: [PATCH 206/526] Fix TSHttpTxnEffectiveUrlStringGet to correctly set the port when it is non-standard. --- proxy/hdrs/HTTP.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index dabbc4ec29e..dabd5841a66 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -1689,6 +1689,7 @@ class UrlPrintHack ink_assert(nullptr == ui->m_ptr_port); // shouldn't be set if not in URL. ui->m_ptr_port = m_port_buff; ui->m_len_port = snprintf(m_port_buff, sizeof(m_port_buff), "%d", hdr->m_port); + ui->m_port = hdr->m_port; m_port_modified_p = true; } else { m_port_modified_p = false; @@ -1712,6 +1713,7 @@ class UrlPrintHack if (m_port_modified_p) { ui->m_len_port = 0; ui->m_ptr_port = nullptr; + ui->m_port = 0; } if (m_host_modified_p) { ui->m_len_host = 0; From 7e429ad401fe96c3b2ad16d3e083ad39b50484c0 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Tue, 29 Jan 2019 14:34:11 +0800 Subject: [PATCH 207/526] Avoid reschedule HostDBContinuation::iterateEvent if there are no more buckets --- iocore/hostdb/HostDB.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index b5f16befe28..8c5ac69cfc5 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -1567,6 +1567,9 @@ HostDBContinuation::iterateEvent(int event, Event *e) } } current_iterate_pos++; + } + + if (current_iterate_pos < hostDB.refcountcache->partition_count()) { // And reschedule ourselves to pickup the next bucket after HOST_DB_RETRY_PERIOD. Debug("hostdb", "iterateEvent event=%d eventp=%p: completed current iteration %ld of %ld", event, e, current_iterate_pos, hostDB.refcountcache->partition_count()); From b39b0f7dd74b66dd11b7d62f49f3432a10dc7fbd Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Tue, 8 Jan 2019 20:33:53 +0000 Subject: [PATCH 208/526] Add normalization the URI before cdniuc validation --- plugins/experimental/uri_signing/Makefile.inc | 2 + plugins/experimental/uri_signing/common.h | 1 + plugins/experimental/uri_signing/jwt.c | 37 +- plugins/experimental/uri_signing/match.c | 2 +- plugins/experimental/uri_signing/match.h | 2 +- plugins/experimental/uri_signing/normalize.c | 382 ++++++++++++++++++ plugins/experimental/uri_signing/normalize.h | 20 + .../unit_tests/uri_signing_test.cc | 182 +++++++++ 8 files changed, 611 insertions(+), 17 deletions(-) create mode 100644 plugins/experimental/uri_signing/normalize.c create mode 100644 plugins/experimental/uri_signing/normalize.h diff --git a/plugins/experimental/uri_signing/Makefile.inc b/plugins/experimental/uri_signing/Makefile.inc index e9807baf678..864d2b48b38 100644 --- a/plugins/experimental/uri_signing/Makefile.inc +++ b/plugins/experimental/uri_signing/Makefile.inc @@ -23,6 +23,7 @@ experimental_uri_signing_uri_signing_la_SOURCES = \ experimental/uri_signing/jwt.c \ experimental/uri_signing/match.c \ experimental/uri_signing/parse.c \ + experimental/uri_signing/normalize.c \ experimental/uri_signing/timing.c experimental_uri_signing_uri_signing_la_LIBADD = @LIBJANSSON@ @LIBCJOSE@ @LIBPCRE@ -lm -lcrypto @@ -39,4 +40,5 @@ experimental_uri_signing_test_uri_signing_SOURCES = \ experimental/uri_signing/cookie.c \ experimental/uri_signing/config.c \ experimental/uri_signing/timing.c \ + experimental/uri_signing/normalize.c \ experimental/uri_signing/match.c diff --git a/plugins/experimental/uri_signing/common.h b/plugins/experimental/uri_signing/common.h index c9304085f58..10505fa8d8a 100644 --- a/plugins/experimental/uri_signing/common.h +++ b/plugins/experimental/uri_signing/common.h @@ -28,6 +28,7 @@ void PrintToStdErr(const char *fmt, ...); #else +#include "ts/ts.h" #define PluginDebug(...) TSDebug("uri_signing", PLUGIN_NAME " " __VA_ARGS__) #define PluginError(...) PluginDebug(__VA_ARGS__), TSError(PLUGIN_NAME " " __VA_ARGS__) diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index 997448bfb2e..a38565c5385 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -19,6 +19,7 @@ #include "common.h" #include "jwt.h" #include "match.h" +#include "normalize.h" #include "ts/ts.h" #include #include @@ -161,14 +162,26 @@ jwt_validate(struct jwt *jwt) bool jwt_check_uri(const char *cdniuc, const char *uri) { - static const char CONT_URI_STR[] = "uri"; - static const char CONT_URI_PATTERN_STR[] = "uri-pattern"; - static const char CONT_URI_REGEX_STR[] = "uri-regex"; + static const char CONT_URI_HASH_STR[] = "hash"; + static const char CONT_URI_REGEX_STR[] = "regex"; if (!cdniuc || !*cdniuc || !uri) { return false; } + /* Normalize the URI */ + int uri_ct = strlen(uri); + int buff_ct = uri_ct + 2; + int err; + char normal_uri[buff_ct]; + + memset(normal_uri, 0, buff_ct); + err = normalize_uri(uri, uri_ct, normal_uri, buff_ct); + + if (err) { + return false; + } + const char *kind = cdniuc, *container = cdniuc; while (*container && *container != ':') { ++container; @@ -179,23 +192,17 @@ jwt_check_uri(const char *cdniuc, const char *uri) ++container; size_t len = container - kind; - PluginDebug("Comparing with match kind \"%.*s\" on \"%s\" to \"%s\"", (int)len - 1, kind, container, uri); + PluginDebug("Comparing with match kind \"%.*s\" on \"%s\" to normalized URI \"%s\"", (int)len - 1, kind, container, normal_uri); switch (len) { - case sizeof CONT_URI_STR: - if (!strncmp(CONT_URI_STR, kind, len - 1)) { - return !strcmp(container, uri); - } - PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_STR, (int)len - 1, kind); - break; - case sizeof CONT_URI_PATTERN_STR: - if (!strncmp(CONT_URI_PATTERN_STR, kind, len - 1)) { - return match_glob(container, uri); + case sizeof CONT_URI_HASH_STR: + if (!strncmp(CONT_URI_HASH_STR, kind, len - 1)) { + return match_hash(container, normal_uri); } - PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_PATTERN_STR, (int)len - 1, kind); + PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_HASH_STR, (int)len - 1, kind); break; case sizeof CONT_URI_REGEX_STR: if (!strncmp(CONT_URI_REGEX_STR, kind, len - 1)) { - return match_regex(container, uri); + return match_regex(container, normal_uri); } PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_REGEX_STR, (int)len - 1, kind); break; diff --git a/plugins/experimental/uri_signing/match.c b/plugins/experimental/uri_signing/match.c index b665dc47bce..18fb31a3017 100644 --- a/plugins/experimental/uri_signing/match.c +++ b/plugins/experimental/uri_signing/match.c @@ -23,7 +23,7 @@ #include bool -match_glob(const char *needle, const char *haystack) +match_hash(const char *needle, const char *haystack) { return false; } diff --git a/plugins/experimental/uri_signing/match.h b/plugins/experimental/uri_signing/match.h index 92b906dbd35..38f3eb28e83 100644 --- a/plugins/experimental/uri_signing/match.h +++ b/plugins/experimental/uri_signing/match.h @@ -17,5 +17,5 @@ */ #include -bool match_glob(const char *needle, const char *haystack); +bool match_hash(const char *needle, const char *haystack); bool match_regex(const char *pattern, const char *uri); diff --git a/plugins/experimental/uri_signing/normalize.c b/plugins/experimental/uri_signing/normalize.c new file mode 100644 index 00000000000..e51411190a0 --- /dev/null +++ b/plugins/experimental/uri_signing/normalize.c @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "normalize.h" +#include "common.h" +#include +#include +#include +#include + +/* Remove Dot Algorithm outlined in RFC3986 section 5.2.4 + * Function writes normalizes path and writes to ret_buffer */ +int +remove_dot_segments(const char *path, int path_ct, char *ret_buffer, int buff_ct) +{ + /* Ensure buffer is at least the size of the path */ + if (buff_ct < path_ct) { + PluginDebug("Path buffer not large enough"); + return -1; + } + + /* Create an input buffer that we can change */ + char inBuff[path_ct + 1]; + memset(inBuff, 0, path_ct + 1); + strcpy(inBuff, path); + + const char *path_end = inBuff + path_ct; + char *seg_start = inBuff; + char *seg_end; + char *write_buffer = ret_buffer; + int seg_len; + + for (;;) { + if (seg_start == path_end) { + break; + } + seg_end = seg_start + 1; + + /* Parse such that Seg start/end contain the next full path segment */ + while (seg_end != path_end && *seg_end != '/') { + seg_end++; + } + + seg_len = seg_end - seg_start + 1; + + /* Remove starting ../ or ./ from input buffer */ + if (!strncmp(seg_start, "../", seg_len) || !strncmp(seg_start, "./", seg_len)) { + if (seg_end != path_end) { + seg_end++; + } + } + + /* Remove starting /./ or /. from input buffer and replace with '/' in output buffer */ + else if (!strncmp(seg_start, "/./", seg_len) || !strncmp(seg_start, "/.", seg_len)) { + *write_buffer = '/'; + write_buffer++; + if (seg_end != path_end) { + seg_end++; + } + } + + /* Replace /../ or /.. with / in write_buffer and remove preceding segment */ + else if (!strncmp(seg_start, "/../", seg_len) || !strncmp(seg_start, "/..", seg_len)) { + int prev_len = 0; + while (*write_buffer != '/' && write_buffer != ret_buffer) { + prev_len++; + write_buffer--; + } + memset(write_buffer, 0, prev_len); + + /* Replace segment with '/' in input buffer */ + if (seg_end != path_end) { + seg_start[seg_len - 1] = '/'; + } else { + seg_start[seg_len - 2] = '/'; + seg_end--; + } + } + + /* Remove starting '.' or '..' from input buffer */ + else if (!strncmp(seg_start, ".", seg_len) || !strncmp(seg_start, "..", seg_len)) { + if (seg_end != path_end) { + seg_end++; + } + } + /* Place the current path segment to the output buffer including initial '/' but not the next '/' */ + else { + /* Write first forward slash to buffer */ + if (*seg_start == '/') { + *write_buffer = *seg_start; + write_buffer++; + seg_start++; + } + + /* Write subsequent characters to buffer */ + while (*seg_start != '/') { + *write_buffer = *seg_start; + write_buffer++; + if (*seg_start == 0) { + break; + } + seg_start++; + } + } + seg_start = seg_end; + } + + PluginDebug("Normalized Path: %s", ret_buffer); + return strlen(ret_buffer); +} + +/* Function percent decodes uri_ct characters of the string uri and writes it to the decoded_uri + * buffer. If lower is true, it sets all characters including decoded ones to lower case. + * The function returns the length of the decoded string or -1 if there was a parsing error + * TODO: ADD functionality to ignore unicode non-standard characters and leave them encoded. Read RFC regarding normalization and + * determine if this is compliant. + */ +int +percent_decode(const char *uri, int uri_ct, char *decoded_uri, bool lower) +{ + static const char *reserved_string = ":/?#[]@!$&\'()*+,;="; + + if (uri_ct <= 0) { + return 0; + } + + int offset = 0; + int i; + for (i = 0; i < uri_ct; i++) { + if (uri[i] == '%') { + /* The next two characters are interpreted as the hex encoded value. Store in encodedVal */ + if (uri_ct < i + 2) { + goto decode_failure; + } + char encodedVal[2] = {0}; + int j; + for (j = 0; j < 2; j++) { + if (isxdigit(uri[i + j + 1])) { + encodedVal[j] = uri[i + j + 1]; + } else { + goto decode_failure; + } + } + int hexVal = 0; + char decodeChar; + sscanf(encodedVal, "%2x", &hexVal); + decodeChar = (char)hexVal; + /* If encoded value is a reserved char, leave encoded*/ + if (strchr(reserved_string, decodeChar)) { + decoded_uri[i - offset] = uri[i]; + decoded_uri[i + 1 - offset] = toupper(uri[i + 1]); + decoded_uri[i + 2 - offset] = toupper(uri[i + 2]); + } + /* If not a reserved char, decode using the decoded_uri buffer */ + else { + if (lower) { + decoded_uri[i - offset] = tolower(decodeChar); + } else { + decoded_uri[i - offset] = decodeChar; + } + offset = offset + 2; + } + i = i + 2; + } + /* Write non-encoded values to decoded buffer */ + else { + if (lower) { + decoded_uri[i - offset] = tolower(uri[i]); + } else { + decoded_uri[i - offset] = uri[i]; + } + } + } + + /* Return the size of the newly decoded string */ + return uri_ct - offset; + +decode_failure: + PluginDebug("ERROR Decoding URI"); + return -1; +} + +/* This function takes a uri and an initialized buffer to populate with the normalized uri. + * Returns non zero for error + * + * The buffer provided must be at least the length of the uri + 1 as the normalized uri will + * potentially be one char larger than the original uri if a backslash is added to the path. + * + * The normalization function returns a string with the following modifications + * 1. Lowecase protocol/domain + * 2. Path segments .. and . are removed from path + * 3. Alphabetical percent encoded octet values are toupper + * 4. Non-reserved percent encoded octet values are decoded + * 5. The Port is removed if it is default + * 6. Defaults to a single backslash for the path segment if path segment is empty + */ +int +normalize_uri(const char *uri, int uri_ct, char *normal_uri, int normal_ct) +{ + PluginDebug("Normalizing URI: %s", uri); + + /* Buffer provided must be large enough to store the uri plus one additional char */ + const char *uri_end = uri + uri_ct; + const char *buff_end = normal_uri + normal_ct; + + if (normal_uri && normal_ct < uri_ct + 1) { + PluginDebug("Buffer to Normalize URI not large enough."); + return -1; + } + + /* Initialize a path buffer to pass to path normalization function later on */ + char path_buffer[normal_ct]; + memset(path_buffer, 0, normal_ct); + + /* Comp variables store starting/ending indexes for each uri component as uri is parsed. + * Write buffer traverses the normalized uri buffer as we build the normalized string. + */ + const char *comp_start = uri; + const char *comp_end = uri; + char *write_buffer = normal_uri; + bool https = false; + + /* Parse the protocol which will end with a colon */ + while (*comp_end != ':' && comp_end != uri_end) { + *write_buffer = tolower(*comp_end); + comp_end++; + write_buffer++; + } + + if (comp_end == uri_end) { + PluginDebug("Reached End of String prematurely"); + goto normalize_failure; + } + + /* Copy the colon */ + *write_buffer = *comp_end; + comp_end++; + write_buffer++; + + /* Ensure the protocol is either http or https */ + if (strcmp("https:", normal_uri) == 0) { + https = true; + } else if (strcmp("http:", normal_uri)) { + PluginDebug("String is neither http or https"); + goto normalize_failure; + } + + /* Protocol must be terminated by two forward slashes */ + int i; + for (i = 0; i < 2; i++) { + if (comp_end == uri_end || *comp_end != '/') { + goto normalize_failure; + } + *write_buffer = *comp_end; + comp_end++; + write_buffer++; + } + + if (comp_end == uri_end) { + goto normalize_failure; + } + + /* Comp_start is index of start of authority component */ + int comp_ct; + comp_start = comp_end; + + /* Set comp start/end to contain authority component */ + bool userInfo = false; + while (comp_end != uri_end && *comp_end != '/' && *comp_end != '?' && *comp_end != '#') { + /* If we encounter userinfo, decode it without altering case and set comp_start/end to only include hostname/port */ + if (*comp_end == '@' && userInfo == false) { + comp_ct = comp_end - comp_start; + comp_ct = percent_decode(comp_start, comp_ct, write_buffer, false); + if (comp_ct < 0) { + goto normalize_failure; + } + comp_start = comp_end; + userInfo = true; + write_buffer = write_buffer + comp_ct; + } + comp_end++; + } + + /* UserInfo without a hostname is invalid */ + if (userInfo == true && comp_end == uri_end) { + goto normalize_failure; + } + + comp_ct = comp_end - comp_start; + + /* - comp start/end holds indices in original uri of hostname/port + * - write_buffer holds pointer to start of hostname/port written to the decode buffer + * - comp_ct holds size of hostname/port in original uri + */ + + /* Parse and decode the hostname and port and set to lower case */ + comp_ct = percent_decode(comp_start, comp_ct, write_buffer, true); + + if (comp_ct < 0) { + goto normalize_failure; + } + + /* Remove the port from the buffer if default */ + while (*write_buffer != 0) { + if (*write_buffer == ':') { + if (https == true && !strncmp(write_buffer, ":443", 5)) { + memset(write_buffer, 0, 4); + break; + } else if (https == false && !strncmp(write_buffer, ":80", 4)) { + memset(write_buffer, 0, 3); + break; + } + } + write_buffer++; + } + + comp_start = comp_end; + + /* If we have reached the end of the authority section with an empty path component, add a trailing backslash */ + if (*comp_end == 0 || *comp_end == '?' || *comp_end == '#') { + *write_buffer = '/'; + write_buffer++; + } + + /* If there is a path component, normalize it */ + else { + /* Set comp start/end pointers to contain the path component */ + while (*comp_end != '?' && *comp_end != '#' && *comp_end != 0) { + comp_end++; + } + /* Decode the path component without altering case and store it to the path_buffer*/ + comp_ct = comp_end - comp_start; + comp_ct = percent_decode(comp_start, comp_ct, path_buffer, false); + + if (comp_ct < 0) { + goto normalize_failure; + } + + /* Remove the . and .. segments from the path and write the now normalized path to the output buffer */ + PluginDebug("Removing Dot Segments"); + int buff_ct = buff_end - write_buffer; + comp_ct = remove_dot_segments(path_buffer, comp_ct, write_buffer, buff_ct); + + if (comp_ct < 0) { + PluginDebug("Failure removing dot segments from path"); + goto normalize_failure; + } + write_buffer = write_buffer + comp_ct; + } + + /* If there is any uri remaining after the path, decode and set case to lower */ + if (comp_end != uri_end) { + comp_start = comp_end; + comp_ct = uri_end - comp_start; + comp_ct = percent_decode(comp_start, comp_ct, write_buffer, false); + if (comp_ct < 0) { + goto normalize_failure; + } + } + + PluginDebug("Normalized URI: %s", normal_uri); + return 0; + +normalize_failure: + PluginDebug("URI Normalization Failure. URI does not fit http or https schemes."); + return -1; +} diff --git a/plugins/experimental/uri_signing/normalize.h b/plugins/experimental/uri_signing/normalize.h new file mode 100644 index 00000000000..a84c9979718 --- /dev/null +++ b/plugins/experimental/uri_signing/normalize.h @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +int normalize_uri(const char *uri, int uri_ct, char *uri_normal, int buffer_size); +int remove_dot_segments(const char *path, int path_ct, char *ret_buffer, int buff_ct); diff --git a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc index d89bf9c1898..30b8681aefc 100644 --- a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc +++ b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc @@ -27,6 +27,7 @@ extern "C" { #include #include #include "../jwt.h" +#include "../normalize.h" } bool @@ -48,6 +49,49 @@ jwt_parsing_helper(const char *jwt_string) return resp; } +bool +normalize_uri_helper(const char *uri, const char *expected_normal) +{ + size_t uri_ct = strlen(uri); + int buff_size = uri_ct + 2; + int err; + char uri_normal[buff_size]; + memset(uri_normal, 0, buff_size); + + err = normalize_uri(uri, uri_ct, uri_normal, buff_size); + + if (err) { + return false; + } + + if (expected_normal && strcmp(expected_normal, uri_normal) == 0) { + return true; + } + + return false; +} + +bool +remove_dot_helper(const char *path, const char *expected_path) +{ + fprintf(stderr, "Removing Dot Segments from Path: %s\n", path); + size_t path_ct = strlen(path); + path_ct++; + int new_ct; + char path_buffer[path_ct]; + memset(path_buffer, 0, path_ct); + + new_ct = remove_dot_segments(path, path_ct, path_buffer, path_ct); + + if (new_ct < 0) { + return false; + } else if (strcmp(expected_path, path_buffer) == 0) { + return true; + } else { + return false; + } +} + TEST_CASE("1", "[JWSParsingTest]") { INFO("TEST 1, Test JWT Parsing From Token Strings"); @@ -89,4 +133,142 @@ TEST_CASE("1", "[JWSParsingTest]") REQUIRE(!jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"cdnistd\":4,\"iss\":\"Content Access " "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); } + fprintf(stderr, "\n"); +} + +TEST_CASE("3", "[RemoveDotSegmentsTest]") +{ + INFO("TEST 3, Test Removal of Dot Segments From Paths"); + + SECTION("../bar test") { REQUIRE(remove_dot_helper("../bar", "bar")); } + + SECTION("./bar test") { REQUIRE(remove_dot_helper("./bar", "bar")); } + + SECTION(".././bar test") { REQUIRE(remove_dot_helper(".././bar", "bar")); } + + SECTION("./../bar test") { REQUIRE(remove_dot_helper("./../bar", "bar")); } + + SECTION("/foo/./bar test") { REQUIRE(remove_dot_helper("/foo/./bar", "/foo/bar")); } + + SECTION("/bar/./ test") { REQUIRE(remove_dot_helper("/bar/./", "/bar/")); } + + SECTION("/. test") { REQUIRE(remove_dot_helper("/.", "/")); } + + SECTION("/bar/. test") { REQUIRE(remove_dot_helper("/bar/.", "/bar/")); } + + SECTION("/foo/../bar test") { REQUIRE(remove_dot_helper("/foo/../bar", "/bar")); } + + SECTION("/bar/../ test") { REQUIRE(remove_dot_helper("/bar/../", "/")); } + + SECTION("/.. test") { REQUIRE(remove_dot_helper("/..", "/")); } + + SECTION("/bar/.. test") { REQUIRE(remove_dot_helper("/bar/..", "/")); } + + SECTION("/foo/bar/.. test") { REQUIRE(remove_dot_helper("/foo/bar/..", "/foo/")); } + + SECTION("Single . test") { REQUIRE(remove_dot_helper(".", "")); } + + SECTION("Single .. test") { REQUIRE(remove_dot_helper("..", "")); } + + SECTION("Test foo/bar/.. test") { REQUIRE(remove_dot_helper("foo/bar/..", "foo/")); } + + SECTION("Test Empty Path Segment") { REQUIRE(remove_dot_helper("", "")); } + + SECTION("Test mixed operations") { REQUIRE(remove_dot_helper("/foo/bar/././something/../foobar", "/foo/bar/foobar")); } + fprintf(stderr, "\n"); +} + +TEST_CASE("4", "[NormalizeTest]") +{ + INFO("TEST 4, Test Normalization of URIs"); + + SECTION("Testing passing too small of a URI to normalize") { REQUIRE(!normalize_uri_helper("ht", NULL)); } + + SECTION("Testing passing non http/https protocol") { REQUIRE(!normalize_uri_helper("ht:", NULL)); } + + SECTION("Passing a uri with half encoded value at end") { REQUIRE(!normalize_uri_helper("http://www.foobar.co%4", NULL)); } + + SECTION("Passing a uri with half encoded value in the middle") + { + REQUIRE(!normalize_uri_helper("http://www.foobar.co%4psomethin/Path", NULL)); + } + + SECTION("Passing a uri with an empty path parameter") + { + REQUIRE(normalize_uri_helper("http://www.foobar.com", "http://www.foobar.com/")); + } + + SECTION("Passing a uri with an empty path parameter and additional query params") + { + REQUIRE(normalize_uri_helper("http://www.foobar.com?query1=foo&query2=bar", "http://www.foobar.com/?query1=foo&query2=bar")); + } + + SECTION("Empty path parameter with port") + { + REQUIRE(normalize_uri_helper("http://www.foobar.com:9301?query1=foo&query2=bar", + "http://www.foobar.com:9301/?query1=foo&query2=bar")); + } + + SECTION("Passing a uri with a username and password") + { + REQUIRE(normalize_uri_helper("http://foo%40:PaSsword@www.Foo%42ar.coM:80/", "http://foo%40:PaSsword@www.foobar.com/")); + } + + SECTION("Testing Removal of standard http Port") + { + REQUIRE(normalize_uri_helper("http://foobar.com:80/Something/Here", "http://foobar.com/Something/Here")); + } + + SECTION("Testing Removal of standard https Port") + { + REQUIRE(normalize_uri_helper("https://foobar.com:443/Something/Here", "https://foobar.com/Something/Here")); + } + + SECTION("Testing passing of non-standard http Port") + { + REQUIRE(normalize_uri_helper("http://foobar.com:443/Something/Here", "http://foobar.com:443/Something/Here")); + } + + SECTION("Testing passing of non-standard https Port") + { + REQUIRE(normalize_uri_helper("https://foobar.com:80/Something/Here", "https://foobar.com:80/Something/Here")); + } + + SECTION("Testing the removal of . and .. in the path ") + { + REQUIRE( + normalize_uri_helper("https://foobar.com:80/Something/Here/././foobar/../foo", "https://foobar.com:80/Something/Here/foo")); + } + + SECTION("Testing . and .. segments in non path components") + { + REQUIRE(normalize_uri_helper("https://foobar.com:80/Something/Here?query1=/././foo/../bar", + "https://foobar.com:80/Something/Here?query1=/././foo/../bar")); + } + + SECTION("Testing standard decdoing of multiple characters") + { + REQUIRE(normalize_uri_helper("https://kelloggs%54ester.com/%53omething/Here", "https://kelloggstester.com/Something/Here")); + } + + SECTION("Testing passing encoded reserved characters") + { + REQUIRE( + normalize_uri_helper("https://kelloggs%54ester.com/%53omething/Here%3f", "https://kelloggstester.com/Something/Here%3F")); + } + + SECTION("Mixed Bag Test case") + { + REQUIRE(normalize_uri_helper("https://foo:something@kellogs%54ester.com:443/%53omething/.././here", + "https://foo:something@kellogstester.com/here")); + } + + SECTION("Testing empty hostname with userinfon") { REQUIRE(!normalize_uri_helper("https://foo:something@", NULL)); } + + SECTION("Testing empty uri after http://") { REQUIRE(!normalize_uri_helper("http://", NULL)); } + + SECTION("Testing http:///////") { REQUIRE(!normalize_uri_helper("http:///////", NULL)); } + + SECTION("Testing empty uri after http://?/") { REQUIRE(!normalize_uri_helper("http://?/", NULL)); } + fprintf(stderr, "\n"); } From 5f9d358b721fbe9b7660e23610d19632f7655503 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Tue, 15 Jan 2019 18:50:55 +0000 Subject: [PATCH 209/526] JWT Parser strips token from URI and places in buffer --- plugins/experimental/uri_signing/parse.c | 28 ++- plugins/experimental/uri_signing/parse.h | 3 +- .../unit_tests/uri_signing_test.cc | 174 ++++++++++++++++++ .../experimental/uri_signing/uri_signing.c | 8 +- 4 files changed, 208 insertions(+), 5 deletions(-) diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index 99ded7b71ab..099acdb315f 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -29,10 +29,11 @@ #include cjose_jws_t * -get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName) +get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName, char *strip_uri, size_t buff_ct, size_t *strip_ct) { /* Reserved characters as defined by the URI Generic Syntax RFC: https://tools.ietf.org/html/rfc3986#section-2.2 */ - const char *reserved_string = ":/?#[]@!$&\'()*+,;="; + static const char *reserved_string = ":/?#[]@!$&\'()*+,;="; + static const char *sub_delim_string = "!$&\'()*+,;="; /* If param name ends in reserved character this will be treated as the termination symbol when parsing for package. Default is * '='. */ @@ -94,6 +95,29 @@ get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName) PluginDebug("Unable to read JWS: %.*s, %s", (int)(key_end - key), key, err.message ? err.message : ""); } else { PluginDebug("Parsed JWS: %.*s (%16p)", (int)(key_end - key), key, jws); + + /* Strip token */ + /* Check that passed buffer is large enough */ + *strip_ct = ((key - uri) + (end - value_end)); + if (buff_ct <= *strip_ct) { + PluginDebug("Strip URI buffer is not large enough"); + return NULL; + } + + if (value_end != end && strchr(sub_delim_string, *value_end)) { + /*Strip from first char of package name to sub-delimeter that terminates the signed JWT */ + memcpy(strip_uri, uri, (key - uri)); + memcpy(strip_uri + (key - uri), value_end + 1, (end - value_end + 1)); + } else { + /*Strip from reserved char to the last char of the JWT */ + memcpy(strip_uri, uri, (key - uri - 1)); + memcpy(strip_uri + (key - uri - 1), value_end, (end - value_end)); + } + + if (strip_uri[*strip_ct - 1] != '\0') { + strip_uri[*strip_ct - 1] = '\0'; + } + PluginDebug("Stripped URI: %s", strip_uri); } return jws; } diff --git a/plugins/experimental/uri_signing/parse.h b/plugins/experimental/uri_signing/parse.h index 8d82c63ea7f..d05bfd59adb 100644 --- a/plugins/experimental/uri_signing/parse.h +++ b/plugins/experimental/uri_signing/parse.h @@ -19,7 +19,8 @@ #include struct _cjose_jws_int; -struct _cjose_jws_int *get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName); +struct _cjose_jws_int *get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName, char *strip_uri, size_t buff_ct, + size_t *strip_ct); struct _cjose_jws_int *get_jws_from_cookie(const char **cookie, size_t *cookie_ct, const char *paramName); struct config; diff --git a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc index 30b8681aefc..b879f7ce449 100644 --- a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc +++ b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc @@ -28,6 +28,7 @@ extern "C" { #include #include "../jwt.h" #include "../normalize.h" +#include "../parse.h" } bool @@ -92,6 +93,28 @@ remove_dot_helper(const char *path, const char *expected_path) } } +bool +jws_parsing_helper(const char *uri, const char *paramName, const char *expected_strip) +{ + bool resp; + size_t uri_ct = strlen(uri); + size_t strip_ct = 0; + char uri_strip[uri_ct + 1]; + memset(uri_strip, 0, sizeof uri_strip); + cjose_jws_t *jws = get_jws_from_uri(uri, uri_ct, paramName, uri_strip, uri_ct, &strip_ct); + if (jws) { + resp = true; + if (strcmp(uri_strip, expected_strip) != 0) { + cjose_jws_release(jws); + resp = false; + } + } else { + resp = false; + } + cjose_jws_release(jws); + return resp; +} + TEST_CASE("1", "[JWSParsingTest]") { INFO("TEST 1, Test JWT Parsing From Token Strings"); @@ -136,6 +159,157 @@ TEST_CASE("1", "[JWSParsingTest]") fprintf(stderr, "\n"); } +TEST_CASE("2", "[JWSFromURLTest]") +{ + INFO("TEST 2, Test JWT Parsing and Stripping From URLs"); + + SECTION("Token at end of URI") + { + REQUIRE(jws_parsing_helper( + "www.foo.com/hellothere/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "URISigningPackage", "www.foo.com/hellothere")); + } + + SECTION("No Token in URL") + { + REQUIRE(!jws_parsing_helper( + "www.foo.com/hellothere/" + "URISigningPackag=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "URISigningPackage", NULL)); + } + + SECTION("Token in middle of the URL") + { + REQUIRE(jws_parsing_helper("www.foo.com/hellothere/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", "www.foo.com/hellothere/Something/Else")); + } + + SECTION("Token at the start of the URL") + { + REQUIRE(jws_parsing_helper(":URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/www.foo.com/hellothere/Something/Else", + "URISigningPackage", "/www.foo.com/hellothere/Something/Else")); + } + + SECTION("Pass empty path parameter at end") + { + REQUIRE(!jws_parsing_helper("www.foobar.com/hellothere/URISigningPackage=", "URISigningPackage", NULL)); + } + + SECTION("Pass empty path parameter in the middle of URL") + { + REQUIRE(!jws_parsing_helper("www.foobar.com/hellothere/URISigningPackage=/Something/Else", "URISigningPackage", NULL)); + } + + SECTION("Partial package name in previous path parameter") + { + REQUIRE(jws_parsing_helper("www.foobar.com/URISig/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", "www.foobar.com/URISig/Something/Else")); + } + + SECTION("Package comes directly after two reserved characters") + { + REQUIRE(jws_parsing_helper("www.foobar.com/" + ":URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", "www.foobar.com//Something/Else")); + } + + SECTION("Package comes directly after string of reserved characters") + { + REQUIRE(jws_parsing_helper("www.foobar.com/?!/" + ":URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", "www.foobar.com/?!//Something/Else")); + } + + SECTION("Invalid token passed before a valid token") + { + REQUIRE(!jws_parsing_helper("www.foobar.com/URISigningPackage=/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", NULL)); + } + + SECTION("Empty string as URL") { REQUIRE(!jws_parsing_helper("", "URISigningPackage", NULL)); } + + SECTION("Empty package name to parser") + { + REQUIRE(!jws_parsing_helper( + "www.foobar.com/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "", NULL)); + } + + SECTION("Custom package name with a reserved character - at the end of the URI") + { + REQUIRE(jws_parsing_helper( + "www.foobar.com/CustomPackage/" + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "CustomPackage/", "www.foobar.com")); + } + + SECTION("Custom package name with a reserved character - in the middle of the URI") + { + REQUIRE(jws_parsing_helper( + "www.foobar.com/CustomPackage/" + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "CustomPackage/", "www.foobar.com/Something/Else")); + } + + SECTION("URI signing package passed as the only a query parameter") + { + REQUIRE(jws_parsing_helper( + "www.foobar.com/Something/" + "Here?URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "URISigningPackage", "www.foobar.com/Something/Here")); + } + + SECTION("URI signing package passed as first of many query parameters") + { + REQUIRE(jws_parsing_helper("www.foobar.com/Something/" + "Here?URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c&query3=foobar&query1=foo&query2=bar", + "URISigningPackage", "www.foobar.com/Something/Here?query3=foobar&query1=foo&query2=bar")); + } + + SECTION("URI signing package passed as one of many query parameters - passed in middle") + { + REQUIRE(jws_parsing_helper("www.foobar.com/Something/" + "Here?query1=foo&query2=bar&URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c&query3=foobar", + "URISigningPackage", "www.foobar.com/Something/Here?query1=foo&query2=bar&query3=foobar")); + } + + SECTION("URI signing package passed as last of many query parameters") + { + REQUIRE(jws_parsing_helper("www.foobar.com/Something/" + "Here?query1=foo&query2=bar&URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "URISigningPackage", "www.foobar.com/Something/Here?query1=foo&query2=bar")); + } +} + TEST_CASE("3", "[RemoveDotSegmentsTest]") { INFO("TEST 3, Test Removal of Dot Segments From Paths"); diff --git a/plugins/experimental/uri_signing/uri_signing.c b/plugins/experimental/uri_signing/uri_signing.c index 044731b98a4..fadd269e540 100644 --- a/plugins/experimental/uri_signing/uri_signing.c +++ b/plugins/experimental/uri_signing/uri_signing.c @@ -175,7 +175,11 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } - cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package); + + char strip_uri[2000] = {0}; + size_t strip_ct; + cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, strip_uri, 2000, &strip_ct); + if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } @@ -222,7 +226,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } - struct jwt *jwt = validate_jws(jws, (struct config *)ih, url, url_ct); + struct jwt *jwt = validate_jws(jws, (struct config *)ih, strip_uri, strip_ct); cjose_jws_release(jws); if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); From 23cdf765e3ca4ac15c7fdbe83ca67cc0a36bcdda Mon Sep 17 00:00:00 2001 From: Vijay Mamidi Date: Mon, 28 Jan 2019 21:45:34 -0800 Subject: [PATCH 210/526] Cached object's size can be used as content length for 304 response --- proxy/http/HttpTransact.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index aba6edcad4f..68ff019b31b 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -6622,7 +6622,13 @@ HttpTransact::handle_content_length_header(State *s, HTTPHdr *header, HTTPHdr *b TxnDebug("http_trans", "[handle_content_length_header] RESPONSE cont len in hdr is %" PRId64, header->get_content_length()); } else { // No content length header. - if (s->source == SOURCE_CACHE) { + // If the source is cache or server returned 304 response, + // we can try to get the content length based on object size. + // Also, we should check the scenario of server sending't a unexpected 304 response for a non conditional request( no cached + // object ) + if (s->source == SOURCE_CACHE || + (s->source == SOURCE_HTTP_ORIGIN_SERVER && s->hdr_info.server_response.status_get() == HTTP_STATUS_NOT_MODIFIED && + s->cache_info.object_read != nullptr)) { // If there is no content-length header, we can // insert one since the cache knows definitely // how long the object is unless we're in a @@ -6637,18 +6643,14 @@ HttpTransact::handle_content_length_header(State *s, HTTPHdr *header, HTTPHdr *b } else if (s->range_setup == RANGE_NOT_TRANSFORM_REQUESTED) { // if we are doing a single Range: request, calculate the new // C-L: header + // either the object is in cache or origin returned a 304 Not Modified response. We can still turn this into a proper Range + // response from the cached object. change_response_header_because_of_range_request(s, header); s->hdr_info.trust_response_cl = true; } else { header->set_content_length(cl); s->hdr_info.trust_response_cl = true; } - } else if (s->source == SOURCE_HTTP_ORIGIN_SERVER && s->hdr_info.server_response.status_get() == HTTP_STATUS_NOT_MODIFIED && - s->range_setup == RANGE_NOT_TRANSFORM_REQUESTED) { - // In this case, we had a cached object, possibly chunked encoded (so we don't have a CL: header), but the origin did a - // 304 Not Modified response. We can still turn this into a proper Range response from the cached object. - change_response_header_because_of_range_request(s, header); - s->hdr_info.trust_response_cl = true; } else { // Check to see if there is no content length // header because the response precludes a From 4f3b3c3b365737826d883784f9e33b7d287a1ee3 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Wed, 30 Jan 2019 22:21:57 +0900 Subject: [PATCH 211/526] Use one setting for H2 consistently for header size limit on H2 connections There were two settings, "proxy.config.http.request_header_max_size" and "proxy.config.http2.max_header_list_size", for the same purpose. --- mgmt/RecordsConfig.cc | 2 +- proxy/http2/HTTP2.cc | 4 +--- proxy/http2/HTTP2.h | 1 - proxy/http2/Http2ConnectionState.cc | 4 ++-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 5da12f1a324..8bd392fa0db 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1319,7 +1319,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.header_table_size", RECD_INT, "4096", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http2.max_header_list_size", RECD_INT, "4294967295", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.http2.max_header_list_size", RECD_INT, "131072", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http2.accept_no_activity_timeout", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index 1ce0a9dc824..55bd94bde85 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -614,7 +614,7 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ const char *value; int len; bool is_trailing_header = trailing_header; - int64_t result = hpack_decode_header_block(handle, hdr, buf_start, buf_len, Http2::max_request_header_size, maximum_table_size); + int64_t result = hpack_decode_header_block(handle, hdr, buf_start, buf_len, Http2::max_header_list_size, maximum_table_size); if (result < 0) { if (result == HPACK_ERROR_COMPRESSION_ERROR) { @@ -726,7 +726,6 @@ uint32_t Http2::initial_window_size = 1048576; uint32_t Http2::max_frame_size = 16384; uint32_t Http2::header_table_size = 4096; uint32_t Http2::max_header_list_size = 4294967295; -uint32_t Http2::max_request_header_size = 131072; uint32_t Http2::accept_no_activity_timeout = 120; uint32_t Http2::no_activity_timeout_in = 120; uint32_t Http2::active_timeout_in = 0; @@ -744,7 +743,6 @@ Http2::init() REC_EstablishStaticConfigInt32U(max_frame_size, "proxy.config.http2.max_frame_size"); REC_EstablishStaticConfigInt32U(header_table_size, "proxy.config.http2.header_table_size"); REC_EstablishStaticConfigInt32U(max_header_list_size, "proxy.config.http2.max_header_list_size"); - REC_EstablishStaticConfigInt32U(max_request_header_size, "proxy.config.http.request_header_max_size"); REC_EstablishStaticConfigInt32U(accept_no_activity_timeout, "proxy.config.http2.accept_no_activity_timeout"); REC_EstablishStaticConfigInt32U(no_activity_timeout_in, "proxy.config.http2.no_activity_timeout_in"); REC_EstablishStaticConfigInt32U(active_timeout_in, "proxy.config.http2.active_timeout_in"); diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index dcf895bf89c..b34474344d2 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -372,7 +372,6 @@ class Http2 static uint32_t max_frame_size; static uint32_t header_table_size; static uint32_t max_header_list_size; - static uint32_t max_request_header_size; static uint32_t accept_no_activity_timeout; static uint32_t no_activity_timeout_in; static uint32_t active_timeout_in; diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 4df6e856a4e..2587a89067d 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -237,7 +237,7 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // keep track of how many bytes we get in the frame stream->request_header_length += payload_length; - if (stream->request_header_length > Http2::max_request_header_size) { + if (stream->request_header_length > Http2::max_header_list_size) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv headers payload for headers greater than header length"); } @@ -800,7 +800,7 @@ rcv_continuation_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // keep track of how many bytes we get in the frame stream->request_header_length += payload_length; - if (stream->request_header_length > Http2::max_request_header_size) { + if (stream->request_header_length > Http2::max_header_list_size) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "continuation payload for headers exceeded"); } From ba0e6d0fa7cab89026b4c4e9182cdbfde0f0701e Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 31 Jan 2019 10:26:57 -0600 Subject: [PATCH 212/526] Update overridable config conversion logic. * Improve the constructors for the converter. * Better naming of the converter members. * Add some documentation for the converter implementation. --- CMakeLists.txt | 2 +- doc/developer-guide/config-vars.en.rst | 26 ++++++++ mgmt/MgmtDefs.h | 89 ++++++++++++++++++++++---- proxy/http/HttpConnectionCount.cc | 8 +-- src/traffic_server/InkAPI.cc | 54 ++++++---------- 5 files changed, 122 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9eb3dd032af..efaa9f8f067 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,4 +122,4 @@ file(GLOB plugin_files ) add_library(plugins SHARED ${plugin_files}) -add_custom_target(clang-format WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} COMMAND make clang-format) +add_custom_target(clang-format WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} COMMAND make -j clang-format) diff --git a/doc/developer-guide/config-vars.en.rst b/doc/developer-guide/config-vars.en.rst index 5e8585182c5..5fa88084e7d 100644 --- a/doc/developer-guide/config-vars.en.rst +++ b/doc/developer-guide/config-vars.en.rst @@ -322,3 +322,29 @@ required for generic access: #. Update the Lua plugin enumeration ``TSLuaOverridableConfigKey`` in |ts_lua_http_config.c|_. #. Update the documentation of :ref:`ts-overridable-config` in |TSHttpOverridableConfig.en.rst|_. + +API conversions +--------------- + +A relatively new feature for overridable variables is the ability to keep them in more natural data +types and convert as needed to the API types. This in turns enables defining the configuration +locally in a module and then "exporting" it to the API interface. Modules then do not have to +include headers for all types in all overridable configurations. + +The conversion is done through an instance of :code:`MgmtConverter`. This has 6 points to +conversions, a load and store function for each of the types :code:`MgmtInt`, :code:`MgmtFloat`, and +:code:`MgmtInt`. The :code:`MgmtByte` type is handled by the :code:`MgmtInt` conversions. In general +each overridable variable will specify two of these, a load and store for a specific type, although +it is possible to provide other pairs, e.g. if a value is an enumeration can should be settable +as a string as well as an integer. + +The module is responsible for creating an instance of :code:`MgmtConverter` with the appropriate +load / store function pairs set. The declaration must be visible in the :ts:git:`proxy/InkAPI.cc` +file. The function :code:`_conf_to_memberp` sets up the conversion. For the value of the enumeration +:c:type:`TSOverridableConfigKey` that specifies the overridable variable, code is added to specify +the member and the conversion. There are default converters for the API types and if the overridable +is one of those, it is only necessary to call :code:`_memberp_to_generic` passing in a pointer to +the variable. For a variable with conversion, :arg:`ret` should be set to point to the variable and +:arg:`conv` set to point to the converter for that variable. If multiple variables are of the same +type they can use the same converter because a pointer to the specific member is passed to the +converter. diff --git a/mgmt/MgmtDefs.h b/mgmt/MgmtDefs.h index 84424c43c9f..ed8f681ccc7 100644 --- a/mgmt/MgmtDefs.h +++ b/mgmt/MgmtDefs.h @@ -50,7 +50,7 @@ typedef enum { * MgmtCallback * Management Callback functions. */ -typedef void *(*MgmtCallback)(void *opaque_cb_data, char *data_raw, int data_len); +using MgmtCallback = void *(*)(void *opaque_cb_data, char *data_raw, int data_len); //------------------------------------------------------------------------- // API conversion functions. @@ -66,19 +66,80 @@ typedef void *(*MgmtCallback)(void *opaque_cb_data, char *data_raw, int data_len * in this header. */ struct MgmtConverter { - // MgmtInt conversions. - std::function get_int{nullptr}; - std::function set_int{nullptr}; - - // MgmtFloat conversions. - std::function get_float{nullptr}; - std::function set_float{nullptr}; - - // MgmtString conversions. - // This is a bit different because it takes std::string_view instead of MgmtString but that's - // worth the difference. - std::function get_string{nullptr}; - std::function set_string{nullptr}; + /** Load a native type into a @c MgmtInt + * + * This is passed a @c void* which is a pointer to the member in the configuration instance. + * This function must return a @c MgmtInt converted from that value. + */ + MgmtInt (*load_int)(void *) = nullptr; + + /** Store a @c MgmtInt into a native type. + * + * This function is passed a @c void* which is a pointer to the member in the configuration + * instance and a @c MgmtInt. The member should be updated to correspond to the @c MgmtInt value. + */ + void (*store_int)(void *, MgmtInt) = nullptr; + + /** Load a @c MgmtFloat from a native type. + * + * This is passed a @c void* which is a pointer to the member in the configuration instance. + * This function must return a @c MgmtFloat converted from that value. + */ + MgmtFloat (*load_float)(void *) = nullptr; + + /** Store a @c MgmtFloat into a native type. + * + * This function is passed a @c void* which is a pointer to the member in the configuration + * instance and a @c MgmtFloat. The member should be updated to correspond to the @c MgmtFloat value. + */ + void (*store_float)(void *, MgmtFloat) = nullptr; + + /** Load a native type into view. + * + * This is passed a @c void* which is a pointer to the member in the configuration instance. + * This function must return a @c string_view which contains the text for the member. + */ + std::string_view (*load_string)(void *) = nullptr; + + /** Store a view in a native type. + * + * This is passed a @c void* which is a pointer to the member in the configuration instance. + * This function must return a @c string_view which contains the text for the member. + */ + void (*store_string)(void *, std::string_view) = nullptr; + + // Convenience constructors because generally only one pair is valid. + MgmtConverter(MgmtInt (*load)(void *), void (*store)(void *, MgmtInt)); + MgmtConverter(MgmtFloat (*load)(void *), void (*store)(void *, MgmtFloat)); + MgmtConverter(std::string_view (*load)(void *), void (*store)(void *, std::string_view)); + + MgmtConverter(MgmtInt (*_load_int)(void *), void (*_store_int)(void *, MgmtInt), MgmtFloat (*_load_float)(void *), + void (*_store_float)(void *, MgmtFloat), std::string_view (*_load_string)(void *), + void (*_store_string)(void *, std::string_view)); }; +inline MgmtConverter::MgmtConverter(MgmtInt (*load)(void *), void (*store)(void *, MgmtInt)) : load_int(load), store_int(store) {} + +inline MgmtConverter::MgmtConverter(MgmtFloat (*load)(void *), void (*store)(void *, MgmtFloat)) + : load_float(load), store_float(store) +{ +} + +inline MgmtConverter::MgmtConverter(std::string_view (*load)(void *), void (*store)(void *, std::string_view)) + : load_string(load), store_string(store) +{ +} + +inline MgmtConverter::MgmtConverter(MgmtInt (*_load_int)(void *), void (*_store_int)(void *, MgmtInt), + MgmtFloat (*_load_float)(void *), void (*_store_float)(void *, MgmtFloat), + std::string_view (*_load_string)(void *), void (*_store_string)(void *, std::string_view)) + : load_int(_load_int), + store_int(_store_int), + load_float(_load_float), + store_float(_store_float), + load_string(_load_string), + store_string(_store_string) +{ +} + #define LM_CONNECTION_SERVER "processerver.sock" diff --git a/proxy/http/HttpConnectionCount.cc b/proxy/http/HttpConnectionCount.cc index 3b596354312..807f6c9bebe 100644 --- a/proxy/http/HttpConnectionCount.cc +++ b/proxy/http/HttpConnectionCount.cc @@ -33,13 +33,9 @@ OutboundConnTrack::Imp OutboundConnTrack::_imp; OutboundConnTrack::GlobalConfig *OutboundConnTrack::_global_config{nullptr}; -const MgmtConverter OutboundConnTrack::MAX_CONV{ +const MgmtConverter OutboundConnTrack::MAX_CONV( [](void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }, - nullptr, - nullptr, - nullptr, - nullptr}; + [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }); // Do integer and string conversions. const MgmtConverter OutboundConnTrack::MATCH_CONV{ diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index b7c540f0bc8..dd8f96ca24a 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -7971,32 +7971,14 @@ template inline void * _memberp_to_generic(T *ptr, MgmtConverter const *&conv) { - static const MgmtConverter IntConverter{ - [](void *data) -> MgmtInt { return *static_cast(data); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }, - nullptr, - nullptr, // float - nullptr, - nullptr // string - }; + static const MgmtConverter IntConverter([](void *data) -> MgmtInt { return *static_cast(data); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }); - static const MgmtConverter ByteConverter{ - [](void *data) -> MgmtInt { return *static_cast(data); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }, - nullptr, - nullptr, // float - nullptr, - nullptr // string - }; + static const MgmtConverter ByteConverter{[](void *data) -> MgmtInt { return *static_cast(data); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }}; - static const MgmtConverter FloatConverter{ - nullptr, // int - nullptr, - [](void *data) -> MgmtFloat { return *static_cast(data); }, - [](void *data, MgmtFloat f) -> void { *static_cast(data) = f; }, - nullptr, - nullptr // string - }; + static const MgmtConverter FloatConverter{[](void *data) -> MgmtFloat { return *static_cast(data); }, + [](void *data, MgmtFloat f) -> void { *static_cast(data) = f; }}; // For now, strings are special. @@ -8398,11 +8380,11 @@ TSHttpTxnConfigIntSet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtInt val void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); - if (!dest || !conv->set_int) { + if (!dest || !conv->store_int) { return TS_ERROR; } - conv->set_int(dest, value); + conv->store_int(dest, value); return TS_SUCCESS; } @@ -8417,11 +8399,11 @@ TSHttpTxnConfigIntGet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtInt *va MgmtConverter const *conv; void *src = _conf_to_memberp(conf, s->t_state.txn_conf, conv); - if (!src || !conv->get_int) { + if (!src || !conv->load_int) { return TS_ERROR; } - *value = conv->get_int(src); + *value = conv->load_int(src); return TS_SUCCESS; } @@ -8438,11 +8420,11 @@ TSHttpTxnConfigFloatSet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtFloat void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); - if (!dest || !conv->set_float) { + if (!dest || !conv->store_float) { return TS_ERROR; } - conv->set_float(dest, value); + conv->store_float(dest, value); return TS_SUCCESS; } @@ -8456,10 +8438,10 @@ TSHttpTxnConfigFloatGet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtFloat MgmtConverter const *conv; void *src = _conf_to_memberp(conf, reinterpret_cast(txnp)->t_state.txn_conf, conv); - if (!src || !conv->get_float) { + if (!src || !conv->load_float) { return TS_ERROR; } - *value = conv->get_float(src); + *value = conv->load_float(src); return TS_SUCCESS; } @@ -8552,8 +8534,8 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char default: { MgmtConverter const *conv; void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); - if (dest != nullptr && conv != nullptr && conv->set_string) { - conv->set_string(dest, std::string_view(value, length)); + if (dest != nullptr && conv != nullptr && conv->store_string) { + conv->store_string(dest, std::string_view(value, length)); } else { return TS_ERROR; } @@ -8589,8 +8571,8 @@ TSHttpTxnConfigStringGet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char default: { MgmtConverter const *conv; void *src = _conf_to_memberp(conf, sm->t_state.txn_conf, conv); - if (src != nullptr && conv != nullptr && conv->get_string) { - auto sv = conv->get_string(src); + if (src != nullptr && conv != nullptr && conv->load_string) { + auto sv = conv->load_string(src); *value = sv.data(); *length = sv.size(); } else { From 7eb4b24787ac15a7502be80002eb01ef0c0e0dda Mon Sep 17 00:00:00 2001 From: Pushkar Pradhan Date: Thu, 31 Jan 2019 15:24:30 -0800 Subject: [PATCH 213/526] Adding back the cookie_remap unit tests. Running cookie_remap autests conditionally only if the plugin is built. --- plugins/experimental/cookie_remap/Makefile.inc | 10 ++++++++++ .../cookiejar/unit_test.cc => test_cookiejar.cc} | 0 .../pluginTest/cookie_remap/bucketcookie.test.py | 1 + .../pluginTest/cookie_remap/collapseslashes.test.py | 1 + .../pluginTest/cookie_remap/connector.test.py | 1 + .../pluginTest/cookie_remap/existscookie.test.py | 1 + .../pluginTest/cookie_remap/matchcookie.test.py | 1 + .../pluginTest/cookie_remap/matchuri.test.py | 1 + .../pluginTest/cookie_remap/matrixparams.test.py | 1 + .../pluginTest/cookie_remap/notexistscookie.test.py | 1 + .../pluginTest/cookie_remap/regexcookie.test.py | 1 + .../pluginTest/cookie_remap/setstatus.test.py | 1 + .../pluginTest/cookie_remap/subcookie.test.py | 1 + .../pluginTest/cookie_remap/substitute.test.py | 1 + 14 files changed, 22 insertions(+) rename plugins/experimental/cookie_remap/{unit_tests/cookiejar/unit_test.cc => test_cookiejar.cc} (100%) diff --git a/plugins/experimental/cookie_remap/Makefile.inc b/plugins/experimental/cookie_remap/Makefile.inc index 11de3ac0104..4af577dd3d4 100644 --- a/plugins/experimental/cookie_remap/Makefile.inc +++ b/plugins/experimental/cookie_remap/Makefile.inc @@ -27,4 +27,14 @@ experimental_cookie_remap_cookie_remap_la_LDFLAGS = \ AM_CPPFLAGS += @YAMLCPP_INCLUDES@ +check_PROGRAMS += \ + experimental/cookie_remap/test_cookiejar + +experimental_cookie_remap_test_cookiejar_CPPFLAGS = $(AM_CPPFLAGS) -Iexperimental/cookie_remap -I$(abs_top_srcdir)/tests/include +experimental_cookie_remap_test_cookiejar_SOURCES = \ + experimental/cookie_remap/test_cookiejar.cc \ + experimental/cookie_remap/strip.c \ + experimental/cookie_remap/cookiejar.cc \ + experimental/cookie_remap/cookiejar.h + # vim: ft=make ts=8 sw=8 et: diff --git a/plugins/experimental/cookie_remap/unit_tests/cookiejar/unit_test.cc b/plugins/experimental/cookie_remap/test_cookiejar.cc similarity index 100% rename from plugins/experimental/cookie_remap/unit_tests/cookiejar/unit_test.cc rename to plugins/experimental/cookie_remap/test_cookiejar.cc diff --git a/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py index 4e1863a6ccd..86d4e50be16 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py index 7cc647f0d0b..7f6db2abb44 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/connector.test.py b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py index 378b880dbeb..2f7c18ee219 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/connector.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py index 6f07b02631c..43598a7bfdc 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py index bafb0ef76ff..cad59810ec3 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py b/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py index 706f353b441..bb968144372 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py index 0d8ca8dec82..68529ba9fcb 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py index 6028ec8e004..c645ca58f1f 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py index bf0c594a614..ad78ec8b76c 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py b/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py index 34c0b344492..bcbeb3ba2b6 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py index 07166891ab9..f8d3d17fdd6 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") diff --git a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py index 7a5dfe5ac4b..273016c160f 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py @@ -20,6 +20,7 @@ Test.Summary = ''' ''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) # need Curl Test.SkipUnless( Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") From dfd5d8f4533fadf1ca15834e0c2cdde06f96453e Mon Sep 17 00:00:00 2001 From: Aaron Canary Date: Wed, 30 Jan 2019 15:10:54 -0600 Subject: [PATCH 214/526] Rename Session and Transaction classes ProxyClientSession -> ProxySession ProxyClientTransaction -> ProxyTransaction HttpServerSession -> Http1ServerSession HttpClientTransaction -> Http1Transaction make files clang-format fixed make docs --- .../client-session-architecture.en.rst | 32 +++++------ include/tscore/IntrusiveHashMap.h | 8 +-- iocore/eventsystem/I_Thread.h | 2 +- proxy/Makefile.am | 8 +-- ...{ProxyClientSession.cc => ProxySession.cc} | 26 ++++----- .../{ProxyClientSession.h => ProxySession.h} | 21 +++---- ...ientTransaction.cc => ProxyTransaction.cc} | 20 +++---- ...ClientTransaction.h => ProxyTransaction.h} | 22 ++++---- proxy/http/Http1ClientSession.cc | 12 ++-- proxy/http/Http1ClientSession.h | 23 ++++---- ...ServerSession.cc => Http1ServerSession.cc} | 30 +++++----- ...tpServerSession.h => Http1ServerSession.h} | 52 ++++++++--------- ...ientTransaction.cc => Http1Transaction.cc} | 12 ++-- ...ClientTransaction.h => Http1Transaction.h} | 16 +++--- proxy/http/HttpSM.cc | 22 ++++---- proxy/http/HttpSM.h | 20 +++---- proxy/http/HttpSessionManager.cc | 20 +++---- proxy/http/HttpSessionManager.h | 19 +++---- proxy/http/HttpTransact.h | 2 +- proxy/http/HttpTunnel.cc | 2 +- proxy/http/Makefile.am | 8 +-- proxy/http2/Http2ClientSession.cc | 2 +- proxy/http2/Http2ClientSession.h | 10 ++-- proxy/http2/Http2Stream.h | 6 +- src/traffic_server/InkAPI.cc | 56 +++++++++---------- src/traffic_server/traffic_server.cc | 2 +- 26 files changed, 229 insertions(+), 224 deletions(-) rename proxy/{ProxyClientSession.cc => ProxySession.cc} (89%) rename proxy/{ProxyClientSession.h => ProxySession.h} (92%) rename proxy/{ProxyClientTransaction.cc => ProxyTransaction.cc} (82%) rename proxy/{ProxyClientTransaction.h => ProxyTransaction.h} (92%) rename proxy/http/{HttpServerSession.cc => Http1ServerSession.cc} (86%) rename proxy/http/{HttpServerSession.h => Http1ServerSession.h} (82%) rename proxy/http/{Http1ClientTransaction.cc => Http1Transaction.cc} (88%) rename proxy/http/{Http1ClientTransaction.h => Http1Transaction.h} (88%) diff --git a/doc/developer-guide/client-session-architecture.en.rst b/doc/developer-guide/client-session-architecture.en.rst index 7df36aa9b68..6f1f36f82ce 100644 --- a/doc/developer-guide/client-session-architecture.en.rst +++ b/doc/developer-guide/client-session-architecture.en.rst @@ -27,34 +27,34 @@ The User Agent interacts with ATS by creating a session with the ATS server and submitting sequences of requests over the session. ATS supports several session protocols including HTTP/1.x and HTTP/2. A HTTP State Machine is created for each request to process the request. -ATS uses the generic classes ProxyClientSession and ProxyClientTransaction to hide the details of +ATS uses the generic classes ProxySession and ProxyTransaction to hide the details of the underlaying protocols from the HTTP State Machine. Classes ======= -ProxyClientSession +ProxySession ------------------ .. figure:: /static/images/sessions/session_hierarchy.png :align: center - :alt: ProxyClientSession hierarchy + :alt: ProxySession hierarchy -The ProxyClientSession class abstracts the key features of a client session. It contains zero or more ProxyClientTransaction objects. It also has a reference to the associated NetVC (either UnixNetVConnection or SSLNetVConnection). The session class is responsible for interfacing with the user agent protocol. +The ProxySession class abstracts the key features of a client session. It contains zero or more ProxyTransaction objects. It also has a reference to the associated NetVC (either UnixNetVConnection or SSLNetVConnection). The session class is responsible for interfacing with the user agent protocol. At this point there are two concrete subclasses: Http1ClientSession and Http2ClientSession. The Http1ClientSession only has at most one transaction active at a time. The HTTP/2 protocol allows for multiple simultaneous active transactions -ProxyClientTransaction +ProxyTransaction ---------------------- .. figure:: /static/images/sessions/transaction_hierarchy.png :align: center - :alt: ProxyClientTransaction hierarchy + :alt: ProxyTransaction hierarchy -The ProxyClientTransaction class abstracts the key features of a client transaction. It has a reference to its -paraent ProxyClientSession. One HttpSM is created for each ProxyClientTransaction. +The ProxyTransaction class abstracts the key features of a client transaction. It has a reference to its +paraent ProxySession. One HttpSM is created for each ProxyTransaction. There are two concrete subclasses: Http1ClientTransaction and Http2Stream. @@ -70,18 +70,18 @@ HTTP/1.x Objects This diagram shows the relationships between objects created as part of a HTTP/1.x session. A NetVC object performs the basic network level protocols. The Http1ClientSession object has a reference to the -associated NetVC object. The NetVC object is available via the :code:`ProxyClientSession::get_netvc()` method. +associated NetVC object. The NetVC object is available via the :code:`ProxySession::get_netvc()` method. The Http1ClientSession object contains a Http1ClientTransaction objet. For each HTTP request, it calls -the :code:`ProxyClientSession::new_transaction()` method to instantiate the Http1ClientTransaction object. With the HTTP/1.x +the :code:`ProxySession::new_transaction()` method to instantiate the Http1ClientTransaction object. With the HTTP/1.x protocol at most one transaction can be active at a time. -When the Http1ClientTransaction object is instantiated via :code:`ProxyClientTransaction::new_transaction()` it allocates a +When the Http1ClientTransaction object is instantiated via :code:`ProxyTransaction::new_transaction()` it allocates a new HttpSM object, initializes it, and calls :code:`HttpSM::attach_client_session()` to associate the Http1ClientTransaction object with the new HttpSM. -The ProxyClientTransaction object refers to the HttpSM via the current_reader member variable. The HttpSM object -refers to ProxyClientTransaction via the ua_session member variable (session in the member name is +The ProxyTransaction object refers to the HttpSM via the current_reader member variable. The HttpSM object +refers to ProxyTransaction via the ua_session member variable (session in the member name is historical because the HttpSM used to refer directly to the ClientSession object). HTTP/2 Objects @@ -93,11 +93,11 @@ HTTP/2 Objects This diagram shows the relationships between objects created as part of a HTTP/2 session. It is very similar to the HTTP/1.x case. The Http2ClientSession object interacts with the NetVC. The Http2Stream object creates -a HttpSM object object when :code:`ProxyClient::new_transaction()` is called. +a HttpSM object object when :code:`ProxySession::new_transaction()` is called. One difference is that the Http/2 protocol allows for multiple simultaneous transactions, so the Http2ClientSession object must be able to manage multiple streams. From the HttpSM perspective it is interacting with a -ProxyClientTransaction object, and there is no difference between working with a Http2Stream and a Http1ClientTransaction. +ProxyTransaction object, and there is no difference between working with a Http2Stream and a Http1ClientTransaction. Transaction and Session Shutdown ================================ @@ -114,5 +114,5 @@ cause use-after-free and other related memory corruption errors. To ensure that sessions and transactions are correctly shutdown the following assertions are maintained. * The Session object will not call :code:`::destroy()` on itself until all child transaction objects are fully shutdown (i.e. TXN_CLOSE hooks are called and the transaction objects have been freed). -* The Transaction object will not call :code:`::destroy()` on itself until the associated HttpSM has been shutdown. In :code:`HttpSM::kill_this()`, the HttpSM will call :code:`ProxyClientTransaction::transaction_done()` on the ua_session object. If the user agent initiates the termination, the ProxyClientTransaction object will send a WRITE_COMPLETE, EOS, or ERROR event on the open VIO object. This should signal to the HttpSM object to shut itself down. +* The Transaction object will not call :code:`::destroy()` on itself until the associated HttpSM has been shutdown. In :code:`HttpSM::kill_this()`, the HttpSM will call :code:`ProxyTransaction::transaction_done()` on the ua_session object. If the user agent initiates the termination, the ProxyTransaction object will send a WRITE_COMPLETE, EOS, or ERROR event on the open VIO object. This should signal to the HttpSM object to shut itself down. diff --git a/include/tscore/IntrusiveHashMap.h b/include/tscore/IntrusiveHashMap.h index c787e4b358b..8a44738a74a 100644 --- a/include/tscore/IntrusiveHashMap.h +++ b/include/tscore/IntrusiveHashMap.h @@ -67,15 +67,15 @@ @a ID The numeric type that is the hash value for an instance of @a Key. - Example for @c HttpServerSession keyed by the origin server IP address. + Example for @c Http1ServerSession keyed by the origin server IP address. @code struct Descriptor { - static sockaddr const* key_of(HttpServerSession const* value) { return &value->ip.sa } + static sockaddr const* key_of(Http1ServerSession const* value) { return &value->ip.sa } static bool equal(sockaddr const* lhs, sockaddr const* rhs) { return ats_ip_eq(lhs, rhs); } static uint32_t hash_of(sockaddr const* key) { return ats_ip_hash(key); } - static HttpServerSession *& next_ptr(HttpServerSession * ssn) { return ssn->_next; } - static HttpServerSession *& prev_ptr(HttpServerSession * ssn) { return ssn->_prev; } + static Http1ServerSession *& next_ptr(Http1ServerSession * ssn) { return ssn->_next; } + static Http1ServerSession *& prev_ptr(Http1ServerSession * ssn) { return ssn->_prev; } }; using Table = IntrusiveHashMap; @endcode diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index 0545d6cde9a..ac8d661cb4b 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -120,7 +120,7 @@ class Thread ProxyAllocator http1ClientSessionAllocator; ProxyAllocator http2ClientSessionAllocator; ProxyAllocator http2StreamAllocator; - ProxyAllocator httpServerSessionAllocator; + ProxyAllocator http1ServerSessionAllocator; ProxyAllocator hdrHeapAllocator; ProxyAllocator strHeapAllocator; ProxyAllocator cacheVConnectionAllocator; diff --git a/proxy/Makefile.am b/proxy/Makefile.am index 0bd93d99822..78f31972f3c 100644 --- a/proxy/Makefile.am +++ b/proxy/Makefile.am @@ -59,10 +59,10 @@ libproxy_a_SOURCES = \ PluginVC.h \ ProtocolProbeSessionAccept.cc \ ProtocolProbeSessionAccept.h \ - ProxyClientSession.cc \ - ProxyClientSession.h \ - ProxyClientTransaction.cc \ - ProxyClientTransaction \ + ProxySession.cc \ + ProxySession.h \ + ProxyTransaction.cc \ + ProxyTransaction.h \ ReverseProxy.cc \ ReverseProxy.h \ StatPages.cc \ diff --git a/proxy/ProxyClientSession.cc b/proxy/ProxySession.cc similarity index 89% rename from proxy/ProxyClientSession.cc rename to proxy/ProxySession.cc index 6e4857bf9aa..be78f6bc4ab 100644 --- a/proxy/ProxyClientSession.cc +++ b/proxy/ProxySession.cc @@ -1,6 +1,6 @@ /** @file - ProxyClientSession - Base class for protocol client sessions. + ProxySession - Base class for protocol client & server sessions. @section license License @@ -23,17 +23,17 @@ #include "HttpConfig.h" #include "HttpDebugNames.h" -#include "ProxyClientSession.h" +#include "ProxySession.h" static int64_t next_cs_id = 0; -ProxyClientSession::ProxyClientSession() : VConnection(nullptr) +ProxySession::ProxySession() : VConnection(nullptr) { ink_zero(this->user_args); } void -ProxyClientSession::set_session_active() +ProxySession::set_session_active() { if (!m_active) { m_active = true; @@ -42,7 +42,7 @@ ProxyClientSession::set_session_active() } void -ProxyClientSession::clear_session_active() +ProxySession::clear_session_active() { if (m_active) { m_active = false; @@ -51,7 +51,7 @@ ProxyClientSession::clear_session_active() } int64_t -ProxyClientSession::next_connection_id() +ProxySession::next_connection_id() { return ink_atomic_increment(&next_cs_id, 1); } @@ -84,7 +84,7 @@ is_valid_hook(TSHttpHookID hookid) } void -ProxyClientSession::free() +ProxySession::free() { if (schedule_event) { schedule_event->cancel(); @@ -96,7 +96,7 @@ ProxyClientSession::free() } int -ProxyClientSession::state_api_callout(int event, void *data) +ProxySession::state_api_callout(int event, void *data) { Event *e = static_cast(data); if (e == schedule_event) { @@ -127,7 +127,7 @@ ProxyClientSession::state_api_callout(int event, void *data) plugin_mutex = hook->m_cont->mutex; plugin_lock = MUTEX_TAKE_TRY_LOCK(hook->m_cont->mutex, mutex->thread_holding); if (!plugin_lock) { - SET_HANDLER(&ProxyClientSession::state_api_callout); + SET_HANDLER(&ProxySession::state_api_callout); if (!schedule_event) { // Don't bother to schedule is there is already one out. schedule_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); } @@ -162,7 +162,7 @@ ProxyClientSession::state_api_callout(int event, void *data) } void -ProxyClientSession::do_api_callout(TSHttpHookID id) +ProxySession::do_api_callout(TSHttpHookID id) { ink_assert(id == TS_HTTP_SSN_START_HOOK || id == TS_HTTP_SSN_CLOSE_HOOK); @@ -171,7 +171,7 @@ ProxyClientSession::do_api_callout(TSHttpHookID id) this->api_current = nullptr; if (this->has_hooks()) { - SET_HANDLER(&ProxyClientSession::state_api_callout); + SET_HANDLER(&ProxySession::state_api_callout); this->state_api_callout(EVENT_NONE, nullptr); } else { this->handle_api_return(TS_EVENT_HTTP_CONTINUE); @@ -179,11 +179,11 @@ ProxyClientSession::do_api_callout(TSHttpHookID id) } void -ProxyClientSession::handle_api_return(int event) +ProxySession::handle_api_return(int event) { TSHttpHookID hookid = this->api_hookid; - SET_HANDLER(&ProxyClientSession::state_api_callout); + SET_HANDLER(&ProxySession::state_api_callout); this->api_hookid = TS_HTTP_LAST_HOOK; this->api_scope = API_HOOK_SCOPE_NONE; diff --git a/proxy/ProxyClientSession.h b/proxy/ProxySession.h similarity index 92% rename from proxy/ProxyClientSession.h rename to proxy/ProxySession.h index 241da9b205f..609af3e708a 100644 --- a/proxy/ProxyClientSession.h +++ b/proxy/ProxySession.h @@ -1,6 +1,6 @@ /** @file - ProxyClientSession - Base class for protocol client sessions. + ProxySession - Base class for protocol client & server sessions. @section license License @@ -28,7 +28,7 @@ #include #include "P_Net.h" #include "InkAPIInternal.h" -#include "http/HttpServerSession.h" +#include "http/Http1ServerSession.h" #include "IPAllow.h" // Emit a debug message conditional on whether this particular client session @@ -36,7 +36,7 @@ // member function. #define SsnDebug(ssn, tag, ...) SpecificDebug((ssn)->debug(), tag, __VA_ARGS__) -class ProxyClientTransaction; +class ProxyTransaction; enum class ProxyErrorClass { NONE, @@ -72,10 +72,11 @@ struct ProxyError { // A little ugly, but this global is tracked by traffic_server. extern bool ts_is_draining; -class ProxyClientSession : public VConnection +/// Abstract class for HttpSM to interface with any session +class ProxySession : public VConnection { public: - ProxyClientSession(); + ProxySession(); virtual void destroy() = 0; virtual void free(); @@ -181,7 +182,7 @@ class ProxyClientSession : public VConnection } // Indicate we are done with a transaction. - virtual void release(ProxyClientTransaction *trans) = 0; + virtual void release(ProxyTransaction *trans) = 0; virtual in_port_t get_outbound_port() const @@ -208,11 +209,11 @@ class ProxyClientSession : public VConnection } virtual void - attach_server_session(HttpServerSession *ssession, bool transaction_done = true) + attach_server_session(Http1ServerSession *ssession, bool transaction_done = true) { } - virtual HttpServerSession * + virtual Http1ServerSession * get_server_session() const { return nullptr; @@ -296,8 +297,8 @@ class ProxyClientSession : public VConnection ink_hrtime ssn_last_txn_time = 0; // noncopyable - ProxyClientSession(ProxyClientSession &) = delete; - ProxyClientSession &operator=(const ProxyClientSession &) = delete; + ProxySession(ProxySession &) = delete; + ProxySession &operator=(const ProxySession &) = delete; protected: // XXX Consider using a bitwise flags variable for the following flags, so diff --git a/proxy/ProxyClientTransaction.cc b/proxy/ProxyTransaction.cc similarity index 82% rename from proxy/ProxyClientTransaction.cc rename to proxy/ProxyTransaction.cc index 17c1c93f5e4..35c1c94520f 100644 --- a/proxy/ProxyClientTransaction.cc +++ b/proxy/ProxyTransaction.cc @@ -1,6 +1,6 @@ /** @file - ProxyClientTransaction - Base class for protocol client transactions. + ProxyTransaction - Base class for protocol client transactions. @section license License @@ -22,12 +22,12 @@ */ #include "http/HttpSM.h" -#include "http/HttpServerSession.h" +#include "http/Http1ServerSession.h" #include "Plugin.h" #define HttpTxnDebug(fmt, ...) SsnDebug(this, "http_txn", fmt, __VA_ARGS__) -ProxyClientTransaction::ProxyClientTransaction() +ProxyTransaction::ProxyTransaction() : VConnection(nullptr), parent(nullptr), current_reader(nullptr), @@ -38,7 +38,7 @@ ProxyClientTransaction::ProxyClientTransaction() } void -ProxyClientTransaction::new_transaction() +ProxyTransaction::new_transaction() { ink_assert(current_reader == nullptr); @@ -61,7 +61,7 @@ ProxyClientTransaction::new_transaction() } void -ProxyClientTransaction::release(IOBufferReader *r) +ProxyTransaction::release(IOBufferReader *r) { HttpTxnDebug("[%" PRId64 "] session released by sm [%" PRId64 "]", parent ? parent->connection_id() : 0, current_reader ? current_reader->sm_id : 0); @@ -73,20 +73,20 @@ ProxyClientTransaction::release(IOBufferReader *r) } void -ProxyClientTransaction::attach_server_session(HttpServerSession *ssession, bool transaction_done) +ProxyTransaction::attach_server_session(Http1ServerSession *ssession, bool transaction_done) { parent->attach_server_session(ssession, transaction_done); } void -ProxyClientTransaction::destroy() +ProxyTransaction::destroy() { current_reader = nullptr; this->mutex.clear(); } Action * -ProxyClientTransaction::adjust_thread(Continuation *cont, int event, void *data) +ProxyTransaction::adjust_thread(Continuation *cont, int event, void *data) { NetVConnection *vc = this->get_netvc(); EThread *this_thread = this_ethread(); @@ -101,7 +101,7 @@ ProxyClientTransaction::adjust_thread(Continuation *cont, int event, void *data) } void -ProxyClientTransaction::set_rx_error_code(ProxyError e) +ProxyTransaction::set_rx_error_code(ProxyError e) { if (this->current_reader) { this->current_reader->t_state.client_info.rx_error_code = e; @@ -109,7 +109,7 @@ ProxyClientTransaction::set_rx_error_code(ProxyError e) } void -ProxyClientTransaction::set_tx_error_code(ProxyError e) +ProxyTransaction::set_tx_error_code(ProxyError e) { if (this->current_reader) { this->current_reader->t_state.client_info.tx_error_code = e; diff --git a/proxy/ProxyClientTransaction.h b/proxy/ProxyTransaction.h similarity index 92% rename from proxy/ProxyClientTransaction.h rename to proxy/ProxyTransaction.h index 14de5559e03..07a3c4288b9 100644 --- a/proxy/ProxyClientTransaction.h +++ b/proxy/ProxyTransaction.h @@ -1,6 +1,6 @@ /** @file - ProxyClientTransaction - Base class for protocol client transactions. + ProxyTransaction - Base class for protocol client/server transactions. @section license License @@ -23,15 +23,17 @@ #pragma once -#include "ProxyClientSession.h" +#include "ProxySession.h" #include class HttpSM; -class HttpServerSession; -class ProxyClientTransaction : public VConnection +class Http1ServerSession; + +// Abstract Class for any transaction with-in the HttpSM +class ProxyTransaction : public VConnection { public: - ProxyClientTransaction(); + ProxyTransaction(); // do_io methods implemented by subclasses @@ -47,7 +49,7 @@ class ProxyClientTransaction : public VConnection virtual void set_inactivity_timeout(ink_hrtime timeout_in) = 0; virtual void cancel_inactivity_timeout() = 0; - virtual void attach_server_session(HttpServerSession *ssession, bool transaction_done = true); + virtual void attach_server_session(Http1ServerSession *ssession, bool transaction_done = true); // See if we need to schedule on the primary thread for the transaction or change the thread that is associated with the VC. // If we reschedule, the scheduled action is returned. Otherwise, NULL is returned @@ -195,14 +197,14 @@ class ProxyClientTransaction : public VConnection virtual void transaction_done() = 0; - ProxyClientSession * + ProxySession * get_parent() { return parent; } virtual void - set_parent(ProxyClientSession *new_parent) + set_parent(ProxySession *new_parent) { parent = new_parent; host_res_style = parent->host_res_style; @@ -212,7 +214,7 @@ class ProxyClientTransaction : public VConnection { } - HttpServerSession * + Http1ServerSession * get_server_session() const { return parent ? parent->get_server_session() : nullptr; @@ -263,7 +265,7 @@ class ProxyClientTransaction : public VConnection void set_tx_error_code(ProxyError e); protected: - ProxyClientSession *parent; + ProxySession *parent; HttpSM *current_reader; IOBufferReader *sm_reader; diff --git a/proxy/http/Http1ClientSession.cc b/proxy/http/Http1ClientSession.cc index 9f1a00224e2..388e8a4f049 100644 --- a/proxy/http/Http1ClientSession.cc +++ b/proxy/http/Http1ClientSession.cc @@ -32,10 +32,10 @@ #include "tscore/ink_resolver.h" #include "Http1ClientSession.h" -#include "Http1ClientTransaction.h" +#include "Http1Transaction.h" #include "HttpSM.h" #include "HttpDebugNames.h" -#include "HttpServerSession.h" +#include "Http1ServerSession.h" #include "Plugin.h" #define HttpSsnDebug(fmt, ...) SsnDebug(this, "http_cs", fmt, __VA_ARGS__) @@ -160,7 +160,7 @@ Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB ink_assert(lock.is_locked()); // Unique client session identifier. - con_id = ProxyClientSession::next_connection_id(); + con_id = ProxySession::next_connection_id(); schedule_event = nullptr; @@ -448,9 +448,9 @@ Http1ClientSession::reenable(VIO *vio) client_vc->reenable(vio); } -// Called from the Http1ClientTransaction::release +// Called from the Http1Transaction::release void -Http1ClientSession::release(ProxyClientTransaction *trans) +Http1ClientSession::release(ProxyTransaction *trans) { ink_assert(read_state == HCS_ACTIVE_READER || read_state == HCS_INIT); @@ -506,7 +506,7 @@ Http1ClientSession::new_transaction() } void -Http1ClientSession::attach_server_session(HttpServerSession *ssession, bool transaction_done) +Http1ClientSession::attach_server_session(Http1ServerSession *ssession, bool transaction_done) { if (ssession) { ink_assert(bound_ss == nullptr); diff --git a/proxy/http/Http1ClientSession.h b/proxy/http/Http1ClientSession.h index ce8346665ce..f2581e075d1 100644 --- a/proxy/http/Http1ClientSession.h +++ b/proxy/http/Http1ClientSession.h @@ -37,23 +37,24 @@ #include "HTTP.h" #include "HttpConfig.h" #include "IPAllow.h" -#include "ProxyClientSession.h" -#include "Http1ClientTransaction.h" +#include "ProxySession.h" +#include "Http1Transaction.h" #ifdef USE_HTTP_DEBUG_LISTS extern ink_mutex debug_cs_list_mutex; #endif class HttpSM; -class HttpServerSession; +class Http1ServerSession; -class Http1ClientSession : public ProxyClientSession +/// Class to manage a Http v1 session to a client +class Http1ClientSession : public ProxySession { public: - typedef ProxyClientSession super; ///< Parent type. + typedef ProxySession super; ///< Parent type. Http1ClientSession(); - // Implement ProxyClientSession interface. + // Implement ProxySession interface. void destroy() override; void free() override; void release_transaction(); @@ -120,11 +121,11 @@ class Http1ClientSession : public ProxyClientSession } // Indicate we are done with a transaction - void release(ProxyClientTransaction *trans) override; + void release(ProxyTransaction *trans) override; - void attach_server_session(HttpServerSession *ssession, bool transaction_done = true) override; + void attach_server_session(Http1ServerSession *ssession, bool transaction_done = true) override; - HttpServerSession * + Http1ServerSession * get_server_session() const override { return bound_ss; @@ -199,7 +200,7 @@ class Http1ClientSession : public ProxyClientSession VIO *ka_vio; VIO *slave_ka_vio; - HttpServerSession *bound_ss; + Http1ServerSession *bound_ss; int released_transactions; @@ -212,7 +213,7 @@ class Http1ClientSession : public ProxyClientSession /// Transparently pass-through non-HTTP traffic. bool f_transparent_passthrough; - Http1ClientTransaction trans; + Http1Transaction trans; }; extern ClassAllocator http1ClientSessionAllocator; diff --git a/proxy/http/HttpServerSession.cc b/proxy/http/Http1ServerSession.cc similarity index 86% rename from proxy/http/HttpServerSession.cc rename to proxy/http/Http1ServerSession.cc index cadbcc0e085..e70dac262e9 100644 --- a/proxy/http/HttpServerSession.cc +++ b/proxy/http/Http1ServerSession.cc @@ -23,7 +23,7 @@ /**************************************************************************** - HttpServerSession.cc + Http1ServerSession.cc Description: @@ -32,15 +32,15 @@ #include "tscore/BufferWriter.h" #include "tscore/bwf_std_format.h" #include "tscore/Allocator.h" -#include "HttpServerSession.h" +#include "Http1ServerSession.h" #include "HttpSessionManager.h" #include "HttpSM.h" static int64_t next_ss_id = (int64_t)0; -ClassAllocator httpServerSessionAllocator("httpServerSessionAllocator"); +ClassAllocator http1ServerSessionAllocator("http1ServerSessionAllocator"); void -HttpServerSession::destroy() +Http1ServerSession::destroy() { ink_release_assert(server_vc == nullptr); ink_assert(read_buffer); @@ -53,14 +53,14 @@ HttpServerSession::destroy() mutex.clear(); if (TS_SERVER_SESSION_SHARING_POOL_THREAD == sharing_pool) { - THREAD_FREE(this, httpServerSessionAllocator, this_thread()); + THREAD_FREE(this, http1ServerSessionAllocator, this_thread()); } else { - httpServerSessionAllocator.free(this); + http1ServerSessionAllocator.free(this); } } void -HttpServerSession::new_connection(NetVConnection *new_vc) +Http1ServerSession::new_connection(NetVConnection *new_vc) { ink_assert(new_vc != nullptr); server_vc = new_vc; @@ -87,7 +87,7 @@ HttpServerSession::new_connection(NetVConnection *new_vc) } void -HttpServerSession::enable_outbound_connection_tracking(OutboundConnTrack::Group *group) +Http1ServerSession::enable_outbound_connection_tracking(OutboundConnTrack::Group *group) { ink_assert(nullptr == conn_track_group); conn_track_group = group; @@ -99,25 +99,25 @@ HttpServerSession::enable_outbound_connection_tracking(OutboundConnTrack::Group } VIO * -HttpServerSession::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) +Http1ServerSession::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) { return server_vc ? server_vc->do_io_read(c, nbytes, buf) : nullptr; } VIO * -HttpServerSession::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) +Http1ServerSession::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) { return server_vc ? server_vc->do_io_write(c, nbytes, buf, owner) : nullptr; } void -HttpServerSession::do_io_shutdown(ShutdownHowTo_t howto) +Http1ServerSession::do_io_shutdown(ShutdownHowTo_t howto) { server_vc->do_io_shutdown(howto); } void -HttpServerSession::do_io_close(int alerrno) +Http1ServerSession::do_io_close(int alerrno) { ts::LocalBufferWriter<256> w; bool debug_p = is_debug_tag_set("http_ss"); @@ -162,17 +162,17 @@ HttpServerSession::do_io_close(int alerrno) } void -HttpServerSession::reenable(VIO *vio) +Http1ServerSession::reenable(VIO *vio) { server_vc->reenable(vio); } -// void HttpServerSession::release() +// void Http1ServerSession::release() // // Releases the session for K-A reuse // void -HttpServerSession::release() +Http1ServerSession::release() { Debug("http_ss", "Releasing session, private_session=%d, sharing_match=%d", private_session, sharing_match); // Set our state to KA for stat issues diff --git a/proxy/http/HttpServerSession.h b/proxy/http/Http1ServerSession.h similarity index 82% rename from proxy/http/HttpServerSession.h rename to proxy/http/Http1ServerSession.h index 5e4224af68d..a66bf1a64fe 100644 --- a/proxy/http/HttpServerSession.h +++ b/proxy/http/Http1ServerSession.h @@ -23,7 +23,7 @@ /**************************************************************************** - HttpServerSession.h + Http1ServerSession.h Description: @@ -64,14 +64,16 @@ enum { HTTP_SS_MAGIC_DEAD = 0xDEADFEED, }; -class HttpServerSession : public VConnection +/// Class to manage a Http v1 session to a server +// TODO: inherit from ProxySession +class Http1ServerSession : public VConnection { - using self_type = HttpServerSession; + using self_type = Http1ServerSession; using super_type = VConnection; public: - HttpServerSession() : super_type(nullptr) {} - HttpServerSession(self_type const &) = delete; + Http1ServerSession() : super_type(nullptr) {} + Http1ServerSession(self_type const &) = delete; self_type &operator=(self_type const &) = delete; void destroy(); @@ -170,8 +172,8 @@ class HttpServerSession : public VConnection static sockaddr const *key_of(self_type const *ssn); static bool equal(sockaddr const *lhs, sockaddr const *rhs); // Add a couple overloads for internal convenience. - static bool equal(sockaddr const *lhs, HttpServerSession const *rhs); - static bool equal(HttpServerSession const *lhs, sockaddr const *rhs); + static bool equal(sockaddr const *lhs, Http1ServerSession const *rhs); + static bool equal(Http1ServerSession const *lhs, sockaddr const *rhs); } _ip_link; /// Hash map descriptor class for FQDN map. @@ -220,86 +222,86 @@ class HttpServerSession : public VConnection IOBufferReader *buf_reader = nullptr; }; -extern ClassAllocator httpServerSessionAllocator; +extern ClassAllocator http1ServerSessionAllocator; // --- Implementation --- inline void -HttpServerSession::attach_hostname(const char *hostname) +Http1ServerSession::attach_hostname(const char *hostname) { if (CRYPTO_HASH_ZERO == hostname_hash) { CryptoContext().hash_immediate(hostname_hash, (unsigned char *)hostname, strlen(hostname)); } } -inline HttpServerSession *& -HttpServerSession::IPLinkage::next_ptr(self_type *ssn) +inline Http1ServerSession *& +Http1ServerSession::IPLinkage::next_ptr(self_type *ssn) { return ssn->_ip_link._next; } -inline HttpServerSession *& -HttpServerSession::IPLinkage::prev_ptr(self_type *ssn) +inline Http1ServerSession *& +Http1ServerSession::IPLinkage::prev_ptr(self_type *ssn) { return ssn->_ip_link._prev; } inline uint32_t -HttpServerSession::IPLinkage::hash_of(sockaddr const *key) +Http1ServerSession::IPLinkage::hash_of(sockaddr const *key) { return ats_ip_hash(key); } inline sockaddr const * -HttpServerSession::IPLinkage::key_of(self_type const *ssn) +Http1ServerSession::IPLinkage::key_of(self_type const *ssn) { return &ssn->get_server_ip().sa; } inline bool -HttpServerSession::IPLinkage::equal(sockaddr const *lhs, sockaddr const *rhs) +Http1ServerSession::IPLinkage::equal(sockaddr const *lhs, sockaddr const *rhs) { return ats_ip_addr_port_eq(lhs, rhs); } inline bool -HttpServerSession::IPLinkage::equal(sockaddr const *lhs, HttpServerSession const *rhs) +Http1ServerSession::IPLinkage::equal(sockaddr const *lhs, Http1ServerSession const *rhs) { return ats_ip_addr_port_eq(lhs, key_of(rhs)); } inline bool -HttpServerSession::IPLinkage::equal(HttpServerSession const *lhs, sockaddr const *rhs) +Http1ServerSession::IPLinkage::equal(Http1ServerSession const *lhs, sockaddr const *rhs) { return ats_ip_addr_port_eq(key_of(lhs), rhs); } -inline HttpServerSession *& -HttpServerSession::FQDNLinkage::next_ptr(self_type *ssn) +inline Http1ServerSession *& +Http1ServerSession::FQDNLinkage::next_ptr(self_type *ssn) { return ssn->_fqdn_link._next; } -inline HttpServerSession *& -HttpServerSession::FQDNLinkage::prev_ptr(self_type *ssn) +inline Http1ServerSession *& +Http1ServerSession::FQDNLinkage::prev_ptr(self_type *ssn) { return ssn->_fqdn_link._prev; } inline uint64_t -HttpServerSession::FQDNLinkage::hash_of(CryptoHash const &key) +Http1ServerSession::FQDNLinkage::hash_of(CryptoHash const &key) { return key.fold(); } inline CryptoHash const & -HttpServerSession::FQDNLinkage::key_of(self_type *ssn) +Http1ServerSession::FQDNLinkage::key_of(self_type *ssn) { return ssn->hostname_hash; } inline bool -HttpServerSession::FQDNLinkage::equal(CryptoHash const &lhs, CryptoHash const &rhs) +Http1ServerSession::FQDNLinkage::equal(CryptoHash const &lhs, CryptoHash const &rhs) { return lhs == rhs; } diff --git a/proxy/http/Http1ClientTransaction.cc b/proxy/http/Http1Transaction.cc similarity index 88% rename from proxy/http/Http1ClientTransaction.cc rename to proxy/http/Http1Transaction.cc index 5c5f1759b90..09d5f0eae9e 100644 --- a/proxy/http/Http1ClientTransaction.cc +++ b/proxy/http/Http1Transaction.cc @@ -1,6 +1,6 @@ /** @file - Http1ClientTransaction.cc - The Transaction class for Http1* + Http1Transaction.cc - The Transaction class for Http1* @section license License @@ -21,12 +21,12 @@ limitations under the License. */ -#include "Http1ClientTransaction.h" +#include "Http1Transaction.h" #include "Http1ClientSession.h" #include "HttpSM.h" void -Http1ClientTransaction::release(IOBufferReader *r) +Http1Transaction::release(IOBufferReader *r) { // Must set this inactivity count here rather than in the session because the state machine // is not availble then @@ -47,7 +47,7 @@ Http1ClientTransaction::release(IOBufferReader *r) } void -Http1ClientTransaction::set_parent(ProxyClientSession *new_parent) +Http1Transaction::set_parent(ProxySession *new_parent) { parent = new_parent; Http1ClientSession *http1_parent = dynamic_cast(new_parent); @@ -61,7 +61,7 @@ Http1ClientTransaction::set_parent(ProxyClientSession *new_parent) } void -Http1ClientTransaction::transaction_done() +Http1Transaction::transaction_done() { if (parent) { static_cast(parent)->release_transaction(); @@ -69,7 +69,7 @@ Http1ClientTransaction::transaction_done() } bool -Http1ClientTransaction::allow_half_open() const +Http1Transaction::allow_half_open() const { bool config_allows_it = (current_reader) ? current_reader->t_state.txn_conf->allow_half_open > 0 : true; if (config_allows_it) { diff --git a/proxy/http/Http1ClientTransaction.h b/proxy/http/Http1Transaction.h similarity index 88% rename from proxy/http/Http1ClientTransaction.h rename to proxy/http/Http1Transaction.h index 8a876660f8e..4175b32a43d 100644 --- a/proxy/http/Http1ClientTransaction.h +++ b/proxy/http/Http1Transaction.h @@ -1,6 +1,6 @@ /** @file - Http1ClientTransaction.h - The Transaction class for Http1* + Http1Transaction.h - The Transaction class for Http1* @section license License @@ -23,16 +23,16 @@ #pragma once -#include "../ProxyClientTransaction.h" +#include "../ProxyTransaction.h" class Continuation; - -class Http1ClientTransaction : public ProxyClientTransaction +/// Concrete class for any Http1 Transaction +class Http1Transaction : public ProxyTransaction { public: - using super_type = ProxyClientTransaction; + using super_type = ProxyTransaction; - Http1ClientTransaction() {} + Http1Transaction() {} // Implement VConnection interface. VIO * do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override @@ -53,7 +53,7 @@ class Http1ClientTransaction : public ProxyClientTransaction } // Don't destroy your elements. Rely on the Http1ClientSession to clean up the - // Http1ClientTransaction class as necessary. The super::destroy() clears the + // Http1Transaction class as necessary. The super::destroy() clears the // mutex, which Http1ClientSession owns. void destroy() override @@ -83,7 +83,7 @@ class Http1ClientTransaction : public ProxyClientTransaction bool allow_half_open() const override; - void set_parent(ProxyClientSession *new_parent) override; + void set_parent(ProxySession *new_parent) override; bool is_outbound_transparent() const override diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index cf94a9f7b08..5cb5da190a6 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -22,12 +22,12 @@ */ -#include "../ProxyClientTransaction.h" +#include "../ProxyTransaction.h" #include "HttpSM.h" #include "HttpTransact.h" #include "HttpTransactHeaders.h" #include "ProxyConfig.h" -#include "HttpServerSession.h" +#include "Http1ServerSession.h" #include "HttpDebugNames.h" #include "HttpSessionManager.h" #include "P_Cache.h" @@ -443,7 +443,7 @@ HttpSM::start_sub_sm() } void -HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader *buffer_reader) +HttpSM::attach_client_session(ProxyTransaction *client_vc, IOBufferReader *buffer_reader) { milestones[TS_MILESTONE_UA_BEGIN] = Thread::get_hrtime(); ink_assert(client_vc != nullptr); @@ -639,7 +639,7 @@ HttpSM::state_read_client_request_header(int event, void *data) // Reset the inactivity timeout if this is the first // time we've been called. The timeout had been set to - // the accept timeout by the ProxyClientTransaction + // the accept timeout by the ProxyTransaction // if ((ua_buffer_reader->read_avail() > 0) && (client_request_hdr_bytes == 0)) { milestones[TS_MILESTONE_UA_FIRST_READ] = Thread::get_hrtime(); @@ -1734,14 +1734,14 @@ HttpSM::state_http_server_open(int event, void *data) pending_action = nullptr; } milestones[TS_MILESTONE_SERVER_CONNECT_END] = Thread::get_hrtime(); - HttpServerSession *session; + Http1ServerSession *session; NetVConnection *netvc = nullptr; switch (event) { case NET_EVENT_OPEN: { session = (TS_SERVER_SESSION_SHARING_POOL_THREAD == t_state.http_config_param->server_session_sharing_pool) ? - THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : - httpServerSessionAllocator.alloc(); + THREAD_ALLOC_INIT(http1ServerSessionAllocator, mutex->thread_holding) : + http1ServerSessionAllocator.alloc(); session->sharing_pool = static_cast(t_state.http_config_param->server_session_sharing_pool); session->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); @@ -4901,7 +4901,7 @@ HttpSM::do_http_server_open(bool raw) // session when we already have an attached server session. else if ((TS_SERVER_SESSION_SHARING_MATCH_NONE == t_state.txn_conf->server_session_sharing_match || is_private()) && (ua_txn != nullptr)) { - HttpServerSession *existing_ss = ua_txn->get_server_session(); + Http1ServerSession *existing_ss = ua_txn->get_server_session(); if (existing_ss) { // [amc] Not sure if this is the best option, but we don't get here unless session sharing is disabled @@ -4928,7 +4928,7 @@ HttpSM::do_http_server_open(bool raw) // to get a new one. // ua_txn is null when t_state.req_flavor == REQ_FLAVOR_SCHEDULED_UPDATE else if (ua_txn != nullptr) { - HttpServerSession *existing_ss = ua_txn->get_server_session(); + Http1ServerSession *existing_ss = ua_txn->get_server_session(); if (existing_ss) { existing_ss->get_netvc()->set_inactivity_timeout(HRTIME_SECONDS(t_state.txn_conf->keep_alive_no_activity_timeout_out)); existing_ss->release(); @@ -5898,7 +5898,7 @@ HttpSM::write_header_into_buffer(HTTPHdr *h, MIOBuffer *b) } void -HttpSM::attach_server_session(HttpServerSession *s) +HttpSM::attach_server_session(Http1ServerSession *s) { hsm_release_assert(server_session == nullptr); hsm_release_assert(server_entry == nullptr); @@ -7973,7 +7973,7 @@ HttpSM::is_private() if (server_session) { res = server_session->private_session; } else if (ua_txn) { - HttpServerSession *ss = ua_txn->get_server_session(); + Http1ServerSession *ss = ua_txn->get_server_session(); if (ss) { res = ss->private_session; } else if (will_be_private_ss) { diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index a26e2e17ac0..f2144b96c30 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -39,7 +39,7 @@ #include "UrlRewrite.h" #include "HttpTunnel.h" #include "InkAPIInternal.h" -#include "../ProxyClientTransaction.h" +#include "../ProxyTransaction.h" #include "HdrUtils.h" #include #include "tscore/History.h" @@ -53,7 +53,7 @@ * the buffer. By allocating memory only when it is required we can * reduce the memory consumed by TS process. * - * IMPORTANT NOTE: enable/disable LAZY_BUF_ALLOC in HttpServerSession.h + * IMPORTANT NOTE: enable/disable LAZY_BUF_ALLOC in Http1ServerSession.h * as well. */ #define LAZY_BUF_ALLOC @@ -72,7 +72,7 @@ static size_t const HTTP_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ // the larger buffer size static size_t const HTTP_SERVER_RESP_HDR_BUFFER_INDEX = BUFFER_SIZE_INDEX_8K; -class HttpServerSession; +class Http1ServerSession; class AuthHttpAdapter; class HttpSM; @@ -228,22 +228,22 @@ class HttpSM : public Continuation void init(); - void attach_client_session(ProxyClientTransaction *client_vc_arg, IOBufferReader *buffer_reader); + void attach_client_session(ProxyTransaction *client_vc_arg, IOBufferReader *buffer_reader); // Called by httpSessionManager so that we can reset // the session timeouts and initiate a read while // holding the lock for the server session - void attach_server_session(HttpServerSession *s); + void attach_server_session(Http1ServerSession *s); // Used to read attributes of // the current active server session - HttpServerSession * + Http1ServerSession * get_server_session() { return server_session; } - ProxyClientTransaction * + ProxyTransaction * get_ua_txn() { return ua_txn; @@ -363,7 +363,7 @@ class HttpSM : public Continuation void remove_ua_entry(); public: - ProxyClientTransaction *ua_txn = nullptr; + ProxyTransaction *ua_txn = nullptr; BackgroundFill_t background_fill = BACKGROUND_FILL_NONE; // AuthHttpAdapter authAdapter; void set_http_schedule(Continuation *); @@ -375,8 +375,8 @@ class HttpSM : public Continuation IOBufferReader *ua_buffer_reader = nullptr; IOBufferReader *ua_raw_buffer_reader = nullptr; - HttpVCTableEntry *server_entry = nullptr; - HttpServerSession *server_session = nullptr; + HttpVCTableEntry *server_entry = nullptr; + Http1ServerSession *server_session = nullptr; /* Because we don't want to take a session from a shared pool if we know that it will be private, * but we cannot set it to private until we have an attached server session. diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index 6cef39f65c2..399f0354936 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -31,8 +31,8 @@ ****************************************************************************/ #include "HttpSessionManager.h" -#include "../ProxyClientSession.h" -#include "HttpServerSession.h" +#include "../ProxySession.h" +#include "Http1ServerSession.h" #include "HttpSM.h" #include "HttpDebugNames.h" @@ -57,13 +57,13 @@ ServerSessionPool::purge() { // @c do_io_close can free the instance which clears the intrusive links and breaks the iterator. // Therefore @c do_io_close is called on a post-incremented iterator. - m_ip_pool.apply([](HttpServerSession *ssn) -> void { ssn->do_io_close(); }); + m_ip_pool.apply([](Http1ServerSession *ssn) -> void { ssn->do_io_close(); }); m_ip_pool.clear(); m_fqdn_pool.clear(); } bool -ServerSessionPool::match(HttpServerSession *ss, sockaddr const *addr, CryptoHash const &hostname_hash, +ServerSessionPool::match(Http1ServerSession *ss, sockaddr const *addr, CryptoHash const &hostname_hash, TSServerSessionSharingMatchType match_style) { return TS_SERVER_SESSION_SHARING_MATCH_NONE != @@ -93,7 +93,7 @@ ServerSessionPool::validate_sni(HttpSM *sm, NetVConnection *netvc) HSMresult_t ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostname_hash, - TSServerSessionSharingMatchType match_style, HttpSM *sm, HttpServerSession *&to_return) + TSServerSessionSharingMatchType match_style, HttpSM *sm, Http1ServerSession *&to_return) { HSMresult_t zret = HSM_NOT_FOUND; to_return = nullptr; @@ -144,7 +144,7 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna } void -ServerSessionPool::releaseSession(HttpServerSession *ss) +ServerSessionPool::releaseSession(Http1ServerSession *ss) { ss->state = HSS_KA_SHARED; // Now we need to issue a read on the connection to detect @@ -176,7 +176,7 @@ int ServerSessionPool::eventHandler(int event, void *data) { NetVConnection *net_vc = nullptr; - HttpServerSession *s = nullptr; + Http1ServerSession *s = nullptr; switch (event) { case VC_EVENT_READ_READY: @@ -275,9 +275,9 @@ HttpSessionManager::purge_keepalives() HSMresult_t HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockaddr const *ip, const char *hostname, - ProxyClientTransaction *ua_txn, HttpSM *sm) + ProxyTransaction *ua_txn, HttpSM *sm) { - HttpServerSession *to_return = nullptr; + Http1ServerSession *to_return = nullptr; TSServerSessionSharingMatchType match_style = static_cast(sm->t_state.txn_conf->server_session_sharing_match); CryptoHash hostname_hash; @@ -370,7 +370,7 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad } HSMresult_t -HttpSessionManager::release_session(HttpServerSession *to_release) +HttpSessionManager::release_session(Http1ServerSession *to_release) { EThread *ethread = this_ethread(); ServerSessionPool *pool = diff --git a/proxy/http/HttpSessionManager.h b/proxy/http/HttpSessionManager.h index f83196aad0e..10930d58cc0 100644 --- a/proxy/http/HttpSessionManager.h +++ b/proxy/http/HttpSessionManager.h @@ -33,10 +33,10 @@ #pragma once #include "P_EventSystem.h" -#include "HttpServerSession.h" +#include "Http1ServerSession.h" #include "tscore/IntrusiveHashMap.h" -class ProxyClientTransaction; +class ProxyTransaction; class HttpSM; void initialize_thread_for_http_sessions(EThread *thread, int thread_index); @@ -67,13 +67,13 @@ class ServerSessionPool : public Continuation static bool validate_sni(HttpSM *sm, NetVConnection *netvc); protected: - using IPTable = IntrusiveHashMap; - using FQDNTable = IntrusiveHashMap; + using IPTable = IntrusiveHashMap; + using FQDNTable = IntrusiveHashMap; public: /** Check if a session matches address and host name. */ - static bool match(HttpServerSession *ss, sockaddr const *addr, CryptoHash const &host_hash, + static bool match(Http1ServerSession *ss, sockaddr const *addr, CryptoHash const &host_hash, TSServerSessionSharingMatchType match_style); /** Get a session from the pool. @@ -84,10 +84,10 @@ class ServerSessionPool : public Continuation @return A pointer to the session or @c NULL if not matching session was found. */ HSMresult_t acquireSession(sockaddr const *addr, CryptoHash const &host_hash, TSServerSessionSharingMatchType match_style, - HttpSM *sm, HttpServerSession *&server_session); + HttpSM *sm, Http1ServerSession *&server_session); /** Release a session to to pool. */ - void releaseSession(HttpServerSession *ss); + void releaseSession(Http1ServerSession *ss); /// Close all sessions and then clear the table. void purge(); @@ -103,9 +103,8 @@ class HttpSessionManager public: HttpSessionManager() : m_g_pool(nullptr) {} ~HttpSessionManager() {} - HSMresult_t acquire_session(Continuation *cont, sockaddr const *addr, const char *hostname, ProxyClientTransaction *ua_txn, - HttpSM *sm); - HSMresult_t release_session(HttpServerSession *to_release); + HSMresult_t acquire_session(Continuation *cont, sockaddr const *addr, const char *hostname, ProxyTransaction *ua_txn, HttpSM *sm); + HSMresult_t release_session(Http1ServerSession *to_release); void purge_keepalives(); void init(); int main_handler(int event, void *data); diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 8183dddc194..d76f0ec0398 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -39,7 +39,7 @@ #include "RemapPluginInfo.h" #include "UrlMapping.h" #include "records/I_RecHttp.h" -#include "ProxyClientSession.h" +#include "ProxySession.h" #define HTTP_RELEASE_ASSERT(X) ink_release_assert(X) diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 84634c37bb2..36d9508331d 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -932,7 +932,7 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) // the amount to read since we know it. We will forward the FIN // to the server on VC_EVENT_WRITE_COMPLETE. if (p->vc_type == HT_HTTP_CLIENT) { - ProxyClientTransaction *ua_vc = static_cast(p->vc); + ProxyTransaction *ua_vc = static_cast(p->vc); if (ua_vc->get_half_close_flag()) { c_write = c->buffer_reader->read_avail(); p->alive = false; diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am index 6f5733229b0..7b5abd7707d 100644 --- a/proxy/http/Makefile.am +++ b/proxy/http/Makefile.am @@ -46,8 +46,8 @@ libhttp_a_SOURCES = \ HttpCacheSM.h \ Http1ClientSession.cc \ Http1ClientSession.h \ - Http1ClientTransaction.cc \ - Http1ClientTransaction.h \ + Http1Transaction.cc \ + Http1Transaction.h \ HttpConfig.cc \ HttpConfig.h \ HttpConnectionCount.cc \ @@ -60,8 +60,8 @@ libhttp_a_SOURCES = \ HttpProxyServerMain.h \ HttpSM.cc \ HttpSM.h \ - HttpServerSession.cc \ - HttpServerSession.h \ + Http1ServerSession.cc \ + Http1ServerSession.h \ HttpSessionManager.cc \ HttpSessionManager.h \ HttpTransact.cc \ diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index 99921dd03bd..3cc9edd58dd 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -158,7 +158,7 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT, new_vc->mutex->thread_holding); // Unique client session identifier. - this->con_id = ProxyClientSession::next_connection_id(); + this->con_id = ProxySession::next_connection_id(); this->client_vc = new_vc; client_vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); this->schedule_event = nullptr; diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index fe8a79500f5..26636fb01cc 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -25,7 +25,7 @@ #include "HTTP2.h" #include "Plugin.h" -#include "ProxyClientSession.h" +#include "ProxySession.h" #include "Http2ConnectionState.h" #include #include "tscore/ink_inet.h" @@ -152,15 +152,15 @@ class Http2Frame IOBufferReader *ioreader; }; -class Http2ClientSession : public ProxyClientSession +class Http2ClientSession : public ProxySession { public: - typedef ProxyClientSession super; ///< Parent type. + typedef ProxySession super; ///< Parent type. typedef int (Http2ClientSession::*SessionHandler)(int, void *); Http2ClientSession(); - // Implement ProxyClientSession interface. + // Implement ProxySession interface. void start() override; void destroy() override; void free() override; @@ -219,7 +219,7 @@ class Http2ClientSession : public ProxyClientSession } void - release(ProxyClientTransaction *trans) override + release(ProxyTransaction *trans) override { } diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index 8286acc1f75..e766ced28df 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -24,7 +24,7 @@ #pragma once #include "HTTP2.h" -#include "../ProxyClientTransaction.h" +#include "../ProxyTransaction.h" #include "Http2DebugNames.h" #include "../http/HttpTunnel.h" // To get ChunkedHandler #include "Http2DependencyTree.h" @@ -34,10 +34,10 @@ class Http2ConnectionState; typedef Http2DependencyTree::Tree DependencyTree; -class Http2Stream : public ProxyClientTransaction +class Http2Stream : public ProxyTransaction { public: - typedef ProxyClientTransaction super; ///< Parent type. + typedef ProxyTransaction super; ///< Parent type. Http2Stream(Http2StreamId sid = 0, ssize_t initial_rwnd = Http2::initial_window_size) : client_rwnd(initial_rwnd), _id(sid) { SET_HANDLER(&Http2Stream::main_event_handler); diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index dd8f96ca24a..c0682c239c5 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -37,9 +37,9 @@ #include "URL.h" #include "MIME.h" #include "HTTP.h" -#include "ProxyClientSession.h" +#include "ProxySession.h" #include "Http2ClientSession.h" -#include "HttpServerSession.h" +#include "Http1ServerSession.h" #include "HttpSM.h" #include "HttpConfig.h" #include "P_Net.h" @@ -4739,7 +4739,7 @@ TSHttpSsnHookAdd(TSHttpSsn ssnp, TSHttpHookID id, TSCont contp) sdk_assert(sdk_sanity_check_continuation(contp) == TS_SUCCESS); sdk_assert(sdk_sanity_check_hook_id(id) == TS_SUCCESS); - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); cs->ssn_hook_append(id, (INKContInternal *)contp); } @@ -4748,28 +4748,28 @@ TSHttpSsnTransactionCount(TSHttpSsn ssnp) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); return cs->get_transact_count(); } TSVConn TSHttpSsnClientVConnGet(TSHttpSsn ssnp) { - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); return reinterpret_cast(cs->get_netvc()); } TSVConn TSHttpSsnServerVConnGet(TSHttpSsn ssnp) { - HttpServerSession *ss = reinterpret_cast(ssnp); + Http1ServerSession *ss = reinterpret_cast(ssnp); return reinterpret_cast(ss->get_netvc()); } class TSHttpSsnCallback : public Continuation { public: - TSHttpSsnCallback(ProxyClientSession *cs, TSEvent event) : Continuation(cs->mutex), m_cs(cs), m_event(event) + TSHttpSsnCallback(ProxySession *cs, TSEvent event) : Continuation(cs->mutex), m_cs(cs), m_event(event) { SET_HANDLER(&TSHttpSsnCallback::event_handler); } @@ -4783,7 +4783,7 @@ class TSHttpSsnCallback : public Continuation } private: - ProxyClientSession *m_cs; + ProxySession *m_cs; TSEvent m_event; }; @@ -4792,8 +4792,8 @@ TSHttpSsnReenable(TSHttpSsn ssnp, TSEvent event) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - ProxyClientSession *cs = reinterpret_cast(ssnp); - EThread *eth = this_ethread(); + ProxySession *cs = reinterpret_cast(ssnp); + EThread *eth = this_ethread(); // If this function is being executed on a thread created by the API // which is DEDICATED, the continuation needs to be called back on a @@ -5493,7 +5493,7 @@ TSHttpTxnTransformRespGet(TSHttpTxn txnp, TSMBuffer *bufp, TSMLoc *obj) sockaddr const * TSHttpSsnClientAddrGet(TSHttpSsn ssnp) { - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); if (cs == nullptr) { return nullptr; @@ -5512,7 +5512,7 @@ TSHttpTxnClientAddrGet(TSHttpTxn txnp) sockaddr const * TSHttpSsnIncomingAddrGet(TSHttpSsn ssnp) { - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); if (cs == nullptr) { return nullptr; @@ -5535,7 +5535,7 @@ TSHttpTxnOutgoingAddrGet(TSHttpTxn txnp) HttpSM *sm = reinterpret_cast(txnp); - HttpServerSession *ssn = sm->get_server_session(); + Http1ServerSession *ssn = sm->get_server_session(); if (ssn == nullptr) { return nullptr; } @@ -5653,7 +5653,7 @@ TSHttpTxnServerPacketMarkSet(TSHttpTxn txnp, int mark) // change the mark on an active server session if (nullptr != sm->ua_txn) { - HttpServerSession *ssn = sm->ua_txn->get_server_session(); + Http1ServerSession *ssn = sm->ua_txn->get_server_session(); if (nullptr != ssn) { NetVConnection *vc = ssn->get_netvc(); if (vc != nullptr) { @@ -5695,7 +5695,7 @@ TSHttpTxnServerPacketTosSet(TSHttpTxn txnp, int tos) // change the tos on an active server session if (nullptr != sm->ua_txn) { - HttpServerSession *ssn = sm->ua_txn->get_server_session(); + Http1ServerSession *ssn = sm->ua_txn->get_server_session(); if (nullptr != ssn) { NetVConnection *vc = ssn->get_netvc(); if (vc != nullptr) { @@ -5737,7 +5737,7 @@ TSHttpTxnServerPacketDscpSet(TSHttpTxn txnp, int dscp) // change the tos on an active server session if (nullptr != sm->ua_txn) { - HttpServerSession *ssn = sm->ua_txn->get_server_session(); + Http1ServerSession *ssn = sm->ua_txn->get_server_session(); if (nullptr != ssn) { NetVConnection *vc = ssn->get_netvc(); if (vc != nullptr) { @@ -6090,7 +6090,7 @@ TSHttpSsnArgSet(TSHttpSsn ssnp, int arg_idx, void *arg) sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); sdk_assert(arg_idx >= 0 && arg_idx < TS_HTTP_MAX_USER_ARG); - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); cs->set_user_arg(arg_idx, arg); } @@ -6101,7 +6101,7 @@ TSHttpSsnArgGet(TSHttpSsn ssnp, int arg_idx) sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); sdk_assert(arg_idx >= 0 && arg_idx < TS_HTTP_MAX_USER_ARG); - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); return cs->get_user_arg(arg_idx); } @@ -6236,14 +6236,14 @@ void TSHttpSsnDebugSet(TSHttpSsn ssnp, int on) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - (reinterpret_cast(ssnp))->set_debug(0 != on); + (reinterpret_cast(ssnp))->set_debug(0 != on); } int TSHttpSsnDebugGet(TSHttpSsn ssnp) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - return (reinterpret_cast(ssnp))->debug(); + return (reinterpret_cast(ssnp))->debug(); } int @@ -7404,8 +7404,8 @@ TSHttpSsnClientFdGet(TSHttpSsn ssnp, int *fdp) { sdk_assert(sdk_sanity_check_null_ptr((void *)fdp) == TS_SUCCESS); - VConnection *basecs = reinterpret_cast(ssnp); - ProxyClientSession *cs = dynamic_cast(basecs); + VConnection *basecs = reinterpret_cast(ssnp); + ProxySession *cs = dynamic_cast(basecs); if (cs == nullptr) { return TS_ERROR; @@ -7438,7 +7438,7 @@ TSHttpTxnServerFdGet(TSHttpTxn txnp, int *fdp) HttpSM *sm = reinterpret_cast(txnp); *fdp = -1; - HttpServerSession *ss = sm->get_server_session(); + Http1ServerSession *ss = sm->get_server_session(); if (ss == nullptr) { return TS_ERROR; } @@ -7806,7 +7806,7 @@ TSFetchRespHdrMLocGet(TSFetchSM fetch_sm) int TSHttpSsnIsInternal(TSHttpSsn ssnp) { - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); if (!cs) { return 0; @@ -9294,7 +9294,7 @@ int64_t TSHttpSsnIdGet(TSHttpSsn ssnp) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); return cs->connection_id(); } @@ -9324,8 +9324,8 @@ TSHttpSsnClientProtocolStackGet(TSHttpSsn ssnp, int n, const char **result, int { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); sdk_assert(n == 0 || result != nullptr); - ProxyClientSession *cs = reinterpret_cast(ssnp); - int count = 0; + ProxySession *cs = reinterpret_cast(ssnp); + int count = 0; if (cs && n > 0) { auto mem = static_cast(alloca(sizeof(std::string_view) * n)); count = cs->populate_protocol(mem, n); @@ -9357,7 +9357,7 @@ const char * TSHttpSsnClientProtocolStackContains(TSHttpSsn ssnp, const char *tag) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - ProxyClientSession *cs = reinterpret_cast(ssnp); + ProxySession *cs = reinterpret_cast(ssnp); return cs->protocol_contains(std::string_view{tag}); } diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 79e534f400b..e607538e2ef 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -76,7 +76,7 @@ extern "C" int plock(int); #include "ProxyConfig.h" #include "HttpProxyServerMain.h" #include "HttpBodyFactory.h" -#include "ProxyClientSession.h" +#include "ProxySession.h" #include "logging/Log.h" #include "CacheControl.h" #include "IPAllow.h" From 87a5e1b1ed90bf695bf18884d659467103b97eb4 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 29 Jan 2019 15:53:10 +0000 Subject: [PATCH 215/526] Adjust tests to make them more resilient for different curl versions. --- tests/gold_tests/tls/tls_client_verify.test.py | 4 ---- tests/gold_tests/tls/tls_client_verify2.test.py | 4 ---- tests/gold_tests/tls/tls_keepalive.test.py | 5 +++++ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py index 866c5bfab28..9860955df65 100644 --- a/tests/gold_tests/tls/tls_client_verify.test.py +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -83,7 +83,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") tr = Test.AddTestRun("Connect to foo.com with bad cert") tr.Setup.Copy("ssl/server.pem") @@ -93,7 +92,6 @@ tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) # Should fail with badly signed certs tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") tr = Test.AddTestRun("Connect to foo.com with cert") tr.Setup.Copy("ssl/signed-foo.pem") @@ -159,7 +157,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert handshake failure", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bar.com with cert") tr.Setup.Copy("ssl/signed-bar.pem") @@ -177,6 +174,5 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert unknown ca", "TLS handshake should fail") diff --git a/tests/gold_tests/tls/tls_client_verify2.test.py b/tests/gold_tests/tls/tls_client_verify2.test.py index e55376d8f2f..885e75f7ae5 100644 --- a/tests/gold_tests/tls/tls_client_verify2.test.py +++ b/tests/gold_tests/tls/tls_client_verify2.test.py @@ -99,7 +99,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bob.bar.com with cert") tr.Setup.Copy("ssl/signed-bob-bar.pem") @@ -117,14 +116,12 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("error", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bob.foo.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("alert", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bob.foo.com with cert") tr.Setup.Copy("ssl/signed-bob-foo.pem") @@ -142,7 +139,6 @@ tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 -tr.Processes.Default.Streams.All = Testers.ContainsExpression("error", "TLS handshake should fail") tr = Test.AddTestRun("Connect to bar.com without cert") tr.StillRunningAfter = ts diff --git a/tests/gold_tests/tls/tls_keepalive.test.py b/tests/gold_tests/tls/tls_keepalive.test.py index e420d54aec8..ff41e459ecd 100644 --- a/tests/gold_tests/tls/tls_keepalive.test.py +++ b/tests/gold_tests/tls/tls_keepalive.test.py @@ -24,6 +24,11 @@ Verify that the client-side keep alive is honored for TLS and different versions of HTTP ''' +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work"), + Condition.HasCurlFeature('http2') +) + ts = Test.MakeATSProcess("ts", select_ports=False) server = Test.MakeOriginServer("server") request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} From 25e91b43ca8533d5a75805f2bb27939bb2d7a789 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 31 Jan 2019 20:03:37 +0000 Subject: [PATCH 216/526] Adjust follow redirect logic to address a crash and avoid arguments shadowing member variables. --- proxy/http/HttpSM.cc | 14 +++++++------- proxy/http/HttpTransact.cc | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 5cb5da190a6..57caf2ba24f 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -7703,7 +7703,7 @@ HttpSM::do_redirect() } void -HttpSM::redirect_request(const char *redirect_url, const int redirect_len) +HttpSM::redirect_request(const char *arg_redirect_url, const int arg_redirect_len) { SMDebug("http_redirect", "[HttpSM::redirect_request]"); // get a reference to the client request header and client url and check to see if the url is valid @@ -7754,22 +7754,22 @@ HttpSM::redirect_request(const char *redirect_url, const int redirect_len) // reset the path from previous redirects (if any) t_state.redirect_info.redirect_url.path_set(nullptr, 0); - // redirectUrl.user_set(redirect_url, redirect_len); - redirectUrl.parse(redirect_url, redirect_len); + redirectUrl.parse(arg_redirect_url, arg_redirect_len); { int _scheme_len = -1; int _host_len = -1; - if (redirectUrl.scheme_get(&_scheme_len) == nullptr && redirectUrl.host_get(&_host_len) != nullptr && redirect_url[0] != '/') { + if (redirectUrl.scheme_get(&_scheme_len) == nullptr && redirectUrl.host_get(&_host_len) != nullptr && + arg_redirect_url[0] != '/') { // RFC7230 § 5.5 // The redirect URL lacked a scheme and so it is a relative URL. // The redirect URL did not begin with a slash, so we parsed some or all // of the the relative URI path as the host. // Prepend a slash and parse again. - char redirect_url_leading_slash[redirect_len + 1]; + char redirect_url_leading_slash[arg_redirect_len + 1]; redirect_url_leading_slash[0] = '/'; - memcpy(redirect_url_leading_slash + 1, redirect_url, redirect_len + 1); + memcpy(redirect_url_leading_slash + 1, arg_redirect_url, arg_redirect_len + 1); url_nuke_proxy_stuff(redirectUrl.m_url_impl); - redirectUrl.parse(redirect_url_leading_slash, redirect_len + 1); + redirectUrl.parse(redirect_url_leading_slash, arg_redirect_len + 1); } } diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 68ff019b31b..272c3397f01 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -7556,7 +7556,7 @@ HttpTransact::build_request(State *s, HTTPHdr *base_request, HTTPHdr *outgoing_r URL *r_url = &s->redirect_info.redirect_url; ink_assert(r_url->valid()); - base_request->url_get()->copy(r_url); + base_request->url_set(r_url); } else { // this is for multiple cache lookup URL *o_url = &s->cache_info.original_url; From 4da4957baac44ce09d0a1db78ecfb0840a1821b2 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 29 Jan 2019 16:27:58 -0600 Subject: [PATCH 217/526] Doc: HdrHeap. --- doc/appendices/glossary.en.rst | 10 + .../core-architecture/heap.en.rst | 193 ++++++++++++++++++ .../core-architecture/index.en.rst | 3 +- doc/uml/Makefile.am | 4 + doc/uml/hdr-heap-class.plantuml | 52 +++++ doc/uml/hdr-heap-str-alloc.plantuml | 29 +++ 6 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 doc/developer-guide/core-architecture/heap.en.rst create mode 100644 doc/uml/hdr-heap-class.plantuml create mode 100644 doc/uml/hdr-heap-str-alloc.plantuml diff --git a/doc/appendices/glossary.en.rst b/doc/appendices/glossary.en.rst index 863853ce575..f4ae4b8d9bb 100644 --- a/doc/appendices/glossary.en.rst +++ b/doc/appendices/glossary.en.rst @@ -42,6 +42,9 @@ Glossary A thread created by |TS| that has an :term:`event loop`. Event loops drive activity in |TS| and are responsible for all network I/O handling, hook processing, and scheduled events. + header heap + A heap to manage transaction local memory for HTTP headers. + session A single connection from a client to Traffic Server, covering all requests and responses on that connection. A session starts when the @@ -165,3 +168,10 @@ Glossary A plugin which operates only on transactions matching specific remap rules as defined in :file:`remap.config`. Contrast with :term:`global plugin`. + + variable sized class + A class where the instances vary in size. This is done by allocating a block of memory at least + as large as the class and then constructing a class instance at the start of the block. The + class must be provided the size of this extended memory during construction and is presumed + to use it as part of the instance. This generally requires calling a helper function to + create the instance and extra care when de-allocating. diff --git a/doc/developer-guide/core-architecture/heap.en.rst b/doc/developer-guide/core-architecture/heap.en.rst new file mode 100644 index 00000000000..5c3d397c641 --- /dev/null +++ b/doc/developer-guide/core-architecture/heap.en.rst @@ -0,0 +1,193 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs +.. highlight:: cpp +.. default-domain:: cpp + +.. _core-hdr-heap: + +Header Heap +*********** + +Memory for HTTP header data is kept in :term:`header heap`\s. + +Classes +======= + +.. class:: HdrHeapObjImpl + + This is the abstract base class for objects allocated in a :class:`HdrHeap`. This allows updating + objects in a heap in a generic way, without having to locate all of the pointers to the objects. + + The type of an instance stored in a heap must be one of the following values. + + .. enumerator:: HDR_HEAP_OBJ_EMPTY = 0 + + Used to mark invalid objects, ones not yet constructed or ones that have been destroyed. + + .. enumerator:: HDR_HEAP_OBJ_RAW = 1 + + Some sort of raw object, I have no idea. + + .. enumerator:: HDR_HEAP_OBJ_URL = 2 + + A URL object. + + .. enumerator:: HDR_HEAP_OBJ_HTTP_HEADER = 3 + + The header for an HTTP request or response. + + .. enumerator:: HDR_HEAP_OBJ_MIME_HEADER = 4 + + A MIME header, containing MIME style fields with names and values. + + .. enumerator:: HDR_HEAP_OBJ_FIELD_BLOCK = 5 + + Who the heck knows? + +.. class:: HdrStrHeap + + This is a :term:`variable sized class`, therefore new instance must be created by :func:`new_HdrStrHeap` + and deallocated by the :code:`destroy` method. + +.. function:: HdrStrHeap * new_HdrStrHeap(int n) + + Create and return a new instance of :class:`HdrStrHeap`. If :arg:`n` is less than ``HDR_STR_HEAP_DEFAULT_SIZE`` + it is increased to that value. + + If the allocated size is ``HDR_STR_HEAP_DEFAULT_SIZE`` (or smaller and upsized to that value) then + the instance is allocated from a thread local pool via :code:`strHeapAllocator`. If larger it + is allocated from global memory via :code:`ats_malloc`. + +.. class:: HdrHeap + + This is a :term:`variable sized class` and therefore new instances must be created by :func:`new_HdrHeap` + and deallocated by the :code:`destroy` method. + + :class:`HdrHeap` manages memory for heap objects directly and memory for strings via ancillary + heaps (which are instances of :class:`HdrStrHeap`). For the string heaps there is at most one + writeable heap, and up to :code:`HDR_BUF_RONLY_HEAPS` read only heaps. + + All objects in the internal heap must be subclasses of :class:`HdrHeapObjImpl`. + + .. function:: size_t required_space_for_evacuation() + + Calculate and return the total live string space for :arg:`this`. + + .. function:: void evacuate_from_str_heaps(HdrStrHeap * new_heap) + + Copy all live strings from the heap objects in :arg:`this` to :arg:`new_heap`. + + .. function:: void coalesce_str_heaps(int incoming_size) + + This garbage collects the string heaps in a half space style, by creating a new string space + (string heap), copying all of the strings there, and then discarding the existing string heaps. + + The total amount of live string space is calculated by + :func:`HdrHeap::required_space_for_evacuation` and a new string heap is created of a size at + least as large as the live string space plus :arg:`incoming_size` bytes. + + All of the live strings are moved to the new string heap by + :func:`HdrHeap::evacuate_from_str_heaps`, the existing string heaps are deallocated, and the + new string heap becomes the writeable string heap for the header heap. The end result is a + single writeable string heap and no read only string heaps, with all live strings resident in + that writeable string heap. + + .. function:: char * allocate_str(int bytes) + + Allocate :arg:`nbytes` of space for a string in the writeable string heap. A pointer to the + first byte is returned, or ``nullptr`` if the space could not be allocated. + + .. function:: HdrHeapObjImpl * allocate_obj(int nbytes, int type) + + Allocate a :arg:`type` object that is :arg:`nbytes` in size in the heap and return a pointer + to it, or ``nullptr`` if the object could not be allocated. + + :arg:`nbytes` must be at most ``HDR_MAX_ALLOC_SIZE``. + + The members of :class:`HdrHeapObjImpl` are initialized. Further initialization is the + responsibility of the caller. + + :arg:`type` must be one of the values specified in :class:`HdrHeapObjImpl`. + +.. function:: HdrHeap * new_HdrHeap(int n) + + Create and return a new instance of :class:`HdrHeap`. If :arg:`n` is less than ``HDR_HEAP_DEFAULT_SIZE`` + it is increased to that value. + + If the allocated size is ``HDR_HEAP_DEFAULT_SIZE`` (or smaller and upsized to that value) then + the instance is allocated from a thread local pool via :code:`hdrHeapAllocator`. If larger it + is allocated from global memory via :code:`ats_malloc`. + +.. topic:: Header Heap Class Structure + + .. figure:: /uml/images/hdr-heap-class.svg + + +Implementation +============== + +String Coalescence +------------------ + +String heaps do do not maintain lists of internal free space. Strings that are released are left in +place, creating dead space in the heap. For this reason it can become necessary to do a garbage +collection operation on the writeable string heap in the header heap by calling +:func:`HdrHeap::coalesce_str_heaps`. This is done when + +* The amount of dead space in the writable string heap exceeds ``MAX_LOST_STR_SPACE``. + +* The header heap is preparing to be serialized. + +* An external string heap is being added and all current read only string heap slots are used. + +Each heap object is responsible for providing a :code:`move_strings` method which copies its strings +to a new string heap. This is a source of pointer invalidation for other parts of the core and the +plugin API. For the latter, insulating from such string movement is the point of the +:c:type:`TSMLoc` type. + +String Allocation +----------------- + +Storage for a string is allocated by :func:`HdrHeap::allocate_str`. If the current amount of dead +space is too large, this is treated as an initial allocation failure. If there is no current +writeable string heap, one is created that is a least as large as the space requested and the size +of the previous writeable string heap. Space for the string is then allocated out of the writeable +string heap. If this fails due to lack of space the current writeable string heap is "demoted" to a +read only string heap and allocation retried (which will cause a new writeable string heap). If the +writeable string heap cannot be demoted due to lack of read only slots, the strings heaps are +coalesced with an additional size request of the requested string size. This will result in a single +writeable string heap and not read only heaps, the former containing all of the existing strings plus +sufficient space to allocate the new string. + +.. topic:: Decision Diagram + + .. figure:: /uml/images/hdr-heap-str-alloc.svg + +Object Allocation +----------------- + +Objects are allocated on the header heap by :func:`HdrHeap::allocate_obj`. Such objects must be one +of a compile time determined set of types [#]_. This method first tries to allocate the object in +existing free space. If that doesn't work then the allocator walks a list of :class:`HdrHeap` +instances looking for space. If no space is found anywhere, a new :class:`HdrHeap` instance is +created with twice the space of the last :class:`HdrHeap` in the list and added to the list to +try. + +Once space is found for the object, the base members of :class:`HdrHeapObjImpl` are initialized with +the objec type and size, with the :arg:`m_obj_flags` set to 0. + +.. rubric:: Footnotes. + +.. [#] Not that I can see any good reason for that, if virtual methods instead of :code:`switch` statements were used. diff --git a/doc/developer-guide/core-architecture/index.en.rst b/doc/developer-guide/core-architecture/index.en.rst index 9b8537548f7..a1efacb0e14 100644 --- a/doc/developer-guide/core-architecture/index.en.rst +++ b/doc/developer-guide/core-architecture/index.en.rst @@ -25,4 +25,5 @@ Core Architecture .. toctree:: :maxdepth: 1 - rpc.en \ No newline at end of file + heap.en + rpc.en diff --git a/doc/uml/Makefile.am b/doc/uml/Makefile.am index 956068f3f90..2a9fa761056 100644 --- a/doc/uml/Makefile.am +++ b/doc/uml/Makefile.am @@ -18,6 +18,7 @@ if BUILD_DOCS images := $(patsubst %.uml,images/%.svg,$(wildcard *.uml)) +images += $(patsubst %.plantuml,images/%.svg,$(wildcard *.plantuml)) PLANTUML_JAR := $(shell ../ext/plantuml_fetch.sh | tail -1) all-am: jar-check $(images) @@ -41,6 +42,9 @@ man: all-am images/%.svg : %.uml $(JAVA) -jar $(PLANTUML_JAR) -o images -tsvg $< +images/%.svg : %.plantuml + $(JAVA) -jar $(PLANTUML_JAR) -o images -tsvg $< + clean-local: rm -f images/*.svg diff --git a/doc/uml/hdr-heap-class.plantuml b/doc/uml/hdr-heap-class.plantuml new file mode 100644 index 00000000000..28212b6cc16 --- /dev/null +++ b/doc/uml/hdr-heap-class.plantuml @@ -0,0 +1,52 @@ +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +' Unless required by applicable law or agreed to in writing, software distributed under the License is distributed +' on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and limitations under the License. + +@startuml + +hide empty members + +class HdrStrHeap << (V, green) >> { + Storage for strings. + == + Extended Storage +} + +class StrHeapDesc { + m_ref_count_ptr + m_heap_start + m_heap_len + m_locked +} + +HdrStrHeap --|> RefCountObj +StrHeapDesc --o RefCountObj : m_ref_count_ptr + +note left of HdrStrHeap : Storage for strings in\nheap objects. + +class HdrHeap << (V, green) >> { + Storage for objects. + == + Extended Storage +} + +class HdrHeapObjImpl { + m_type + m_length + m_obj_flags +} + +HdrHeap --* "3" StrHeapDesc : m_ronly_heap +HdrHeap --o HdrStrHeap : m_read_write_heap +HdrHeap --o HdrHeap : next + +note left of HdrHeap : Heap storage for objects + +abstract class HdrHeapObjImpl + +HdrHeap --o "*" HdrHeapObjImpl : [heap storage] + +@enduml diff --git a/doc/uml/hdr-heap-str-alloc.plantuml b/doc/uml/hdr-heap-str-alloc.plantuml new file mode 100644 index 00000000000..7f54f297eef --- /dev/null +++ b/doc/uml/hdr-heap-str-alloc.plantuml @@ -0,0 +1,29 @@ +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +' Unless required by applicable law or agreed to in writing, software distributed under the License is distributed +' on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and limitations under the License. + +@startuml + +start + +if (dead space) then (too much) + :coalesce; +endif + +repeat +if (writeable heap?) then (false) + :create writeable heap; +endif +if (can allocate) then (true) + stop +endif +if (demote_heap) then (no open slots) + :coalesce; +endif +repeat while(loop) +detach + +@enduml From f8ae96426d31bea9c47c59eb7e97b94be65816ed Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 1 Feb 2019 09:20:51 -0600 Subject: [PATCH 218/526] Doc: TSEventThreadSelf. --- .../api/functions/TSEventThreadSelf.en.rst | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 doc/developer-guide/api/functions/TSEventThreadSelf.en.rst diff --git a/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst b/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst new file mode 100644 index 00000000000..0c7728aac6b --- /dev/null +++ b/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst @@ -0,0 +1,36 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSEventThreadSelf +***************** + +Synopsis +======== + +`#include ` + +.. function:: TSEventThread TSEventThreadSelf(void) + +Description +=========== + +Return a value that refers to the current :term:`event thread`. This must be called from a thread +with an :term:`event loop`. This will be true from any callback invoked from a hook or a scheduled +event. From 16f66f077f856a467f460e7abd60aa248dffce99 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 28 Jan 2019 13:13:13 -0600 Subject: [PATCH 219/526] Remove class StrTest, which is not used. --- proxy/hdrs/HdrHeap.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index be065a2cf72..70cf6d501d7 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -1170,11 +1170,6 @@ HdrStrHeap::expand(char *ptr, int old_size, int new_size) } } -struct StrTest { - char *ptr; - int len; -}; - #if TS_HAS_TESTS #include "tscore/TestBox.h" REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) From 57ecc4b56cbf619d65dd9c860f4df7cf44ca588d Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 7 Jan 2019 21:05:16 +0000 Subject: [PATCH 220/526] Fix OSCP stapling with dual cert --- iocore/net/OCSPStapling.cc | 120 +++++++++++++++++++++------------- iocore/net/SSLNetProcessor.cc | 2 + 2 files changed, 76 insertions(+), 46 deletions(-) diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index d5867b78b52..b4fcd77e8a7 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -46,25 +46,32 @@ struct certinfo { time_t expire_time; }; +/* + * In the case of multiple certificates associated with a SSL_CTX, we must store a map + * of cached responses + */ +using certinfo_map = std::map; + void -certinfo_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) +certinfo_map_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) { - certinfo *cinf = (certinfo *)ptr; + certinfo_map *map = (certinfo_map *)ptr; - if (!cinf) { + if (!map) { return; } - if (cinf->uri) { - OPENSSL_free(cinf->uri); - } - if (cinf->certname) { - ats_free(cinf->certname); - } - if (cinf->cid) { - OCSP_CERTID_free(cinf->cid); + + for (certinfo_map::iterator iter = map->begin(); iter != map->end(); ++iter) { + if (iter->second->uri) { + OPENSSL_free(iter->second->uri); + } + if (iter->second->certname) { + ats_free(iter->second->certname); + } + ink_mutex_destroy(&iter->second->stapling_mutex); + OPENSSL_free(iter->second); } - ink_mutex_destroy(&cinf->stapling_mutex); - OPENSSL_free(cinf); + free(map); } static int ssl_stapling_index = -1; @@ -75,7 +82,7 @@ ssl_stapling_ex_init() if (ssl_stapling_index != -1) { return; } - ssl_stapling_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, certinfo_free); + ssl_stapling_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, certinfo_map_free); } static X509 * @@ -136,7 +143,6 @@ stapling_get_issuer(SSL_CTX *ssl_ctx, X509 *x) bool ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) { - certinfo *cinf; scoped_X509 issuer; STACK_OF(OPENSSL_STRING) *aia = nullptr; @@ -145,15 +151,19 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) return false; } - cinf = (certinfo *)SSL_CTX_get_ex_data(ctx, ssl_stapling_index); - if (cinf) { + certinfo_map *map = static_cast(SSL_CTX_get_ex_data(ctx, ssl_stapling_index)); + if (map && map->find(cert) != map->end()) { Note("certificate already initialized for %s", certname); return false; } - cinf = (certinfo *)OPENSSL_malloc(sizeof(certinfo)); + if (!map) { + map = new certinfo_map; + } + certinfo *cinf = static_cast(OPENSSL_malloc(sizeof(certinfo))); if (!cinf) { Error("error allocating memory for %s", certname); + delete map; return false; } @@ -190,7 +200,8 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) goto err; } - SSL_CTX_set_ex_data(ctx, ssl_stapling_index, cinf); + map->insert(std::make_pair(cert, cinf)); + SSL_CTX_set_ex_data(ctx, ssl_stapling_index, map); Note("successfully initialized stapling for %s into SSL_CTX: %p", certname, ctx); return true; @@ -207,17 +218,21 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) if (cinf) { OPENSSL_free(cinf); } + if (map) { + delete map; + } return false; } -static certinfo * +static certinfo_map * stapling_get_cert_info(SSL_CTX *ctx) { - certinfo *cinf; + certinfo_map *map; - cinf = (certinfo *)SSL_CTX_get_ex_data(ctx, ssl_stapling_index); - if (cinf && cinf->cid) { - return cinf; + // Only return the map if it contains at least one element with a valid entry + map = static_cast(SSL_CTX_get_ex_data(ctx, ssl_stapling_index)); + if (map && !map->empty() && map->begin()->second && map->begin()->second->cid) { + return map; } return nullptr; @@ -427,7 +442,6 @@ void ocsp_update() { SSL_CTX *ctx; - certinfo *cinf = nullptr; OCSP_RESPONSE *resp = nullptr; time_t current_time; @@ -437,22 +451,27 @@ ocsp_update() for (unsigned i = 0; i < ctxCount; i++) { SSLCertContext *cc = certLookup->get(i); if (cc && cc->ctx) { - ctx = cc->ctx; - cinf = stapling_get_cert_info(ctx); - if (cinf) { - ink_mutex_acquire(&cinf->stapling_mutex); - current_time = time(nullptr); - if (cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) { - ink_mutex_release(&cinf->stapling_mutex); - if (stapling_refresh_response(cinf, &resp)) { - Debug("Successfully refreshed OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); - SSL_INCREMENT_DYN_STAT(ssl_ocsp_refreshed_cert_stat); + ctx = cc->ctx; + certinfo *cinf = nullptr; + certinfo_map *map = stapling_get_cert_info(ctx); + if (map) { + // Walk over all certs associated with this CTX + for (certinfo_map::iterator iter = map->begin(); iter != map->end(); ++iter) { + cinf = iter->second; + ink_mutex_acquire(&cinf->stapling_mutex); + current_time = time(nullptr); + if (cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) { + ink_mutex_release(&cinf->stapling_mutex); + if (stapling_refresh_response(cinf, &resp)) { + Debug("Successfully refreshed OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); + SSL_INCREMENT_DYN_STAT(ssl_ocsp_refreshed_cert_stat); + } else { + Error("Failed to refresh OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); + SSL_INCREMENT_DYN_STAT(ssl_ocsp_refresh_cert_failure_stat); + } } else { - Error("Failed to refresh OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); - SSL_INCREMENT_DYN_STAT(ssl_ocsp_refresh_cert_failure_stat); + ink_mutex_release(&cinf->stapling_mutex); } - } else { - ink_mutex_release(&cinf->stapling_mutex); } } } @@ -463,20 +482,29 @@ ocsp_update() int ssl_callback_ocsp_stapling(SSL *ssl) { - certinfo *cinf = nullptr; - time_t current_time; - // Assume SSL_get_SSL_CTX() is the same as reaching into the ssl structure // Using the official call, to avoid leaking internal openssl knowledge // originally was, cinf = stapling_get_cert_info(ssl->ctx); - cinf = stapling_get_cert_info(SSL_get_SSL_CTX(ssl)); - if (cinf == nullptr) { - Debug("ssl_ocsp", "ssl_callback_ocsp_stapling: failed to get certificate information"); + certinfo_map *map = stapling_get_cert_info(SSL_get_SSL_CTX(ssl)); + if (map == nullptr) { + Debug("ssl_ocsp", "ssl_callback_ocsp_stapling: failed to get certificate map"); + return SSL_TLSEXT_ERR_NOACK; + } + // Fetch the specific certificate used in this negotiation + X509 *cert = SSL_get_certificate(ssl); + if (!cert) { + Error("ssl_callback_ocsp_stapling: failed to get certificate"); + return SSL_TLSEXT_ERR_NOACK; + } + certinfo_map::iterator iter = map->find(cert); + if (iter == map->end()) { + Error("ssl_callback_ocsp_stapling: failed to get certificate information"); return SSL_TLSEXT_ERR_NOACK; } + certinfo *cinf = iter->second; ink_mutex_acquire(&cinf->stapling_mutex); - current_time = time(nullptr); + time_t current_time = time(nullptr); if (cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) { ink_mutex_release(&cinf->stapling_mutex); Debug("ssl_ocsp", "ssl_callback_ocsp_stapling: failed to get certificate status for %s", cinf->certname); diff --git a/iocore/net/SSLNetProcessor.cc b/iocore/net/SSLNetProcessor.cc index a79fd0fe667..cff8992429d 100644 --- a/iocore/net/SSLNetProcessor.cc +++ b/iocore/net/SSLNetProcessor.cc @@ -77,6 +77,8 @@ SSLNetProcessor::start(int, size_t stacksize) #ifdef TS_USE_TLS_OCSP if (SSLConfigParams::ssl_ocsp_enabled) { + // Call the update initially to get things populated + ocsp_update(); EventType ET_OCSP = eventProcessor.spawn_event_threads("ET_OCSP", 1, stacksize); eventProcessor.schedule_every(new OCSPContinuation(), HRTIME_SECONDS(SSLConfigParams::ssl_ocsp_update_period), ET_OCSP); } From 3d21674ccc729cd753c85cab6ffbebafa7f65386 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 14 Jan 2019 17:17:18 -0600 Subject: [PATCH 221/526] Regex: update to use string_view. --- include/tscore/Regex.h | 44 +++++++++++++++++++++++++++++----- proxy/http/remap/UrlRewrite.cc | 3 ++- src/tscore/Regex.cc | 22 +++++++---------- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/include/tscore/Regex.h b/include/tscore/Regex.h index 8888ae60094..f80121e47a6 100644 --- a/include/tscore/Regex.h +++ b/include/tscore/Regex.h @@ -23,6 +23,8 @@ #pragma once +#include + #include "tscore/ink_config.h" #ifdef HAVE_PCRE_PCRE_H @@ -45,16 +47,46 @@ class Regex { public: /// Default number of capture groups. - static constexpr size_t DEFAULT_GROUP_COUNT = 30; + static constexpr size_t DEFAULT_GROUP_COUNT = 10; Regex() = default; + ~Regex(); + + /** Compile the @a pattern into a regular expression. + * + * @param pattern Source pattern for regular expression (null terminated). + * @param flags Compilation flags. + * @return @a true if compiled successfully, @a false otherwise. + * + * @a flags should be the bitwise @c or of @c REFlags values. + */ bool compile(const char *pattern, const unsigned flags = 0); - // It is safe to call exec() concurrently on the same object instance - bool exec(const char *str); - bool exec(const char *str, int length); - bool exec(const char *str, int length, int *ovector, int ovecsize); + + /** Execute the regular expression. + * + * @param str String to match against. + * @return @c true if the patter matched, @a false if not. + * + * It is safe to call this method concurrently on the same instance of @a this. + */ + bool exec(std::string_view const &str); + + /** Execute the regular expression. + * + * @param str String to match against. + * @param ovector Capture results. + * @param ovecsize Number of elements in @a ovector. + * @return @c true if the patter matched, @a false if not. + * + * It is safe to call this method concurrently on the same instance of @a this. + * + * Each capture group takes 3 elements of @a ovector, therefore @a ovecsize must + * be a multiple of 3 and at least three times the number of desired capture groups. + */ + bool exec(std::string_view const &str, int *ovector, int ovecsize); + + /// @return The number of groups captured in the last call to @c exec. int get_capture_count(); - ~Regex(); private: pcre *regex = nullptr; diff --git a/proxy/http/remap/UrlRewrite.cc b/proxy/http/remap/UrlRewrite.cc index 3f3d0ed4cb4..13e39808d47 100644 --- a/proxy/http/remap/UrlRewrite.cc +++ b/proxy/http/remap/UrlRewrite.cc @@ -897,7 +897,8 @@ UrlRewrite::_regexMappingLookup(RegexMappingList ®ex_mappings, URL *request_u } int matches_info[MAX_REGEX_SUBS * 3]; - bool match_result = list_iter->regular_expression.exec(request_host, request_host_len, matches_info, countof(matches_info)); + bool match_result = + list_iter->regular_expression.exec(std::string_view(request_host, request_host_len), matches_info, countof(matches_info)); if (match_result == true) { Debug("url_rewrite_regex", diff --git a/src/tscore/Regex.cc b/src/tscore/Regex.cc index 79028927e8b..d067072a27f 100644 --- a/src/tscore/Regex.cc +++ b/src/tscore/Regex.cc @@ -21,6 +21,8 @@ limitations under the License. */ +#include + #include "tscore/ink_platform.h" #include "tscore/ink_thread.h" #include "tscore/ink_memory.h" @@ -101,25 +103,19 @@ Regex::get_capture_count() } bool -Regex::exec(const char *str) -{ - return exec(str, strlen(str)); -} - -bool -Regex::exec(const char *str, int length) +Regex::exec(std::string_view const &str) { - int ovector[DEFAULT_GROUP_COUNT]; - return exec(str, length, ovector, countof(ovector)); + std::array ovector; + return this->exec(str, ovector.data(), ovector.size()); } bool -Regex::exec(const char *str, int length, int *ovector, int ovecsize) +Regex::exec(std::string_view const &str, int *ovector, int ovecsize) { int rv; - rv = pcre_exec(regex, regex_extra, str, length, 0, 0, ovector, ovecsize); - return rv > 0 ? true : false; + rv = pcre_exec(regex, regex_extra, str.data(), str.size(), 0, 0, ovector, ovecsize); + return rv > 0; } Regex::~Regex() @@ -238,7 +234,7 @@ DFA::match(const char *str, int length) const dfa_pattern *p = _my_patterns; while (p) { - rc = p->_re->exec(str, length); + rc = p->_re->exec({str, size_t(length)}); if (rc > 0) { return p->_idx; } From a1ccd7af373bf334d27814ad6afa196382f256af Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 30 Jan 2019 20:18:17 -0600 Subject: [PATCH 222/526] Doc: More HdrHeap documentation. --- .../core-architecture/heap.en.rst | 65 ++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/doc/developer-guide/core-architecture/heap.en.rst b/doc/developer-guide/core-architecture/heap.en.rst index 5c3d397c641..188b2c90bc1 100644 --- a/doc/developer-guide/core-architecture/heap.en.rst +++ b/doc/developer-guide/core-architecture/heap.en.rst @@ -121,6 +121,15 @@ Classes :arg:`type` must be one of the values specified in :class:`HdrHeapObjImpl`. + .. function:: int marshal_length + + Compute and return the size of the buffer needed to serialize :arg:`this`. + + .. function:: int marshal(char * buffer, int length) + + Serialize :arg:`this` to :arg:`buffer` of size :arg:`length`. It is required that + :arg:`length` be at least the value returned by :func:`HdrHeap::marshal_length`. + .. function:: HdrHeap * new_HdrHeap(int n) Create and return a new instance of :class:`HdrHeap`. If :arg:`n` is less than ``HDR_HEAP_DEFAULT_SIZE`` @@ -148,8 +157,6 @@ collection operation on the writeable string heap in the header heap by calling * The amount of dead space in the writable string heap exceeds ``MAX_LOST_STR_SPACE``. -* The header heap is preparing to be serialized. - * An external string heap is being added and all current read only string heap slots are used. Each heap object is responsible for providing a :code:`move_strings` method which copies its strings @@ -188,6 +195,58 @@ try. Once space is found for the object, the base members of :class:`HdrHeapObjImpl` are initialized with the objec type and size, with the :arg:`m_obj_flags` set to 0. +Serialization +------------- + +Because heaps store the HTTP request / response data, a header heap needs to be serialized to be put +in to the cache. For performance reasons, it is desirable to be able to unserialize the serialized +data in place, rather than copying it again. That is, the data is read from disk into a block of +memory and then that memory is converted to a live data structure. In this case the memory used by +the heap is owned by some other object and the header heap must not do any clean up. This is +signaled by the `m_writeable` flag. In an unserialized header heap this is set to ``false`` and such +a header heap is not allowed to allocate any additional objects or strings - it is immutable. + +The primary mechanism to do this is to use swizzling on the pointers in the structure. During +serialization pointers are converted to offsets and during unserialization these offsets are +converted back to pointers. To make this simpler, unserialized header heaps are marked read only so +that updating does not have to be supported. Additionally, :class:`HdrHeap` is a POD and therefore +has no virtual function table pointer to be stored or restored [#]_. + +To serialize, first :func:`HdrHeap::marshal_length` is called to get a buffer size. The +serialization buffer is created with sufficient space for the header heap and that space is passed +to :func:`HdrHeap::marshal` to perform the actual serialization. The object heaps are serialized +followed by the string heaps. No coalescence is done, on the presumption that because the amount +of dead space is limited by coalescence (as needed) on every string creation. + +When serializing strings, each object is responsible for swizzling its own pointers. Because the +object heaps have already been serialized and all of the header heap object types are also PODs, +these serialized objects can have the pointer swizzling method, :code:`marshal`, called directly +on them. This method is provided with a set of "translations" which indicate the base offset for +each range of object and string heap memory. The object marshalling can then compute the correct +offset to store for each live string pointer. + +Inheriting Strings +------------------ + +The string heaps are designed to be reference counted so that they can be shared as read only +objects between heaps. This enables copying heap objects between heaps less expensive as the +strings pointers in them can be preserved in the new heap by sharing the string heaps in which +those strings reside. + +This can still be a bit complex as it is possible that the combined number of string heaps is more +than the limit. In this case, the target header heap does string coalescence so that it is reduced to +having a single writeable string heap with enough free space to hold all of the strings in the +source header heap. As a result, it is required that all heap objects already be present in the +target header heap before the strings are inherited. This means that the string coalescence will +properly copy the strings of and update the strings pointers in the copied heap objects. + .. rubric:: Footnotes. -.. [#] Not that I can see any good reason for that, if virtual methods instead of :code:`switch` statements were used. +.. [#] + + Not that I can see any good reason for that, if virtual methods instead of :code:`switch` + statements were used. + +.. [#] + + Which makes the initialization logic to "fixup" the virtual function pointer rather silly. From 76e50f0ea46923b39fc3a408826b8be6959fe799 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Fri, 1 Feb 2019 18:25:54 +0800 Subject: [PATCH 223/526] Avoid rescheduling HostDBContinuation to the ET_DNS thread. With TS-196 (commit a5345da8), the HostDBContinuation is no longer reschedule to the ET_DNS. However, the changes only applied to the `HostDBProcessor::getbyname_imm` interface. And the NCA feature was removed by TS-455 (commit 112b03aa), so it is safe to bring the changes to all the interfaces of HostDBProcessor. --- iocore/hostdb/HostDB.cc | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 8c5ac69cfc5..947b2c234b2 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -679,14 +679,7 @@ HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockad c->init(hash, opt); SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent); - // Since ProxyMutexPtr has a cast operator, gcc-3.x get upset - // about ambiguity when doing this comparison, so by reversing - // the operands, I force it to pick the cast operation /leif. - if (thread->mutex == cont->mutex) { - thread->schedule_in(c, MUTEX_RETRY_DELAY); - } else { - dnsProcessor.thread->schedule_imm(c); - } + thread->schedule_in(c, MUTEX_RETRY_DELAY); return &c->action; } @@ -797,11 +790,7 @@ HostDBProcessor::getSRVbyname_imm(Continuation *cont, process_srv_info_pfn proce c->init(hash, copt); SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent); - if (thread->mutex == cont->mutex) { - thread->schedule_in(c, MUTEX_RETRY_DELAY); - } else { - dnsProcessor.thread->schedule_imm(c); - } + thread->schedule_in(c, MUTEX_RETRY_DELAY); return &c->action; } @@ -903,11 +892,7 @@ HostDBProcessor::iterate(Continuation *cont) c->current_iterate_pos = 0; SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::iterateEvent); - if (thread->mutex == cont->mutex) { - thread->schedule_in(c, HOST_DB_RETRY_PERIOD); - } else { - dnsProcessor.thread->schedule_imm(c); - } + thread->schedule_in(c, HOST_DB_RETRY_PERIOD); return &c->action; } From 841697c62b0c2fab27c990177b07bd3dc3de999f Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 23 Jan 2019 15:54:59 -0600 Subject: [PATCH 224/526] Add IOBufferReader::block_read_view method. --- iocore/eventsystem/I_IOBuffer.h | 6 ++++++ iocore/eventsystem/P_IOBuffer.h | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h index dd1255bd376..b57d28fa435 100644 --- a/iocore/eventsystem/I_IOBuffer.h +++ b/iocore/eventsystem/I_IOBuffer.h @@ -714,6 +714,12 @@ class IOBufferReader */ int64_t block_read_avail(); + /** Get a view of the data available to read. + * + * @return A view encompassing currently available readable data. + */ + std::string_view block_read_view(); + void skip_empty_blocks(); /** diff --git a/iocore/eventsystem/P_IOBuffer.h b/iocore/eventsystem/P_IOBuffer.h index c34af228c20..d6a4bb49454 100644 --- a/iocore/eventsystem/P_IOBuffer.h +++ b/iocore/eventsystem/P_IOBuffer.h @@ -621,6 +621,13 @@ IOBufferReader::block_read_avail() return (int64_t)(block->end() - (block->start() + start_offset)); } +inline std::string_view +IOBufferReader::block_read_view() +{ + const char *start = this->start(); // empty blocks are skipped in here. + return start ? std::string_view{start, static_cast(block->end() - start)} : std::string_view{}; +} + TS_INLINE int IOBufferReader::block_count() { From ab7f97736b267228257ffa9e9056b2e244e985c7 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Fri, 1 Feb 2019 23:30:08 +0000 Subject: [PATCH 225/526] Don't Alert and immediate shutdown during a normal shutdown. --- mgmt/ProcessManager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc index d5932be6dc8..4f52c9e46a5 100644 --- a/mgmt/ProcessManager.cc +++ b/mgmt/ProcessManager.cc @@ -164,13 +164,13 @@ ProcessManager::processManagerThread(void *arg) if (pmgmt->require_lm) { ret = pmgmt->pollLMConnection(); - if (ret < 0 && pmgmt->running) { + if (ret < 0 && pmgmt->running && !shutdown_event_system) { Alert("exiting with read error from process manager: %s", strerror(-ret)); } } ret = pmgmt->processSignalQueue(); - if (ret < 0 && pmgmt->running) { + if (ret < 0 && pmgmt->running && !shutdown_event_system) { Alert("exiting with write error from process manager: %s", strerror(-ret)); } } From 2899db1d931a81a31446e0b221d5499602f9eb11 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 9 Jan 2019 08:24:28 -0600 Subject: [PATCH 226/526] Update HdrUtils to be C++17. --- proxy/hdrs/HdrUtils.cc | 225 ++++++++++++++----------------------- proxy/hdrs/HdrUtils.h | 223 ++++++++++++++++++++---------------- proxy/http/HttpTransact.cc | 35 +++--- 3 files changed, 225 insertions(+), 258 deletions(-) diff --git a/proxy/hdrs/HdrUtils.cc b/proxy/hdrs/HdrUtils.cc index 37a00acda71..f6ee4d4d895 100644 --- a/proxy/hdrs/HdrUtils.cc +++ b/proxy/hdrs/HdrUtils.cc @@ -1,24 +1,21 @@ /** @file - A brief file description + Utilities for HTTP MIME headers. - @section license License + @section license License - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. */ /**************************************************************************** @@ -34,147 +31,99 @@ #include "tscore/ink_platform.h" #include "HdrUtils.h" -#define GETNEXT() \ - { \ - cur += 1; \ - if (cur >= end) { \ - goto done; \ - } \ - } - void HdrCsvIter::find_csv() { - const char *cur, *end, *last_data, *csv_start; - -RETRY: - - cur = m_csv_start; - end = m_value_start + m_value_len; - last_data = nullptr; - csv_start = nullptr; - - if (cur >= end) { - goto done; - } -skip_leading_whitespace: - if (ParseRules::is_ws(*cur)) { - GETNEXT(); - goto skip_leading_whitespace; - } - csv_start = cur; -parse_value: - // ink_assert((',' > '"') && (',' > ' ') && (',' > '\t')); - // Cookie/Set-Cookie use ';' as the separator - if (m_separator == ',') { - while ((cur < end - 1) && (*cur > ',')) { - last_data = cur; - cur++; + char sep_list[3] = {'"', m_separator, 0}; + + m_csv.clear(); + + while (m_cur_field) { + while (m_value) { + bool in_quote_p = false; + // Index of next interesting character. + TextView::size_type idx = 0; + // Characters of interest in a null terminated string. + while (idx < m_value.size()) { + // Next character of interest. + idx = m_value.find_first_of(sep_list, idx); + if (TextView::npos == idx) { + // no more, consume all of @a src. + break; + } else if ('"' == m_value[idx]) { + if (in_quote_p) { + // quoted-pair only allowed inside a quoted-string. + // Can always back up because if @a in_quote_p there's a least a preceding quote. + if (m_value[idx - 1] != '\\') { + in_quote_p = false; + } + } else { + in_quote_p = true; + } + ++idx; + } else if (m_separator == m_value[idx]) { // separator. + if (in_quote_p) { + // quoted separator, skip and continue. + ++idx; + } else { + // found token, finish up. + break; + } + } + } + + // Trim and then see if there's anything left. + m_csv = m_value.take_prefix_at(idx).trim_if(&ParseRules::is_ws); + if (m_csv && '"' == m_csv[0]) { + m_csv.remove_prefix(1); + } + if (m_csv && '"' == m_csv[m_csv.size() - 1]) { + m_csv.remove_suffix(1); + } + // If there's any value after that, we're done. + if (m_csv) { + return; + } } - } - - if (*cur == m_separator) { - goto done; - } - if (*cur == '\"') { - // If the quote come before any text - // skip it - if (cur == csv_start) { - csv_start++; + // Current field is exhausted, try the next if following dupes. + if (m_follow_dups && m_cur_field->m_next_dup) { + this->field_init(m_cur_field->m_next_dup); + } else { + m_cur_field = nullptr; } - GETNEXT(); - goto parse_value_quote; - } - if (!ParseRules::is_ws(*cur)) { - last_data = cur; } - GETNEXT(); - goto parse_value; -parse_value_quote: - if ((*cur == '\"') && (cur[-1] != '\\')) { - GETNEXT(); - goto parse_value; - } - last_data = cur; - GETNEXT(); - goto parse_value_quote; -done: - m_csv_end = cur; - m_csv_start = csv_start; - - if (last_data) { - m_csv_len = (int)(last_data - csv_start) + 1; - } else { - // Nothing found. See if there is another - // field in the dup list - if (m_cur_field->m_next_dup && m_follow_dups) { - field_init(m_cur_field->m_next_dup); - goto RETRY; - } +} - m_csv_len = 0; +ts::TextView +HdrCsvIter::get_nth(MIMEField *field, int nth, bool follow_dups) +{ + ink_assert(nth >= 0); + int i = 0; + + auto tv = get_first(field, follow_dups); // index zero sub-value. + while (tv && nth > i) { + tv = get_next(); + ++i; } + return tv; } const char * HdrCsvIter::get_nth(MIMEField *field, int *len, int n, bool follow_dups) { - const char *s; - int i, l; - - ink_assert(n >= 0); - i = 0; - - s = get_first(field, &l, follow_dups); // get index 0 - while (s && (n > i)) { - s = get_next(&l); - i++; - } // if had index i, but want n > i, get next - *len = (s ? l : 0); // length is zero if NULL string - return (s); + auto tv = this->get_nth(field, n, follow_dups); + *len = int(tv.size()); + return tv.data(); } int HdrCsvIter::count_values(MIMEField *field, bool follow_dups) { - const char *s; - int count, l; - - count = 0; - s = get_first(field, &l, follow_dups); // get index 0 - while (s) { - s = get_next(&l); + int count = 0; + auto val = get_first(field, follow_dups); // get index 0 + while (val) { + val = get_next(); ++count; - } // get next - return (count); -} - -/* -int main() { - - char* tests[] = {"\"I\", \"hate\", \"strings\"", - "This, is a , test", - "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p", - "\"This is,\" a test" - }; - - for (int i = 0; i < 4; i++) { - printf ("Testing: %s\n", tests[i]); - - HdrCsvIter iter; - int len; - const char* v = iter.get_first(tests[i], strlen(tests[i]), &len); - - while (v) { - char* str_v = (char*)ats_malloc(len+1); - memcpy(str_v, v, len); - str_v[len] = '\0'; - printf ("%s|", str_v); - v = iter.get_next(&len); - } - - printf("\n\n"); - } - + } + return count; } -*/ diff --git a/proxy/hdrs/HdrUtils.h b/proxy/hdrs/HdrUtils.h index df40737feb7..2d128c73e75 100644 --- a/proxy/hdrs/HdrUtils.h +++ b/proxy/hdrs/HdrUtils.h @@ -33,68 +33,98 @@ #pragma once +#include "tscpp/util/TextView.h" #include "tscore/ParseRules.h" #include "MIME.h" -// csv = comma separated value +/** Accessor class to iterate over values in a multi-valued field. + * + * This implements the logic for quoted strings as specified in the RFC. + */ class HdrCsvIter { + using TextView = ts::TextView; + public: - // MIME standard separator ',' is used as the default value - // Set-cookie/Cookie uses ';' - HdrCsvIter(const char s = ',') - : m_value_start(nullptr), - m_value_len(0), - m_bytes_consumed(0), - m_follow_dups(false), - m_csv_start(nullptr), - m_csv_len(0), - m_csv_end(nullptr), - m_csv_index(0), - m_cur_field(nullptr), - m_separator(s) - { - } + /** Construct the iterator in the initial state. + * + * @param s The separator character for sub-values. + */ + HdrCsvIter(char s = ',') : m_separator(s) {} + + /** Get the first sub-value. + * + * @param m The multi-valued field. + * @param follow_dups Continue on to duplicate fields flag. + * @return A view of the first sub-value in multi-valued data. + */ + TextView get_first(const MIMEField *m, bool follow_dups = true); const char *get_first(const MIMEField *m, int *len, bool follow_dups = true); + + /** Get the next sub-value. + * + * @return A view of the next subvalue, or an empty view if no more values. + * + * If @a follow_dups was set in the constructor, this will continue on to additional fields + * if those fields have the same name as the original field (e.g, are duplicates). + */ + TextView get_next(); const char *get_next(int *len); + + /** Get the current sub-value. + * + * @return A view of the current subvalue, or an empty view if no more values. + * + * The state of the iterator is not modified. + */ + TextView get_current(); const char *get_current(int *len); + /** Get the @a nth sub-value in the field @a m. + * + * @param m Field. + * @param nth Index of the target sub-value. + * @param follow_dups Follow duplicate fields if necessary. + * @return The subvalue at index @a n, or an empty view if that does not exist. + */ + TextView get_nth(MIMEField *m, int nth, bool follow_dups = true); const char *get_nth(MIMEField *m, int *len, int n, bool follow_dups = true); - int count_values(MIMEField *field, bool follow_dups = true); - int get_index(); + int count_values(MIMEField *field, bool follow_dups = true); - int get_first_int(MIMEField *m, int *valid = nullptr); - int get_next_int(int *valid = nullptr); + /** Get the first sub-value as an integer. + * + * @param m Field with the value. + * @param result [out] Set to the integer sub-value. + * @return @c true if there was an integer and @a result was set, @c false otherwise. + */ + bool get_first_int(MIMEField *m, int &result); + + /** Get the next subvalue as an integer. + * + * @param result [out] Set to the integer sub-value. + * @return @c true if there was an integer and @a result was set, @c false otherwise. + */ + bool get_next_int(int &result); private: void find_csv(); - const char *m_value_start; - int m_value_len; - int m_bytes_consumed; - bool m_follow_dups; - - // m_csv_start - the start of the current comma separated value - // leading white space, and leading quotes have - // been skipped over - // m_csv_len - the length of the current comma separated value - // not including leading whitespace, trailing - // whitespace (unless quoted) or the terminating - // comma, or trailing quotes have been removed - // m_csv_end - the terminating comma of the csv unit. Either - // the terminating comma or the final character - // if this is the last csv in the string - // m_cvs_index - the integer index of current csv starting - // at zero - const char *m_csv_start; - int m_csv_len; - const char *m_csv_end; - int m_csv_index; - const MIMEField *m_cur_field; - - // for the Cookie/Set-cookie headers, the separator is ';' - const char m_separator; + /// The current field value. + TextView m_value; + + /// Whether duplicates are being followed. + bool m_follow_dups = false; + + /// The current sub-value. + TextView m_csv; + + /// The field containing the current sub-value. + const MIMEField *m_cur_field = nullptr; + + /// Separator for sub-values. + /// for the Cookie/Set-cookie headers, the separator is ';' + const char m_separator; // required constructor parameter, no initialization here. void field_init(const MIMEField *m); }; @@ -102,86 +132,83 @@ class HdrCsvIter inline void HdrCsvIter::field_init(const MIMEField *m) { - m_cur_field = m; - m_value_start = m->m_ptr_value; - m_value_len = m->m_len_value; - m_csv_start = m_value_start; + m_cur_field = m; + m_value.assign(m->m_ptr_value, m->m_len_value); } inline const char * HdrCsvIter::get_first(const MIMEField *m, int *len, bool follow_dups) { - field_init(m); + auto tv = this->get_first(m, follow_dups); + *len = static_cast(tv.size()); + return tv.data(); +} +inline ts::TextView +HdrCsvIter::get_first(const MIMEField *m, bool follow_dups) +{ + field_init(m); m_follow_dups = follow_dups; + this->find_csv(); + return m_csv; +} - m_bytes_consumed = 0; - m_csv_index = -1; - - if (m_csv_start) { - find_csv(); - } else { - m_csv_len = 0; - } - - *len = m_csv_len; - return m_csv_start; +inline ts::TextView +HdrCsvIter::get_next() +{ + this->find_csv(); + return m_csv; } inline const char * HdrCsvIter::get_next(int *len) { - if (m_csv_start) { - // Skip past the current csv - m_csv_start = m_csv_end + 1; - find_csv(); - } + auto tv = this->get_next(); + *len = static_cast(tv.size()); + return tv.data(); +} - *len = m_csv_len; - return m_csv_start; +inline ts::TextView +HdrCsvIter::get_current() +{ + return m_csv; } inline const char * HdrCsvIter::get_current(int *len) { - *len = m_csv_len; - return m_csv_start; + *len = static_cast(m_csv.size()); + return m_csv.data(); } -inline int -HdrCsvIter::get_first_int(MIMEField *m, int *valid) +inline bool +HdrCsvIter::get_first_int(MIMEField *m, int &result) { - int len; - const char *r = get_first(m, &len); - - if (r) { - if (valid) { - *valid = 1; + auto val = this->get_first(m); + + if (val) { + TextView parsed; + int n = ts::svtoi(val, &parsed); + if (parsed) { + result = n; + return true; } - return ink_atoi(r, len); - } else { - if (valid) { - *valid = 0; - } - return 0; } + return false; } -inline int -HdrCsvIter::get_next_int(int *valid) +inline bool +HdrCsvIter::get_next_int(int &result) { - int len; - const char *r = get_next(&len); - - if (r) { - if (valid) { - *valid = 1; - } - return ink_atoi(r, len); - } else { - if (valid) { - *valid = 0; + auto val = this->get_next(); + + if (val) { + TextView parsed; + int n = ts::svtoi(val, &parsed); + if (parsed) { + result = n; + return true; } - return 0; } + return false; } diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 272c3397f01..604db29a9f5 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -8703,42 +8703,34 @@ HttpTransact::update_size_and_time_stats(State *s, ink_hrtime total_time, ink_hr void HttpTransact::delete_warning_value(HTTPHdr *to_warn, HTTPWarningCode warning_code) { - int w_code = (int)warning_code; + int w_code = static_cast(warning_code); MIMEField *field = to_warn->field_find(MIME_FIELD_WARNING, MIME_LEN_WARNING); - ; // Loop over the values to see if we need to do anything if (field) { HdrCsvIter iter; - - int valid; int val_code; - - const char *value_str; - int value_len; - MIMEField *new_field = nullptr; - val_code = iter.get_first_int(field, &valid); - while (valid) { + bool valid_p = iter.get_first_int(field, val_code); + + while (valid_p) { if (val_code == w_code) { - // Ok, found the value we're look to delete - // Look over and create a new field - // appending all elements that are not this - // value - val_code = iter.get_first_int(field, &valid); + // Ok, found the value we're look to delete Look over and create a new field appending all + // elements that are not this value + valid_p = iter.get_first_int(field, val_code); - while (valid) { + while (valid_p) { if (val_code != warning_code) { - value_str = iter.get_current(&value_len); + auto value = iter.get_current(); if (new_field) { - new_field->value_append(to_warn->m_heap, to_warn->m_mime, value_str, value_len, true); + new_field->value_append(to_warn->m_heap, to_warn->m_mime, value.data(), value.size(), true); } else { new_field = to_warn->field_create(); - to_warn->field_value_set(new_field, value_str, value_len); + to_warn->field_value_set(new_field, value.data(), value.size()); } } - val_code = iter.get_next_int(&valid); + valid_p = iter.get_next_int(val_code); } to_warn->field_delete(MIME_FIELD_WARNING, MIME_LEN_WARNING); @@ -8749,8 +8741,7 @@ HttpTransact::delete_warning_value(HTTPHdr *to_warn, HTTPWarningCode warning_cod return; } - - val_code = iter.get_next_int(&valid); + valid_p = iter.get_next_int(val_code); } } } From 2913eb97ecd0ea7c3d9575bdd05c7aa941db019b Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 11 Jan 2019 15:51:05 -0600 Subject: [PATCH 227/526] Add HdrCsvIter tests. --- CMakeLists.txt | 2 + proxy/hdrs/Makefile.am | 23 ++++- proxy/hdrs/unit_tests/test_HdrUtils.cc | 122 ++++++++++++++++++++++++ proxy/hdrs/unit_tests/unit_test_main.cc | 25 +++++ 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 proxy/hdrs/unit_tests/test_HdrUtils.cc create mode 100644 proxy/hdrs/unit_tests/unit_test_main.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index efaa9f8f067..03d296b9082 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,9 +86,11 @@ CC_EXEC(test_tsutil src/tscpp/util/unit_tests) CPP_LIB(proxy proxy proxy) CPP_ADD_SOURCES(proxy proxy/http) +CPP_ADD_SOURCES(proxy proxy/http/unit_tests) CPP_ADD_SOURCES(proxy proxy/http2) CPP_ADD_SOURCES(proxy proxy/http/remap) CPP_ADD_SOURCES(proxy proxy/hdrs) +CPP_ADD_SOURCES(proxy proxy/hdrs/unit_tests) CPP_ADD_SOURCES(proxy proxy/logging) CPP_LIB(iocore iocore iocore) diff --git a/proxy/hdrs/Makefile.am b/proxy/hdrs/Makefile.am index 0f08679ed40..172c0d83e46 100644 --- a/proxy/hdrs/Makefile.am +++ b/proxy/hdrs/Makefile.am @@ -61,9 +61,9 @@ load_http_hdr_LDADD = -L. -lhdrs \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la -check_PROGRAMS = test_mime +check_PROGRAMS = test_mime test_proxy_hdrs -TESTS = test_mime +TESTS = $(check_PROGRAMS) test_mime_LDADD = -L. -lhdrs \ $(top_builddir)/src/tscore/libtscore.la \ @@ -77,6 +77,25 @@ test_mime_LDADD = -L. -lhdrs \ test_mime_SOURCES = test_mime.cc +test_proxy_hdrs_CPPFLAGS = $(AM_CPPFLAGS)\ + -I$(abs_top_srcdir)/tests/include + +test_proxy_hdrs_SOURCES = \ + unit_tests/unit_test_main.cc \ + unit_tests/test_HdrUtils.cc + +test_proxy_hdrs_LDADD = \ + $(top_builddir)/src/tscore/libtscore.la \ + -L. -lhdrs \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/proxy/shared/libUglyLogStubs.a \ + @HWLOC_LIBS@ \ + @LIBCAP@ + #test_UNUSED_SOURCES = \ # test_urlhash.cc diff --git a/proxy/hdrs/unit_tests/test_HdrUtils.cc b/proxy/hdrs/unit_tests/test_HdrUtils.cc new file mode 100644 index 00000000000..fc178a256f9 --- /dev/null +++ b/proxy/hdrs/unit_tests/test_HdrUtils.cc @@ -0,0 +1,122 @@ +/** @file + + Catch-based tests for HdrsUtils.cc + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "catch.hpp" + +#include "HdrHeap.h" +#include "MIME.h" +#include "HdrUtils.h" + +TEST_CASE("HdrUtils", "[proxy][hdrutils]") +{ + static constexpr ts::TextView text{"One: alpha\r\n" + "Two: alpha, bravo\r\n" + "Three: zwoop, \"A,B\" , , phil , \"unterminated\r\n" + "Five: alpha, bravo, charlie\r\n" + "Four: itchi, \"ni, \\\"san\" , \"\" , \"\r\n" + "Five: delta, echo\r\n" + "\r\n"}; + + static constexpr std::string_view ONE_TAG{"One"}; + static constexpr std::string_view TWO_TAG{"Two"}; + static constexpr std::string_view THREE_TAG{"Three"}; + static constexpr std::string_view FOUR_TAG{"Four"}; + static constexpr std::string_view FIVE_TAG{"Five"}; + + HdrHeap *heap = new_HdrHeap(HDR_HEAP_DEFAULT_SIZE + 64); + MIMEParser parser; + char const *real_s = text.data(); + char const *real_e = text.data_end(); + MIMEHdr mime; + + mime.create(heap); + mime_parser_init(&parser); + + auto result = mime_parser_parse(&parser, heap, mime.m_mime, &real_s, real_e, false, true); + REQUIRE(PARSE_RESULT_DONE == result); + + HdrCsvIter iter; + + MIMEField *field{mime.field_find(ONE_TAG.data(), int(ONE_TAG.size()))}; + REQUIRE(field != nullptr); + + auto value = iter.get_first(field); + REQUIRE(value == "alpha"); + + field = mime.field_find(TWO_TAG.data(), int(TWO_TAG.size())); + value = iter.get_first(field); + REQUIRE(value == "alpha"); + value = iter.get_next(); + REQUIRE(value == "bravo"); + value = iter.get_next(); + REQUIRE(value.empty()); + + field = mime.field_find(THREE_TAG.data(), int(THREE_TAG.size())); + value = iter.get_first(field); + REQUIRE(value == "zwoop"); + value = iter.get_next(); + REQUIRE(value == "A,B"); // quotes escape separator, and are stripped. + value = iter.get_next(); + REQUIRE(value == "phil"); + value = iter.get_next(); + REQUIRE(value == "unterminated"); + value = iter.get_next(); + REQUIRE(value.empty()); + + field = mime.field_find(FOUR_TAG.data(), int(FOUR_TAG.size())); + value = iter.get_first(field); + REQUIRE(value == "itchi"); + value = iter.get_next(); + REQUIRE(value == "ni, \\\"san"); // verify escaped quotes are passed through. + value = iter.get_next(); + REQUIRE(value.empty()); + + // Check that duplicates are handled correctly. + field = mime.field_find(FIVE_TAG.data(), int(FIVE_TAG.size())); + value = iter.get_first(field); + REQUIRE(value == "alpha"); + value = iter.get_next(); + REQUIRE(value == "bravo"); + value = iter.get_next(); + REQUIRE(value == "charlie"); + value = iter.get_next(); + REQUIRE(value == "delta"); + value = iter.get_next(); + REQUIRE(value == "echo"); + value = iter.get_next(); + REQUIRE(value.empty()); + + field = mime.field_find(FIVE_TAG.data(), int(FIVE_TAG.size())); + value = iter.get_first(field, false); + REQUIRE(value == "alpha"); + value = iter.get_next(); + REQUIRE(value == "bravo"); + value = iter.get_next(); + REQUIRE(value == "charlie"); + value = iter.get_next(); + REQUIRE(value.empty()); +} diff --git a/proxy/hdrs/unit_tests/unit_test_main.cc b/proxy/hdrs/unit_tests/unit_test_main.cc new file mode 100644 index 00000000000..6aed3a63097 --- /dev/null +++ b/proxy/hdrs/unit_tests/unit_test_main.cc @@ -0,0 +1,25 @@ +/** @file + + This file used for catch based tests. It is the main() stub. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" From 33818ba9aa228bd451b004d2d5f408fad7471a60 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 31 Jan 2019 22:48:11 +0000 Subject: [PATCH 228/526] Adjust connection timeout for TLS --- iocore/net/UnixNetVConnection.cc | 5 +++-- proxy/http/HttpSM.cc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 8b117f1ae36..181deb3e00d 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -358,8 +358,9 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) // vc is an SSLNetVConnection. if (!vc->getSSLHandShakeComplete()) { if (vc->trackFirstHandshake()) { - // Send the write ready on up to the state machine - write_signal_and_update(VC_EVENT_WRITE_READY, vc); + // Eat the first write-ready. Until the TLS handshake is complete, + // we should still be under the connect timeout and shouldn't bother + // the state machine until the TLS handshake is complete vc->write.triggered = 0; nh->write_ready_list.remove(vc); } diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 57caf2ba24f..56e2b817a7b 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1784,6 +1784,7 @@ HttpSM::state_http_server_open(int event, void *data) } return 0; } + case VC_EVENT_READ_COMPLETE: case VC_EVENT_WRITE_READY: case VC_EVENT_WRITE_COMPLETE: // Update the time out to the regular connection timeout. From fc4975d97bfe652442b62cb3e1a614abd92952b5 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Fri, 1 Feb 2019 23:05:47 +0000 Subject: [PATCH 229/526] Address performance issue by removing use of read_avail --- proxy/http2/Http2ConnectionState.cc | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 2587a89067d..a6843ec1a59 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -1354,7 +1354,6 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len const ssize_t window_size = std::min(this->client_rwnd, stream->client_rwnd); const size_t buf_len = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_DATA]); const size_t write_available_size = std::min(buf_len, static_cast(window_size)); - size_t read_available_size = 0; payload_length = 0; uint8_t flags = 0x00; @@ -1363,23 +1362,21 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); - if (current_reader) { - read_available_size = static_cast(current_reader->read_avail()); - } else { + if (!current_reader) { Http2StreamDebug(this->ua_session, stream->get_id(), "couldn't get data reader"); return Http2SendDataFrameResult::ERROR; } // Select appropriate payload length - if (read_available_size > 0) { + if (current_reader->is_read_avail_more_than(0)) { // We only need to check for window size when there is a payload if (window_size <= 0) { Http2StreamDebug(this->ua_session, stream->get_id(), "No window"); return Http2SendDataFrameResult::NO_WINDOW; } // Copy into the payload buffer. Seems like we should be able to skip this copy step - payload_length = std::min(read_available_size, write_available_size); - current_reader->memcpy(payload_buffer, static_cast(payload_length)); + payload_length = write_available_size; + payload_length = current_reader->read(payload_buffer, static_cast(write_available_size)); } else { payload_length = 0; } @@ -1392,7 +1389,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len return Http2SendDataFrameResult::NO_PAYLOAD; } - if (stream->is_body_done() && read_available_size <= write_available_size) { + if (stream->is_body_done() && !current_reader->is_read_avail_more_than(0)) { flags |= HTTP2_FLAGS_DATA_END_STREAM; } @@ -1410,7 +1407,6 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len data.finalize(payload_length); stream->update_sent_count(payload_length); - current_reader->consume(payload_length); // xmit event SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); From ce23b3cb9145edfd4f5162a451f83f5a62e5ffa3 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Mon, 28 Jan 2019 17:59:34 -0600 Subject: [PATCH 230/526] Add tsapi Au test. Initially just testing TSHttpTxnEffectiveUrlStringGet() function. --- .../pluginTest/test_hooks/spurious.in | 4 - .../pluginTest/test_hooks/test_hooks.test.py | 5 +- tests/gold_tests/pluginTest/tsapi/log.gold | 10 + .../pluginTest/tsapi/ssl/server.key | 15 ++ .../pluginTest/tsapi/ssl/server.pem | 32 +++ .../gold_tests/pluginTest/tsapi/tsapi.test.py | 93 +++++++++ tests/tools/plugins/test_tsapi.cc | 197 ++++++++++++++++++ 7 files changed, 349 insertions(+), 7 deletions(-) delete mode 100644 tests/gold_tests/pluginTest/test_hooks/spurious.in create mode 100644 tests/gold_tests/pluginTest/tsapi/log.gold create mode 100644 tests/gold_tests/pluginTest/tsapi/ssl/server.key create mode 100644 tests/gold_tests/pluginTest/tsapi/ssl/server.pem create mode 100644 tests/gold_tests/pluginTest/tsapi/tsapi.test.py create mode 100644 tests/tools/plugins/test_tsapi.cc diff --git a/tests/gold_tests/pluginTest/test_hooks/spurious.in b/tests/gold_tests/pluginTest/test_hooks/spurious.in deleted file mode 100644 index a88748bbbdf..00000000000 --- a/tests/gold_tests/pluginTest/test_hooks/spurious.in +++ /dev/null @@ -1,4 +0,0 @@ -Global: event=TS_EVENT_VCONN_START -Global: ssl flag=1 -Global: event=TS_EVENT_VCONN_CLOSE -Global: ssl flag=1 diff --git a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py index ae413defe3f..ef7b08688b8 100644 --- a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py +++ b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py @@ -67,9 +67,9 @@ ) tr = Test.AddTestRun() -# Wait for the micro server +# Probe server port to check if ready. tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) -# Delay on readiness of our ssl ports +# Probe TS cleartext port to check if ready (probing TLS port causes spurious VCONN hook triggers). tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) # tr.Processes.Default.Command = ( @@ -78,7 +78,6 @@ tr.Processes.Default.ReturnCode = 0 tr = Test.AddTestRun() -# A small delay so the test_hooks test plugin can assume there is only one HTTP transaction in progress at a time. tr.Processes.Default.Command = ( 'curl --verbose --ipv4 --http2 --insecure --header "Host: one" https://localhost:{0}/argh'.format(ts.Variables.ssl_port) ) diff --git a/tests/gold_tests/pluginTest/tsapi/log.gold b/tests/gold_tests/pluginTest/tsapi/log.gold new file mode 100644 index 00000000000..3960bfec884 --- /dev/null +++ b/tests/gold_tests/pluginTest/tsapi/log.gold @@ -0,0 +1,10 @@ +Global: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +TSHttpTxnEffectiveUrlStringGet(): http://myhost.test:SERVER_PORT/ +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +TSHttpTxnEffectiveUrlStringGet(): http://myhost.test:SERVER_PORT/ +Global: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +TSHttpTxnEffectiveUrlStringGet(): https://myhost.test:SERVER_PORT/ +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +TSHttpTxnEffectiveUrlStringGet(): https://myhost.test:SERVER_PORT/ diff --git a/tests/gold_tests/pluginTest/tsapi/ssl/server.key b/tests/gold_tests/pluginTest/tsapi/ssl/server.key new file mode 100644 index 00000000000..4c7a661a6bd --- /dev/null +++ b/tests/gold_tests/pluginTest/tsapi/ssl/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/tsapi/ssl/server.pem b/tests/gold_tests/pluginTest/tsapi/ssl/server.pem new file mode 100644 index 00000000000..58b9b9715b7 --- /dev/null +++ b/tests/gold_tests/pluginTest/tsapi/ssl/server.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICszCCAhwCCQD4jSkztmlO1TANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTcwODI4MDM0NDQ1WhcNMjcwODI2MDM0NDQ1WjCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 +uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE +lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 +Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEATX7975NdhIbJ +glda+sXI9a86GgOpiuKO+vKubRJQZA+UlPf2vHEONjC2+7Y1aZvZYaKYL74vxGky +zkgp6ANSPl45lqD632x0e1Z7vzW5TkqK1JB2/xH2WgDcQZmP0FuQHzVNs4GjghDr +HCp1+sQDhfPB4aLmLFeyN0TkhdH1N3M= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/tsapi/tsapi.test.py b/tests/gold_tests/pluginTest/tsapi/tsapi.test.py new file mode 100644 index 00000000000..6293ae0cb59 --- /dev/null +++ b/tests/gold_tests/pluginTest/tsapi/tsapi.test.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test TS API. +''' + +Test.SkipUnless( + Condition.HasATSFeature('TS_USE_TLS_ALPN'), + Condition.HasCurlFeature('http2'), +) +Test.ContinueOnFail = True + +# test_tsapi.so will output test logging to this file. +Test.Env["OUTPUT_FILE"] = Test.RunDirectory + "/log.txt" + +server = Test.MakeOriginServer("server") + +request_header = { + "headers": "GET / HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": "" } +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "112233" } +server.addResponse("sessionlog.json", request_header, response_header) + +ts = Test.MakeATSProcess("ts", select_ports=False) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 + +ts.Disk.records_config.update({ + 'proxy.config.http.cache.http': 0, # Make sure each request is forwarded to the origin server. + 'proxy.config.proxy_name': 'Poxy_Proxy', # This will be the server name. + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': ( + 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port)), + 'proxy.config.url_remap.remap_required': 0, + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|test_tsapi', +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + "map http://myhost.test:{0} http://127.0.0.1:{0}".format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + "map https://myhost.test:{0} http://127.0.0.1:{0}".format(server.Variables.Port) +) + +Test.PreparePlugin(Test.Variables.AtsTestToolsDir + '/plugins/test_tsapi.cc', ts) + +tr = Test.AddTestRun() +# Probe server port to check if ready. +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +# Probe TS cleartext port to check if ready. +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +# +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --header "Host: myhost.test:{0}" http://localhost:{1}/'.format(server.Variables.Port, ts.Variables.port) +) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --http2 --insecure --header ' + + '"Host: myhost.test:{0}" https://localhost:{1}/'.format(server.Variables.Port, ts.Variables.ssl_port) +) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +# Change server port number (which can vary) to a fixed string for compare to gold file. +tr.Processes.Default.Command = "sed 's/:{0}/:SERVER_PORT/' < {1}/log.txt > {1}/log2.txt".format( + server.Variables.Port, Test.RunDirectory) +tr.Processes.Default.ReturnCode = 0 +f = tr.Disk.File("log2.txt") +f.Content = "log.gold" diff --git a/tests/tools/plugins/test_tsapi.cc b/tests/tools/plugins/test_tsapi.cc new file mode 100644 index 00000000000..986a904c751 --- /dev/null +++ b/tests/tools/plugins/test_tsapi.cc @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* +Regression testing code for TS API. Not comprehensive, hopefully will be built up over time. +*/ + +#include +#include +#include + +#include + +// TSReleaseAssert() doesn't seem to produce any logging output for a debug build, so do both kinds of assert. +// +#define ALWAYS_ASSERT(EXPR) \ + { \ + TSAssert(EXPR); \ + TSReleaseAssert(EXPR); \ + } + +namespace +{ +#define PINAME "test_tsapi" +char PIName[] = PINAME; + +// NOTE: It's important to flush this after writing so that a gold test using this plugin can examine the log before TS +// terminates. +// +std::fstream logFile; + +TSCont tCont, gCont; + +void +testsForReadReqHdrHook(TSHttpTxn txn) +{ + logFile << "TSHttpTxnEffectiveUrlStringGet(): "; + int urlLength; + char *urlStr = TSHttpTxnEffectiveUrlStringGet(txn, &urlLength); + if (!urlStr) { + logFile << "URL null" << std::endl; + } else if (0 == urlLength) { + logFile << "URL length zero" << std::endl; + } else if (0 > urlLength) { + logFile << "URL length negative" << std::endl; + } else { + logFile << std::string_view(urlStr, urlLength) << std::endl; + + TSfree(urlStr); + } +} + +int +transactionContFunc(TSCont, TSEvent event, void *eventData) +{ + logFile << "Transaction: event=" << TSHttpEventNameLookup(event) << std::endl; + + TSDebug(PIName, "Transaction: event=%s(%d) eventData=%p", TSHttpEventNameLookup(event), event, eventData); + + switch (event) { + case TS_EVENT_HTTP_READ_REQUEST_HDR: { + auto txn = static_cast(eventData); + + testsForReadReqHdrHook(txn); + + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + } break; + + default: { + ALWAYS_ASSERT(false) + } break; + + } // end switch + + return 0; +} + +int +globalContFunc(TSCont, TSEvent event, void *eventData) +{ + logFile << "Global: event=" << TSHttpEventNameLookup(event) << std::endl; + + TSDebug(PIName, "Global: event=%s(%d) eventData=%p", TSHttpEventNameLookup(event), event, eventData); + + switch (event) { + case TS_EVENT_HTTP_TXN_START: { + auto txn = static_cast(eventData); + + TSHttpTxnHookAdd(txn, TS_HTTP_READ_REQUEST_HDR_HOOK, tCont); + + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + } break; + + case TS_EVENT_HTTP_READ_REQUEST_HDR: { + auto txn = static_cast(eventData); + + testsForReadReqHdrHook(txn); + + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + } break; + + default: { + ALWAYS_ASSERT(false) + } break; + + } // end switch + + return 0; +} + +} // end anonymous namespace + +void +TSPluginInit(int argc, const char *argv[]) +{ + TSDebug(PIName, "TSPluginInit()"); + + TSPluginRegistrationInfo info; + + info.plugin_name = PIName; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError(PINAME ": Plugin registration failed"); + + return; + } + + const char *fileSpec = std::getenv("OUTPUT_FILE"); + + if (nullptr == fileSpec) { + TSError(PINAME ": Environment variable OUTPUT_FILE not found."); + + return; + } + + // Disable output buffering for logFile, so that explicit flushing is not necessary. + logFile.rdbuf()->pubsetbuf(nullptr, 0); + + logFile.open(fileSpec, std::ios::out); + if (!logFile.is_open()) { + TSError(PINAME ": could not open log file \"%s\"", fileSpec); + + return; + } + + // Mutex to protext the logFile object. + // + TSMutex mtx = TSMutexCreate(); + + gCont = TSContCreate(globalContFunc, mtx); + + TSHttpHookAdd(TS_HTTP_TXN_START_HOOK, gCont); + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, gCont); + + tCont = TSContCreate(transactionContFunc, mtx); +} + +namespace +{ +class Cleanup +{ +public: + ~Cleanup() + { + // In practice it is not strictly necessary to destroy remaining continuations on program exit. + + if (tCont) { + TSContDestroy(tCont); + } + if (gCont) { + TSContDestroy(gCont); + } + } +}; + +// Do any needed cleanup for this source file at program termination time. +// +Cleanup cleanup; + +} // end anonymous namespace From be56b3aa010723d15fdcc77ff2cb85fbaceb8fc5 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Wed, 30 Jan 2019 18:02:47 +0000 Subject: [PATCH 231/526] Use POSIX ERE for uri signing regex evaluation --- plugins/experimental/uri_signing/match.c | 29 ++++++++++++------ .../unit_tests/uri_signing_test.cc | 30 +++++++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/plugins/experimental/uri_signing/match.c b/plugins/experimental/uri_signing/match.c index 18fb31a3017..40d4d472e93 100644 --- a/plugins/experimental/uri_signing/match.c +++ b/plugins/experimental/uri_signing/match.c @@ -16,10 +16,10 @@ * limitations under the License. */ +#include #include "common.h" #include "ts/ts.h" #include -#include #include bool @@ -31,16 +31,27 @@ match_hash(const char *needle, const char *haystack) bool match_regex(const char *pattern, const char *uri) { - const char *err; - int err_off; + struct re_pattern_buffer pat_buff; + + pat_buff.translate = 0; + pat_buff.fastmap = 0; + pat_buff.buffer = 0; + pat_buff.allocated = 0; + + re_syntax_options = RE_SYNTAX_POSIX_MINIMAL_EXTENDED; + PluginDebug("Testing regex pattern /%s/ against \"%s\"", pattern, uri); - pcre *re = pcre_compile(pattern, PCRE_ANCHORED | PCRE_UCP | PCRE_UTF8, &err, &err_off, NULL); - if (!re) { - PluginDebug("Regex /%s/ failed to compile.", pattern); + + const char *comp_err = re_compile_pattern(pattern, strlen(pattern), &pat_buff); + + if (comp_err) { + PluginDebug("Regex Compilation ERROR: %s", comp_err); return false; } - int rc = pcre_exec(re, NULL, uri, strlen(uri), 0, 0, NULL, 0); - pcre_free(re); - return rc >= 0; + int match_ret; + match_ret = re_match(&pat_buff, uri, strlen(uri), 0, 0); + regfree(&pat_buff); + + return match_ret >= 0; } diff --git a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc index b879f7ce449..f39758e097c 100644 --- a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc +++ b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc @@ -29,6 +29,7 @@ extern "C" { #include "../jwt.h" #include "../normalize.h" #include "../parse.h" +#include "../match.h" } bool @@ -446,3 +447,32 @@ TEST_CASE("4", "[NormalizeTest]") SECTION("Testing empty uri after http://?/") { REQUIRE(!normalize_uri_helper("http://?/", NULL)); } fprintf(stderr, "\n"); } + +TEST_CASE("5", "[RegexTests]") +{ + INFO("TEST 5, Test Regex Matching"); + + SECTION("Standard regex") + { + REQUIRE(match_regex("http://kelloggsTester.souza.local/KellogsDir/*", + "http://kelloggsTester.souza.local/KellogsDir/some_manifest.m3u8")); + } + + SECTION("Back references are not supported") { REQUIRE(!match_regex("(b*a)\\1$", "bbbbba")); } + + SECTION("Escape a special character") { REQUIRE(match_regex("money\\$", "money$bags")); } + + SECTION("Dollar sign") + { + REQUIRE(!match_regex(".+foobar$", "foobarfoofoo")); + REQUIRE(match_regex(".+foobar$", "foofoofoobar")); + } + + SECTION("Number Quantifier with Groups") + { + REQUIRE(match_regex("(abab){2}", "abababab")); + REQUIRE(!match_regex("(abab){2}", "abab")); + } + + SECTION("Alternation") { REQUIRE(match_regex("cat|dog", "dog")); } +} From 134f9aa0034a73f8f7a971a2b75aa762bf7c37f9 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 4 Feb 2019 13:39:01 -0700 Subject: [PATCH 232/526] Makes the master triggers works with the 9.x Jenkins Tab name --- ci/jenkins/bin/gh-mirror.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jenkins/bin/gh-mirror.sh b/ci/jenkins/bin/gh-mirror.sh index 4f9fab92d0d..8ed06fd54df 100755 --- a/ci/jenkins/bin/gh-mirror.sh +++ b/ci/jenkins/bin/gh-mirror.sh @@ -54,14 +54,14 @@ function checkBuild() { echo -n "$diff" | ${GREP} -F -e doc/ >/dev/null if [ 0 == $? ]; then echo "Triggerd Docs build for ${branch}" - ${CURL} -o /dev/null -s ${BASE_URL}/view/${branch}/job/docs-${branch}/${token} + ${CURL} -o /dev/null -s ${BASE_URL}/job/docs-${branch}/${token} fi # Check if commits have non doc/ changes echo -n "$diff" | ${GREP} -F -v -e doc/ >/dev/null if [ 0 == $? ]; then echo "Triggered main build for ${branch}" - ${CURL} -o /dev/null -s ${BASE_URL}/view/${branch}/job/start-${branch}/${token} + ${CURL} -o /dev/null -s ${BASE_URL}/job/start-${branch}/${token} fi } From 2ab930c3e653af0a13c8b962bac94131940a5677 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 4 Feb 2019 16:02:21 -0600 Subject: [PATCH 233/526] Cleanup: Convert HTTPHdr::length_get to real method. --- proxy/hdrs/HTTP.cc | 20 ++++++++++---------- proxy/hdrs/HTTP.h | 15 +-------------- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index dabd5841a66..6c4187785af 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -592,21 +592,21 @@ http_hdr_describe(HdrHeapObjImpl *raw, bool recurse) -------------------------------------------------------------------------*/ int -http_hdr_length_get(HTTPHdrImpl *hdr) +HTTPHdr::length_get() const { int length = 0; - if (hdr->m_polarity == HTTP_TYPE_REQUEST) { - if (hdr->u.req.m_ptr_method) { - length = hdr->u.req.m_len_method; + if (m_http->m_polarity == HTTP_TYPE_REQUEST) { + if (m_http->u.req.m_ptr_method) { + length = m_http->u.req.m_len_method; } else { length = 0; } length += 1; // " " - if (hdr->u.req.m_url_impl) { - length += url_length_get(hdr->u.req.m_url_impl); + if (m_http->u.req.m_url_impl) { + length += url_length_get(m_http->u.req.m_url_impl); } length += 1; // " " @@ -614,9 +614,9 @@ http_hdr_length_get(HTTPHdrImpl *hdr) length += 8; // HTTP/%d.%d length += 2; // "\r\n" - } else if (hdr->m_polarity == HTTP_TYPE_RESPONSE) { - if (hdr->u.resp.m_ptr_reason) { - length = hdr->u.resp.m_len_reason; + } else if (m_http->m_polarity == HTTP_TYPE_RESPONSE) { + if (m_http->u.resp.m_ptr_reason) { + length = m_http->u.resp.m_len_reason; } else { length = 0; } @@ -632,7 +632,7 @@ http_hdr_length_get(HTTPHdrImpl *hdr) length += 2; // "\r\n" } - length += mime_hdr_length_get(hdr->m_fields_impl); + length += mime_hdr_length_get(m_http->m_fields_impl); return length; } diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h index e9718c3e18b..a5fc4e69589 100644 --- a/proxy/hdrs/HTTP.h +++ b/proxy/hdrs/HTTP.h @@ -423,9 +423,6 @@ inkcoreapi int http_hdr_print(HdrHeap *heap, HTTPHdrImpl *hh, char *buf, int buf void http_hdr_describe(HdrHeapObjImpl *obj, bool recurse = true); -int http_hdr_length_get(HTTPHdrImpl *hh); -// HTTPType http_hdr_type_get (HTTPHdrImpl *hh); - // int32_t http_hdr_version_get (HTTPHdrImpl *hh); inkcoreapi void http_hdr_version_set(HTTPHdrImpl *hh, int32_t ver); @@ -520,7 +517,7 @@ class HTTPHdr : public MIMEHdr int print(char *buf, int bufsize, int *bufindex, int *dumpoffset); - int length_get(); + int length_get() const; HTTPType type_get() const; @@ -843,16 +840,6 @@ HTTPHdr::print(char *buf, int bufsize, int *bufindex, int *dumpoffset) return http_hdr_print(m_heap, m_http, buf, bufsize, bufindex, dumpoffset); } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -inline int -HTTPHdr::length_get() -{ - ink_assert(valid()); - return http_hdr_length_get(m_http); -} - /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ From ad2461db80947f5072327268bdacfe3ed6917984 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 4 Feb 2019 15:47:39 -0600 Subject: [PATCH 234/526] Doc: TSHttpTxnHdrLengthGet, TSMBuffer, TSMLoc. --- .../api/functions/TSHttpHdrLengthGet.en.rst | 11 ++++++++++- doc/developer-guide/api/functions/TSTypes.en.rst | 11 +++++++++++ doc/developer-guide/core-architecture/heap.en.rst | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst index 9db63fdef81..495ef1a991c 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst @@ -26,7 +26,16 @@ Synopsis `#include ` -.. function:: int TSHttpHdrLengthGet(TSMBuffer bufp, TSMLoc offset) +.. function:: int TSHttpHdrLengthGet(TSMBuffer bufp, TSMLoc mloc) Description =========== + +Return the length in characters of the HTTP header specified by :arg:`bufp` and :arg:`mloc` which +must specify a valid HTTP header. Usually these values would have been obtained via an earlier call +to +:func:`TSHttpTxnServerReqGet`, +:func:`TSHttpTxnClientReqGet`, +:func:`TSHttpTxnServerRespGet`, +:func:`TSHttpTxnClientRespGet`, +or via calls to create a new HTTP header such as :func:`TSHttpHdrCreate`. diff --git a/doc/developer-guide/api/functions/TSTypes.en.rst b/doc/developer-guide/api/functions/TSTypes.en.rst index 8abbea62cbd..675029a1d5c 100644 --- a/doc/developer-guide/api/functions/TSTypes.en.rst +++ b/doc/developer-guide/api/functions/TSTypes.en.rst @@ -102,6 +102,11 @@ more widely. Those are described on this page. .. type:: TSMBuffer + Internally, data for a transaction is stored in one more more :term:`header heap`\s. These are + storage local to the transaction, and generally each HTTP header is stored in a separate one. + This type is a handle to a header heap, and is provided or required by functions that locate HTTP + header related data. + .. type:: TSMgmtCounter .. type:: TSMgmtFloat @@ -120,6 +125,12 @@ more widely. Those are described on this page. .. type:: TSMLoc + This is a memory location relative to a :term:`header heap` represented by a :c:type:`TSMBuffer` and + must always be used in conjuction with that :c:type:`TSMBuffer` instance. It identifies a specific + object in the :c:type:`TSMBuffer`. This indirection is needed so that the :c:type:`TSMBuffer` + can reallocate space as needed. Therefore a raw address obtained from a :c:type:`TSMLoc` should + be considered volatile that may become invalid across any API call. + .. var:: TSMLoc TS_NULL_MLOC A predefined null valued :type:`TSMLoc` used to indicate the absence of an :type:`TSMLoc`. diff --git a/doc/developer-guide/core-architecture/heap.en.rst b/doc/developer-guide/core-architecture/heap.en.rst index 188b2c90bc1..f05fbbb1f20 100644 --- a/doc/developer-guide/core-architecture/heap.en.rst +++ b/doc/developer-guide/core-architecture/heap.en.rst @@ -121,7 +121,7 @@ Classes :arg:`type` must be one of the values specified in :class:`HdrHeapObjImpl`. - .. function:: int marshal_length + .. function:: int marshal_length() Compute and return the size of the buffer needed to serialize :arg:`this`. From 48f7ebf81f2161f8ebdcea4bc1d958e1d47ccc3b Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 4 Feb 2019 22:39:07 +0000 Subject: [PATCH 235/526] Pull Age out of the gold test --- tests/gold_tests/headers/cache_and_req_body-miss.gold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gold_tests/headers/cache_and_req_body-miss.gold b/tests/gold_tests/headers/cache_and_req_body-miss.gold index 46d4e085907..7c8159d190c 100644 --- a/tests/gold_tests/headers/cache_and_req_body-miss.gold +++ b/tests/gold_tests/headers/cache_and_req_body-miss.gold @@ -3,7 +3,7 @@ Last-Modified: `` Cache-Control: max-age=1 Content-Length: 3 Date: `` -Age: 0 +Age: `` Connection: keep-alive Via: `` Server: `` From e1ea2cb67c7a61f4b096dd45bee0aac5070f61bb Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 4 Feb 2019 21:33:14 +0000 Subject: [PATCH 236/526] Make mutex lock routines deal with null mutex and general clean up. --- iocore/eventsystem/I_Lock.h | 185 +++-------------------------- iocore/hostdb/HostDB.cc | 37 +++--- iocore/hostdb/P_HostDB.h | 2 +- iocore/hostdb/P_RefCountCache.h | 6 +- iocore/net/UnixNetAccept.cc | 10 +- proxy/ProxySession.cc | 27 ++--- proxy/http/HttpSM.cc | 44 ++----- proxy/http/HttpSessionManager.cc | 9 +- src/traffic_server/InkAPI.cc | 28 ++--- src/traffic_server/InkIOCoreAPI.cc | 23 ++-- 10 files changed, 98 insertions(+), 273 deletions(-) diff --git a/iocore/eventsystem/I_Lock.h b/iocore/eventsystem/I_Lock.h index 9664606e55e..00177253de0 100644 --- a/iocore/eventsystem/I_Lock.h +++ b/iocore/eventsystem/I_Lock.h @@ -242,7 +242,7 @@ Mutex_trylock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, #endif - ProxyMutex *m, EThread *t) + Ptr &m, EThread *t) { ink_assert(t != nullptr); ink_assert(t == (EThread *)this_thread()); @@ -281,90 +281,12 @@ Mutex_trylock( return true; } -inline bool -Mutex_trylock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif - Ptr &m, EThread *t) -{ - return Mutex_trylock( -#ifdef DEBUG - location, ahandler, -#endif - m.get(), t); -} - -inline bool -Mutex_trylock_spin( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif - ProxyMutex *m, EThread *t, int spincnt = 1) -{ - ink_assert(t != nullptr); - if (m->thread_holding != t) { - int locked; - do { - if ((locked = ink_mutex_try_acquire(&m->the_mutex))) { - break; - } - } while (--spincnt); - if (!locked) { -#ifdef DEBUG - lock_waiting(m->srcloc, m->handler); -#ifdef LOCK_CONTENTION_PROFILING - m->unsuccessful_nonblocking_acquires++; - m->nonblocking_acquires++; - m->total_acquires++; - m->print_lock_stats(0); -#endif // LOCK_CONTENTION_PROFILING -#endif // DEBUG - return false; - } - m->thread_holding = t; - ink_assert(m->thread_holding); -#ifdef DEBUG - m->srcloc = location; - m->handler = ahandler; - m->hold_time = Thread::get_hrtime(); -#ifdef MAX_LOCK_TAKEN - m->taken++; -#endif // MAX_LOCK_TAKEN -#endif // DEBUG - } -#ifdef DEBUG -#ifdef LOCK_CONTENTION_PROFILING - m->successful_nonblocking_acquires++; - m->nonblocking_acquires++; - m->total_acquires++; - m->print_lock_stats(0); -#endif // LOCK_CONTENTION_PROFILING -#endif // DEBUG - m->nthread_holding++; - return true; -} - -inline bool -Mutex_trylock_spin( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif - Ptr &m, EThread *t, int spincnt = 1) -{ - return Mutex_trylock_spin( -#ifdef DEBUG - location, ahandler, -#endif - m.get(), t, spincnt); -} - inline int Mutex_lock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, #endif - ProxyMutex *m, EThread *t) + Ptr &m, EThread *t) { ink_assert(t != nullptr); if (m->thread_holding != t) { @@ -391,22 +313,8 @@ Mutex_lock( return true; } -inline int -Mutex_lock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif - Ptr &m, EThread *t) -{ - return Mutex_lock( -#ifdef DEBUG - location, ahandler, -#endif - m.get(), t); -} - inline void -Mutex_unlock(ProxyMutex *m, EThread *t) +Mutex_unlock(Ptr &m, EThread *t) { if (m->nthread_holding) { ink_assert(t == m->thread_holding); @@ -429,12 +337,6 @@ Mutex_unlock(ProxyMutex *m, EThread *t) } } -inline void -Mutex_unlock(Ptr &m, EThread *t) -{ - Mutex_unlock(m.get(), t); -} - /** Scoped lock class for ProxyMutex */ class MutexLock @@ -444,20 +346,6 @@ class MutexLock bool locked_p; public: - MutexLock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif // DEBUG - ProxyMutex *am, EThread *t) - : m(am), locked_p(true) - { - Mutex_lock( -#ifdef DEBUG - location, ahandler, -#endif // DEBUG - m.get(), t); - } - MutexLock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, @@ -469,7 +357,7 @@ class MutexLock #ifdef DEBUG location, ahandler, #endif // DEBUG - m.get(), t); + m, t); } void @@ -493,20 +381,6 @@ class MutexTryLock bool lock_acquired; public: - MutexTryLock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif // DEBUG - ProxyMutex *am, EThread *t) - : m(am) - { - lock_acquired = Mutex_trylock( -#ifdef DEBUG - location, ahandler, -#endif // DEBUG - m.get(), t); - } - MutexTryLock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, @@ -514,45 +388,21 @@ class MutexTryLock Ptr &am, EThread *t) : m(am) { - lock_acquired = Mutex_trylock( -#ifdef DEBUG - location, ahandler, -#endif // DEBUG - m.get(), t); - } - - MutexTryLock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif // DEBUG - ProxyMutex *am, EThread *t, int sp) - : m(am) - { - lock_acquired = Mutex_trylock_spin( + if (am) { + lock_acquired = Mutex_trylock( #ifdef DEBUG - location, ahandler, -#endif // DEBUG - m.get(), t, sp); - } - - MutexTryLock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif // DEBUG - Ptr &am, EThread *t, int sp) - : m(am) - { - lock_acquired = Mutex_trylock_spin( -#ifdef DEBUG - location, ahandler, + location, ahandler, #endif // DEBUG - m.get(), t, sp); + m, t); + } else { + lock_acquired = true; + } } ~MutexTryLock() { - if (lock_acquired) { - Mutex_unlock(m.get(), m->thread_holding); + if (lock_acquired && m.get()) { + Mutex_unlock(m, m->thread_holding); } } @@ -561,16 +411,17 @@ class MutexTryLock void acquire(EThread *t) { - MUTEX_TAKE_LOCK(m.get(), t); lock_acquired = true; + if (m.get()) { + MUTEX_TAKE_LOCK(m, t); + } } void release() { - ink_assert(lock_acquired); // generate a warning because it shouldn't be done. - if (lock_acquired) { - Mutex_unlock(m.get(), m->thread_holding); + if (lock_acquired && m.get()) { + Mutex_unlock(m, m->thread_holding); } lock_acquired = false; } diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 947b2c234b2..dd80c58ecc6 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -449,13 +449,13 @@ HostDBContinuation::init(HostDBHash const &the_hash, Options const &opt) void HostDBContinuation::refresh_hash() { - ProxyMutex *old_bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + Ptr old_bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); // We're not pending DNS anymore. remove_trigger_pending_dns(); hash.refresh(); // Update the mutex if it's from the bucket. // Some call sites modify this after calling @c init so need to check. - if (mutex.get() == old_bucket_mutex) { + if (mutex == old_bucket_mutex) { mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); } } @@ -533,7 +533,7 @@ db_mark_for(IpAddr const &ip) } Ptr -probe(ProxyMutex *mutex, HostDBHash const &hash, bool ignore_timeout) +probe(Ptr mutex, HostDBHash const &hash, bool ignore_timeout) { // If hostdb is disabled, don't return anything if (!hostdb_enable) { @@ -609,8 +609,8 @@ HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockad HostResStyle host_res_style, int dns_lookup_timeout) { HostDBHash hash; - EThread *thread = this_ethread(); - ProxyMutex *mutex = thread->mutex.get(); + EThread *thread = this_ethread(); + Ptr mutex = thread->mutex; ip_text_buffer ipb; HOSTDB_INCREMENT_DYN_STAT(hostdb_total_lookups_stat); @@ -640,7 +640,7 @@ HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockad // find the partition lock // // TODO: Could we reuse the "mutex" above safely? I think so but not sure. - ProxyMutex *bmutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + Ptr bmutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); MUTEX_TRY_LOCK(lock, bmutex, thread); MUTEX_TRY_LOCK(lock2, cont->mutex, thread); @@ -763,7 +763,7 @@ HostDBProcessor::getSRVbyname_imm(Continuation *cont, process_srv_info_pfn proce // Attempt to find the result in-line, for level 1 hits if (!force_dns) { // find the partition lock - ProxyMutex *bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + Ptr bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); MUTEX_TRY_LOCK(lock, bucket_mutex, thread); // If we can get the lock and a level 1 probe succeeds, return @@ -836,7 +836,7 @@ HostDBProcessor::getbyname_imm(Continuation *cont, process_hostdb_info_pfn proce do { loop = false; // loop only on explicit set for retry // find the partition lock - ProxyMutex *bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + Ptr bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); SCOPED_MUTEX_LOCK(lock, bucket_mutex, thread); // do a level 1 probe for immediate result. Ptr r = probe(bucket_mutex, hash, false); @@ -952,8 +952,8 @@ HostDBProcessor::setby(const char *hostname, int len, sockaddr const *ip, HostDB // Attempt to find the result in-line, for level 1 hits - ProxyMutex *mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); - EThread *thread = this_ethread(); + Ptr mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + EThread *thread = this_ethread(); MUTEX_TRY_LOCK(lock, mutex, thread); if (lock.is_locked()) { @@ -999,7 +999,7 @@ HostDBProcessor::setby_srv(const char *hostname, int len, const char *target, Ho int HostDBContinuation::setbyEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) { - Ptr r = probe(mutex.get(), hash, false); + Ptr r = probe(mutex, hash, false); if (r) { do_setby(r.get(), &app, hash.host_name, hash.ip, is_srv()); @@ -1056,8 +1056,11 @@ int HostDBContinuation::removeEvent(int /* event ATS_UNUSED */, Event *e) { Continuation *cont = action.continuation; - - MUTEX_TRY_LOCK(lock, cont ? cont->mutex.get() : (ProxyMutex *)nullptr, e->ethread); + Ptr proxy_mutex; + if (cont) { + proxy_mutex = cont->mutex; + } + MUTEX_TRY_LOCK(lock, proxy_mutex, e->ethread); if (!lock.is_locked()) { e->schedule_in(HOST_DB_RETRY_PERIOD); return EVENT_CONT; @@ -1068,7 +1071,7 @@ HostDBContinuation::removeEvent(int /* event ATS_UNUSED */, Event *e) cont->handleEvent(EVENT_HOST_DB_IP_REMOVED, (void *)nullptr); } } else { - Ptr r = probe(mutex.get(), hash, false); + Ptr r = probe(mutex, hash, false); bool res = remove_round_robin(r.get(), hash.host_name, hash.ip); if (cont) { cont->handleEvent(EVENT_HOST_DB_IP_REMOVED, res ? static_cast(&hash.ip) : static_cast(nullptr)); @@ -1277,7 +1280,7 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) ttl = failed ? 0 : e->ttl / 60; int ttl_seconds = failed ? 0 : e->ttl; // ebalsa: moving to second accuracy - Ptr old_r = probe(mutex.get(), hash, false); + Ptr old_r = probe(mutex, hash, false); // If the DNS lookup failed with NXDOMAIN, remove the old record if (e && e->isNameError() && old_r) { hostDB.refcountcache->erase(old_r->key); @@ -1535,7 +1538,7 @@ HostDBContinuation::iterateEvent(int event, Event *e) // let's iterate through another record and then reschedule ourself. if (current_iterate_pos < hostDB.refcountcache->partition_count()) { // TODO: configurable number at a time? - ProxyMutex *bucket_mutex = hostDB.refcountcache->get_partition(current_iterate_pos).lock.get(); + Ptr bucket_mutex = hostDB.refcountcache->get_partition(current_iterate_pos).lock; MUTEX_TRY_LOCK(lock_bucket, bucket_mutex, t); if (!lock_bucket.is_locked()) { // we couldn't get the bucket lock, let's just reschedule and try later. @@ -1612,7 +1615,7 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) if (!force_dns) { // Do the probe // - Ptr r = probe(mutex.get(), hash, false); + Ptr r = probe(mutex, hash, false); if (r) { HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat); diff --git a/iocore/hostdb/P_HostDB.h b/iocore/hostdb/P_HostDB.h index c29d3c613e7..43c5a5ba4bc 100644 --- a/iocore/hostdb/P_HostDB.h +++ b/iocore/hostdb/P_HostDB.h @@ -47,6 +47,6 @@ static constexpr ts::ModuleVersion HOSTDB_MODULE_INTERNAL_VERSION{HOSTDB_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE}; -Ptr probe(ProxyMutex *mutex, CryptoHash const &hash, bool ignore_timeout); +Ptr probe(Ptr mutex, CryptoHash const &hash, bool ignore_timeout); void make_crypto_hash(CryptoHash &hash, const char *hostname, int len, int port, const char *pDNSServers, HostDBMark mark); diff --git a/iocore/hostdb/P_RefCountCache.h b/iocore/hostdb/P_RefCountCache.h index 8655d345607..9ca3764e807 100644 --- a/iocore/hostdb/P_RefCountCache.h +++ b/iocore/hostdb/P_RefCountCache.h @@ -408,7 +408,7 @@ template class RefCountCache // Some methods to get some internal state int partition_for_key(uint64_t key); - ProxyMutex *lock_for_key(uint64_t key); + Ptr lock_for_key(uint64_t key); size_t partition_count() const; RefCountCachePartition &get_partition(int pnum); size_t count() const; @@ -510,10 +510,10 @@ RefCountCache::get_header() } template -ProxyMutex * +Ptr RefCountCache::lock_for_key(uint64_t key) { - return this->partitions[this->partition_for_key(key)]->lock.get(); + return this->partitions[this->partition_for_key(key)]->lock; } template diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index b730c86ca77..0164d64b4b3 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -78,7 +78,7 @@ net_accept(NetAccept *na, void *ep, bool blockable) Connection con; if (!blockable) { - if (!MUTEX_TAKE_TRY_LOCK(na->action_->mutex.get(), e->ethread)) { + if (!MUTEX_TAKE_TRY_LOCK(na->action_->mutex, e->ethread)) { return 0; } } @@ -149,7 +149,7 @@ net_accept(NetAccept *na, void *ep, bool blockable) Ldone: if (!blockable) { - MUTEX_UNTAKE_LOCK(na->action_->mutex.get(), e->ethread); + MUTEX_UNTAKE_LOCK(na->action_->mutex, e->ethread); } return count; } @@ -379,12 +379,12 @@ NetAccept::acceptEvent(int event, void *ep) (void)event; Event *e = (Event *)ep; // PollDescriptor *pd = get_PollDescriptor(e->ethread); - ProxyMutex *m = nullptr; + Ptr m; if (action_->mutex) { - m = action_->mutex.get(); + m = action_->mutex; } else { - m = mutex.get(); + m = mutex; } MUTEX_TRY_LOCK(lock, m, e->ethread); diff --git a/proxy/ProxySession.cc b/proxy/ProxySession.cc index be78f6bc4ab..6afb04f65ea 100644 --- a/proxy/ProxySession.cc +++ b/proxy/ProxySession.cc @@ -119,29 +119,20 @@ ProxySession::state_api_callout(int event, void *data) } if (this->api_current) { - bool plugin_lock = false; - APIHook *hook = this->api_current; - Ptr plugin_mutex; - - if (hook->m_cont->mutex) { - plugin_mutex = hook->m_cont->mutex; - plugin_lock = MUTEX_TAKE_TRY_LOCK(hook->m_cont->mutex, mutex->thread_holding); - if (!plugin_lock) { - SET_HANDLER(&ProxySession::state_api_callout); - if (!schedule_event) { // Don't bother to schedule is there is already one out. - schedule_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); - } - return 0; + APIHook *hook = this->api_current; + + MUTEX_TRY_LOCK(lock, hook->m_cont->mutex, mutex->thread_holding); + // Have a mutex but did't get the lock, reschedule + if (!lock.is_locked()) { + SET_HANDLER(&ProxySession::state_api_callout); + if (!schedule_event) { // Don't bother to schedule is there is already one out. + schedule_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); } + return 0; } this->api_current = this->api_current->next(); hook->invoke(eventmap[this->api_hookid], this); - - if (plugin_lock) { - Mutex_unlock(plugin_mutex, this_ethread()); - } - return 0; } } diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 56e2b817a7b..b2c9cb9e405 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1428,34 +1428,19 @@ plugins required to work with sni_routing. callout_state = HTTP_API_IN_CALLOUT; } - /* The MUTEX_TRY_LOCK macro was changed so - that it can't handle NULL mutex'es. The plugins - can use null mutexes so we have to do this manually. - We need to take a smart pointer to the mutex since - the plugin could release it's mutex while we're on - the callout - */ - bool plugin_lock; - Ptr plugin_mutex; - if (cur_hook->m_cont->mutex) { - plugin_mutex = cur_hook->m_cont->mutex; - plugin_lock = MUTEX_TAKE_TRY_LOCK(cur_hook->m_cont->mutex, mutex->thread_holding); - - if (!plugin_lock) { - api_timer = -Thread::get_hrtime_updated(); - HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_api_callout); - ink_assert(pending_action == nullptr); - pending_action = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); - // Should @a callout_state be reset back to HTTP_API_NO_CALLOUT here? Because the default - // handler has been changed the value isn't important to the rest of the state machine - // but not resetting means there is no way to reliably detect re-entrance to this state with an - // outstanding callout. - return 0; - } - } else { - plugin_lock = false; + MUTEX_TRY_LOCK(lock, cur_hook->m_cont->mutex, mutex->thread_holding); + // Have a mutex but didn't get the lock, reschedule + if (!lock.is_locked()) { + api_timer = -Thread::get_hrtime_updated(); + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_api_callout); + ink_assert(pending_action == nullptr); + pending_action = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); + // Should @a callout_state be reset back to HTTP_API_NO_CALLOUT here? Because the default + // handler has been changed the value isn't important to the rest of the state machine + // but not resetting means there is no way to reliably detect re-entrance to this state with an + // outstanding callout. + return 0; } - SMDebug("http", "[%" PRId64 "] calling plugin on hook %s at hook %p", sm_id, HttpDebugNames::get_api_hook_name(cur_hook_id), cur_hook); @@ -1473,11 +1458,6 @@ plugins required to work with sni_routing. // tracking a non-complete callout from a chain so just let it ride. It will get cleaned // up in state_api_callback when the plugin re-enables this transaction. } - - if (plugin_lock) { - Mutex_unlock(plugin_mutex, mutex->thread_holding); - } - return 0; } } diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index 399f0354936..33033b2d48d 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -320,10 +320,11 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad // client session { // Now check to see if we have a connection in our shared connection pool - EThread *ethread = this_ethread(); - ProxyMutex *pool_mutex = (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) ? - ethread->server_session_pool->mutex.get() : - m_g_pool->mutex.get(); + EThread *ethread = this_ethread(); + Ptr pool_mutex = + (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) ? + ethread->server_session_pool->mutex : + m_g_pool->mutex; MUTEX_TRY_LOCK(lock, pool_mutex, ethread); if (lock.is_locked()) { if (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) { diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index c0682c239c5..cbe88ce9397 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -1295,16 +1295,12 @@ APIHook::invoke(int event, void *edata) ink_assert(!"not reached"); } } - if (m_cont->mutex != nullptr) { - MUTEX_TRY_LOCK(lock, m_cont->mutex, this_ethread()); - if (!lock.is_locked()) { - // If we cannot get the lock, the caller needs to restructure to handle rescheduling - ink_release_assert(0); - } - return m_cont->handleEvent(event, edata); - } else { - return m_cont->handleEvent(event, edata); + MUTEX_TRY_LOCK(lock, m_cont->mutex, this_ethread()); + if (!lock.is_locked()) { + // If we cannot get the lock, the caller needs to restructure to handle rescheduling + ink_release_assert(0); } + return m_cont->handleEvent(event, edata); } APIHook * @@ -4682,16 +4678,12 @@ int TSContCall(TSCont contp, TSEvent event, void *edata) { Continuation *c = (Continuation *)contp; - if (c->mutex != nullptr) { - MUTEX_TRY_LOCK(lock, c->mutex, this_ethread()); - if (!lock.is_locked()) { - // If we cannot get the lock, the caller needs to restructure to handle rescheduling - ink_release_assert(0); - } - return c->handleEvent((int)event, edata); - } else { - return c->handleEvent((int)event, edata); + MUTEX_TRY_LOCK(lock, c->mutex, this_ethread()); + if (!lock.is_locked()) { + // If we cannot get the lock, the caller needs to restructure to handle rescheduling + ink_release_assert(0); } + return c->handleEvent((int)event, edata); } TSMutex diff --git a/src/traffic_server/InkIOCoreAPI.cc b/src/traffic_server/InkIOCoreAPI.cc index 221aa368ee5..d8c3fda4452 100644 --- a/src/traffic_server/InkIOCoreAPI.cc +++ b/src/traffic_server/InkIOCoreAPI.cc @@ -54,7 +54,7 @@ sdk_sanity_check_mutex(TSMutex mutex) return TS_ERROR; } - ProxyMutex *mutexp = (ProxyMutex *)mutex; + ProxyMutex *mutexp = reinterpret_cast(mutex); if (mutexp->refcount() < 0) { return TS_ERROR; @@ -242,6 +242,7 @@ TSMutex TSMutexCreate() { ProxyMutex *mutexp = new_ProxyMutex(); + mutexp->refcount_inc(); // TODO: Remove this when allocations can never fail. sdk_assert(sdk_sanity_check_mutex((TSMutex)mutexp) == TS_SUCCESS); @@ -253,9 +254,12 @@ void TSMutexDestroy(TSMutex m) { sdk_assert(sdk_sanity_check_mutex(m) == TS_SUCCESS); - ink_release_assert(((ProxyMutex *)m)->refcount() == 0); - - ((ProxyMutex *)m)->free(); + ProxyMutex *mutexp = reinterpret_cast(m); + // Decrement the refcount added in TSMutexCreate. Delete if this + // was the last ref count + if (mutexp && mutexp->refcount_dec() == 0) { + mutexp->free(); + } } /* The following two APIs are for Into work, actually, APIs of Mutex @@ -292,21 +296,24 @@ void TSMutexLock(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - MUTEX_TAKE_LOCK((ProxyMutex *)mutexp, this_ethread()); + Ptr proxy_mutex(reinterpret_cast(mutexp)); + MUTEX_TAKE_LOCK(proxy_mutex, this_ethread()); } TSReturnCode TSMutexLockTry(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - return (MUTEX_TAKE_TRY_LOCK((ProxyMutex *)mutexp, this_ethread()) ? TS_SUCCESS : TS_ERROR); + Ptr proxy_mutex(reinterpret_cast(mutexp)); + return (MUTEX_TAKE_TRY_LOCK(proxy_mutex, this_ethread()) ? TS_SUCCESS : TS_ERROR); } void TSMutexUnlock(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - MUTEX_UNTAKE_LOCK((ProxyMutex *)mutexp, this_ethread()); + Ptr proxy_mutex(reinterpret_cast(mutexp)); + MUTEX_UNTAKE_LOCK(proxy_mutex, this_ethread()); } /* VIOs */ @@ -409,7 +416,7 @@ TSVIOMutexGet(TSVIO viop) sdk_assert(sdk_sanity_check_iocore_structure(viop) == TS_SUCCESS); VIO *vio = (VIO *)viop; - return (TSMutex)(vio->mutex.get()); + return reinterpret_cast(vio->mutex.get()); } /* High Resolution Time */ From 21c82cf370a5c4c53ebdde23f161af3485b95aa8 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 4 Feb 2019 12:47:10 -0800 Subject: [PATCH 237/526] lua plugin: add support for relative path scripts; moves inline script to switch To use inline scripts, prefix with --inline; eg: @plugin=tslua.so @pparam=--states=4 @pparam=--inline=print(123) --- plugins/lua/ts_lua.c | 39 +++++++++++++++++++++++++++------------ plugins/lua/ts_lua_util.c | 2 +- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/plugins/lua/ts_lua.c b/plugins/lua/ts_lua.c index 0d8af2c3fa5..5aef7b551ef 100644 --- a/plugins/lua/ts_lua.c +++ b/plugins/lua/ts_lua.c @@ -63,11 +63,14 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) TSReturnCode TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) { - int fn; int ret; + char script[TS_LUA_MAX_SCRIPT_FNAME_LENGTH]; + char *inline_script = ""; + int fn = 0; int states = TS_LUA_MAX_STATE_COUNT; static const struct option longopt[] = { {"states", required_argument, 0, 's'}, + {"inline", required_argument, 0, 'i'}, {0, 0, 0, 0}, }; @@ -84,6 +87,8 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s TSDebug(TS_LUA_DEBUG_TAG, "[%s] setting number of lua VM [%d]", __FUNCTION__, states); // set state break; + case 'i': + inline_script = optarg; } if (opt == -1) { @@ -97,17 +102,24 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s return TS_ERROR; } - if (argc - optind < 1) { + if (argc - optind > 0) { + fn = 1; + if (argv[optind][0] == '/') { + snprintf(script, sizeof(script), "%s", argv[optind]); + } else { + snprintf(script, sizeof(script), "%s/%s", TSConfigDirGet(), argv[optind]); + } + } + + if (strlen(inline_script) == 0 && argc - optind < 1) { strncpy(errbuf, "[TSRemapNewInstance] - lua script file or string is required !!", errbuf_size - 1); errbuf[errbuf_size - 1] = '\0'; return TS_ERROR; } - fn = 1; - - if (argv[optind][0] != '/') { - fn = 0; - } else if (strlen(argv[optind]) >= TS_LUA_MAX_SCRIPT_FNAME_LENGTH - 16) { + if (strlen(script) >= TS_LUA_MAX_SCRIPT_FNAME_LENGTH - 16) { + strncpy(errbuf, "[TSRemapNewInstance] - lua script file name too long !!", errbuf_size - 1); + errbuf[errbuf_size - 1] = '\0'; return TS_ERROR; } @@ -116,8 +128,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s // check to make sure it is a lua file and there is no parameter for the lua file if (fn && (argc - optind < 2)) { TSDebug(TS_LUA_DEBUG_TAG, "[%s] checking if script has been registered", __FUNCTION__); - char script[TS_LUA_MAX_SCRIPT_FNAME_LENGTH]; - snprintf(script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", argv[optind]); + // we only need to check the first lua VM for script registration conf = ts_lua_script_registered(ts_lua_main_ctx_array[0].lua, script); } @@ -138,9 +149,9 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s conf->init_func = 0; if (fn) { - snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", argv[optind]); + snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", script); } else { - conf->content = argv[optind]; + conf->content = inline_script; } ts_lua_init_instance(conf); @@ -508,7 +519,11 @@ TSPluginInit(int argc, const char *argv[]) conf->remap = 0; conf->states = states; - snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", argv[optind]); + if (argv[optind][0] == '/') { + snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", argv[optind]); + } else { + snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s/%s", TSConfigDirGet(), argv[optind]); + } ts_lua_init_instance(conf); diff --git a/plugins/lua/ts_lua_util.c b/plugins/lua/ts_lua_util.c index b72b076f004..34d2b1ea54e 100644 --- a/plugins/lua/ts_lua_util.c +++ b/plugins/lua/ts_lua_util.c @@ -201,7 +201,7 @@ ts_lua_add_module(ts_lua_instance_conf *conf, ts_lua_main_ctx *arr, int n, int a if (conf->content) { if (luaL_loadstring(L, conf->content)) { - snprintf(errbuf, errbuf_size, "[%s] luaL_loadstring %s failed: %s", __FUNCTION__, conf->script, lua_tostring(L, -1)); + snprintf(errbuf, errbuf_size, "[%s] luaL_loadstring failed: %s", __FUNCTION__, lua_tostring(L, -1)); lua_pop(L, 1); TSMutexUnlock(arr[i].mutexp); return -1; From ce706ce7be76bc08a49b86d58c08f01ba55684bf Mon Sep 17 00:00:00 2001 From: dyrock Date: Tue, 5 Feb 2019 20:47:59 +0000 Subject: [PATCH 238/526] Plugin Traffic Dump: new feature for setting limit on disk usage. (Changed to use ts::file and std::string_view) --- doc/admin-guide/plugins/traffic_dump.en.rst | 10 +- plugins/experimental/traffic_dump/README | 7 + .../experimental/traffic_dump/traffic_dump.cc | 165 ++++++++++-------- 3 files changed, 106 insertions(+), 76 deletions(-) diff --git a/doc/admin-guide/plugins/traffic_dump.en.rst b/doc/admin-guide/plugins/traffic_dump.en.rst index be57f4b750b..2d012f11855 100644 --- a/doc/admin-guide/plugins/traffic_dump.en.rst +++ b/doc/admin-guide/plugins/traffic_dump.en.rst @@ -36,14 +36,20 @@ Plugin Configuration * ``Traffic Dump`` is a global plugin and is configured via :file:`plugin.config`. .. option:: --logdir - (`required`, default:empty/unused) - specifies the directory for writing all dump files. If path is relative, it is relative to the Traffic Server directory. The plugin will use first three characters of client ip to create subdirs in an attempt to spread dumps evenly and avoid too many files in a single directory. + (`required`) - specifies the directory for writing all dump files. If path is relative, it is relative to the Traffic Server directory. The plugin will use first three characters of client ip to create subdirs in an attempt to spread dumps evenly and avoid too many files in a single directory. .. option:: --sample - (`optional`, default:1000) - specifies the sampling ratio N. Traffic Dump will capture every one out of N sessions. This ratio can also be changed via traffic_ctl without restarting ATS. + (`required`) - specifies the sampling ratio N. Traffic Dump will capture every one out of N sessions. This ratio can also be changed via traffic_ctl without restarting ATS. + + .. option:: --limit + + (`required`) - specifies the max disk usage N bytes(approximate). Traffic Dump will stop capturing new sessions once disk usage exceeds this limit. * ``traffic_ctl`` command. ``traffic_ctl plugin msg traffic_dump.sample N`` - changes the sampling ratio N as mentioned above. + ``traffic_ctl plugin msg traffic_dump.reset`` - resets the disk usage counter. + ``traffic_ctl plugin msg traffic_dump.limit N`` - changes the max disk usage. Replay Format ============= diff --git a/plugins/experimental/traffic_dump/README b/plugins/experimental/traffic_dump/README index d4e13444139..4cf2e0b1e9d 100644 --- a/plugins/experimental/traffic_dump/README +++ b/plugins/experimental/traffic_dump/README @@ -11,6 +11,13 @@ Traffic Dump is a global plugin and is configured by arguments in plugin.config. --sample The sampling ratio. By setting this number to N, Traffic Dump will capture every one out of N sessions. This ratio can also be changed via traffic_ctl without restarting ATS. +--limit + The max disk usage (approximate). By setting this number to N, Traffic Dump will stop capturing new sessions once the disk usage exceeds N bytes. + Traffic_Ctl Command: traffic_ctl plugin msg traffic_dump.sample N Same as setting --sample=N in plugin.config. +traffic_ctl plugin msg traffic_dump.reset + Reset disk usage. +traffic_ctl plugin msg traffic_dump.limit N + Set max disk usage. diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc b/plugins/experimental/traffic_dump/traffic_dump.cc index fff6e0323bb..d8733ca5646 100644 --- a/plugins/experimental/traffic_dump/traffic_dump.cc +++ b/plugins/experimental/traffic_dump/traffic_dump.cc @@ -26,7 +26,6 @@ #include #include -#include #include #include #include @@ -38,31 +37,37 @@ #include #include #include +#include +#include "tscore/ts_file.h" #include "ts/ts.h" -const char *PLUGIN_NAME = "traffic_dump"; -static const std::string closing = "]}]}"; - -static std::string LOG_DIR = "dump"; // default log directory -static int s_arg_idx = 0; // Session Arg Index to pass on session data -static std::atomic sample_pool_size(1000); // Sampling ratio - +namespace +{ +const char *PLUGIN_NAME = "traffic_dump"; +const std::string closing = "]}]}"; + +ts::file::path log_path{"dump"}; // default log directory +int s_arg_idx = 0; // Session Arg Index to pass on session data +std::atomic sample_pool_size(1000); // Sampling ratio +std::atomic max_disk_usage(10000000); //< Max disk space for logs (approximate) +std::atomic disk_usage(0); //< Actual disk usage // handler declaration -static int session_aio_handler(TSCont contp, TSEvent event, void *edata); -static int session_txn_handler(TSCont contp, TSEvent event, void *edata); +int session_aio_handler(TSCont contp, TSEvent event, void *edata); +int session_txn_handler(TSCont contp, TSEvent event, void *edata); -// Custom structure for per session data +/// Custom structure for per session data struct SsnData { - int log_fd = -1; // Log file descriptor - int aio_count = 0; // Active AIO counts - int64_t write_offset = 0; // AIO write offset - bool first = true; // First Transaction - bool ssn_closed = false; // Session closed flag + int log_fd = -1; //< Log file descriptor + int aio_count = 0; //< Active AIO counts + int64_t write_offset = 0; //< AIO write offset + bool first = true; //< First Transaction + bool ssn_closed = false; //< Session closed flag + ts::file::path log_name; //< Log file path - TSCont aio_cont = nullptr; // AIO callback - TSCont txn_cont = nullptr; // Transaction callback - TSMutex disk_io_mutex = nullptr; // AIO mutex + TSCont aio_cont = nullptr; //< AIO callback + TSCont txn_cont = nullptr; //< Transaction callback + TSMutex disk_io_mutex = nullptr; //< AIO mutex SsnData() { @@ -110,7 +115,7 @@ struct SsnData { /// Local helper functions about json formatting /// min_write(): Inline function for repeating code -static inline void +inline void min_write(const char *buf, int64_t &prevIdx, int64_t &idx, std::ostream &jsonfile) { if (prevIdx < idx) { @@ -121,7 +126,7 @@ min_write(const char *buf, int64_t &prevIdx, int64_t &idx, std::ostream &jsonfil /// esc_json_out(): Escape characters in a buffer and output to ofstream object /// in a way to minimize ofstream operations -static int +int esc_json_out(const char *buf, int64_t len, std::ostream &jsonfile) { if (buf == nullptr) @@ -176,14 +181,14 @@ esc_json_out(const char *buf, int64_t len, std::ostream &jsonfile) } /// escape_json(): escape chars in a string and returns json string -static std::string +std::string escape_json(std::string const &s) { std::ostringstream o; esc_json_out(s.c_str(), s.length(), o); return o.str(); } -static std::string +std::string escape_json(const char *buf, int64_t size) { std::ostringstream o; @@ -191,34 +196,21 @@ escape_json(const char *buf, int64_t size) return o.str(); } -/// json_entry(): Formats to map-style entry i.e. "field": "value" -// static inline std::string -// json_entry(std::string const &name, std::string const &value) -// { -// return "\"" + escape_json(name) + "\": \"" + escape_json(value) + "\""; -// } - -static inline std::string +inline std::string json_entry(std::string const &name, const char *buf, int64_t size) { return "\"" + escape_json(name) + "\":\"" + escape_json(buf, size) + "\""; } /// json_entry_array(): Formats to array-style entry i.e. ["field","value"] -// static inline std::string -// json_entry_array(std::string const &name, std::string const &value) -// { -// return "[\"" + escape_json(name) + "\", \"" + escape_json(value) + "\"]"; -// } - -static inline std::string +inline std::string json_entry_array(const char *name, int name_len, const char *value, int value_len) { return "[\"" + escape_json(name, name_len) + "\", \"" + escape_json(value, value_len) + "\"]"; } /// Helper functions to collect txn information from TSMBuffer -static std::string +std::string collect_headers(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t body_bytes) { std::string result = "{"; @@ -308,6 +300,12 @@ session_aio_handler(TSCont contp, TSEvent event, void *edata) TSContDataSet(contp, nullptr); close(ssnData->log_fd); TSMutexUnlock(ssnData->disk_io_mutex); + std::error_code ec; + ts::file::file_status st = ts::file::status(ssnData->log_name, ec); + if (!ec) { + disk_usage += ts::file::file_size(st); + TSDebug(PLUGIN_NAME, "Finish a session with log file of %" PRId64 "bytes", ts::file::file_size(st)); + } delete ssnData; return TS_SUCCESS; } @@ -412,9 +410,25 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata) // Also handles LIFECYCLE_MSG from traffic_ctl case TS_EVENT_LIFECYCLE_MSG: { TSPluginMsg *msg = static_cast(edata); - if (strlen(msg->tag) == 19 && strncmp(msg->tag, "traffic_dump.sample", 19) == 0) { - sample_pool_size = static_cast(strtol(static_cast(msg->data), nullptr, 0)); - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Received Msg to change sample size to %" PRId64 "", sample_pool_size.load()); + // String view of plugin message prefix + static constexpr std::string_view PLUGIN_PREFIX("traffic_dump."_sv); + + std::string_view tag(msg->tag, strlen(msg->tag)); + + if (tag.substr(0, PLUGIN_PREFIX.size()) == PLUGIN_PREFIX) { + tag.remove_prefix(PLUGIN_PREFIX.size()); + if (tag == "sample") { + sample_pool_size = static_cast(strtol(static_cast(msg->data), nullptr, 0)); + TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to change sample size to %" PRId64 "bytes", + sample_pool_size.load()); + } else if (tag == "reset") { + disk_usage = 0; + TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to reset disk usage counter"); + } else if (tag == "limit") { + max_disk_usage = static_cast(strtol(static_cast(msg->data), nullptr, 0)); + TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to change max disk usage to %" PRId64 "bytes", + max_disk_usage.load()); + } } return TS_SUCCESS; } @@ -424,6 +438,10 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata) if (id % sample_pool_size != 0) { TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore session %" PRId64 "...", id); break; + } else if (disk_usage >= max_disk_usage) { + TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore session %" PRId64 "due to disk usage %" PRId64 "bytes", id, + disk_usage.load()); + break; } // Beginning of a new session /// Get epoch time @@ -470,28 +488,26 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata) // Initialize AIO file TSMutexLock(ssnData->disk_io_mutex); if (ssnData->log_fd < 0) { - std::string path = LOG_DIR + "/" + std::string(client_str, 3); - std::string fname = path + "/" + session_id; + ts::file::path log_p = log_path / ts::file::path(std::string(client_str, 3)); + ts::file::path log_f = log_p / ts::file::path(session_id); // Create subdir if not existing - struct stat st; - if (stat(path.c_str(), &st) == -1) { - if (mkdir(path.c_str(), 0755) == -1) { - TSDebug(PLUGIN_NAME, "global_ssn_handler(): failed to create dir (%d)%s", errno, strerror(errno)); - - TSError("[%s] Failed to create dir. %s", PLUGIN_NAME, strerror(errno)); - } + std::error_code ec; + ts::file::status(log_p, ec); + if (ec && mkdir(log_p.c_str(), 0755) == -1) { + TSDebug(PLUGIN_NAME, "global_ssn_handler(): Failed to create dir %s", log_p.c_str()); + TSError("[%s] Failed to create dir %s", PLUGIN_NAME, log_p.c_str()); } // Try to open log files for AIO - ssnData->log_fd = open(fname.c_str(), O_RDWR | O_CREAT, S_IRWXU); + ssnData->log_fd = open(log_f.c_str(), O_RDWR | O_CREAT, S_IRWXU); if (ssnData->log_fd < 0) { TSMutexUnlock(ssnData->disk_io_mutex); - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Failed to open log files. Abort."); + TSDebug(PLUGIN_NAME, "global_ssn_handler(): Failed to open log files %s. Abort.", log_f.c_str()); TSHttpSsnReenable(ssnp, TS_EVENT_HTTP_CONTINUE); return TS_EVENT_HTTP_CONTINUE; } - + ssnData->log_name = log_f; // Write log file beginning to disk ssnData->write_to_disk(beginning); } @@ -526,6 +542,8 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata) return TS_SUCCESS; } +} // End of anonymous namespace + void TSPluginInit(int argc, const char *argv[]) { @@ -533,37 +551,29 @@ TSPluginInit(int argc, const char *argv[]) TSPluginRegistrationInfo info; info.plugin_name = "traffic_dump"; - info.vendor_name = "Oath"; - info.support_email = "edge@oath.com"; - - std::string installDir = TSInstallDirGet(); - LOG_DIR = installDir + "/" + LOG_DIR + "/"; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; /// Commandline options - static const struct option longopts[] = { - {"logdir", required_argument, nullptr, 'l'}, {"sample", required_argument, nullptr, 's'}, {nullptr, no_argument, nullptr, 0}}; - int opt = 0; + static const struct option longopts[] = {{"logdir", required_argument, nullptr, 'l'}, + {"sample", required_argument, nullptr, 's'}, + {"limit", required_argument, nullptr, 'm'}, + {nullptr, no_argument, nullptr, 0}}; + int opt = 0; while (opt >= 0) { opt = getopt_long(argc, (char *const *)argv, "l:", longopts, nullptr); switch (opt) { case 'l': { - LOG_DIR = std::string(optarg); - if (LOG_DIR[0] != '/') { - LOG_DIR = installDir + "/" + std::string(optarg) + "/"; - } - TSDebug(PLUGIN_NAME, "Initialized with log dir: %s", LOG_DIR.c_str()); - struct stat st; - if (stat(LOG_DIR.c_str(), &st) == -1) { - TSDebug(PLUGIN_NAME, "Log dir error: (%d) %s", errno, strerror(errno)); - } else { - TSDebug(PLUGIN_NAME, "Log dir opened successfully"); - } + log_path = ts::file::path{optarg}; break; } case 's': { sample_pool_size = static_cast(std::strtol(optarg, nullptr, 0)); break; } + case 'm': { + max_disk_usage = static_cast(std::strtol(optarg, nullptr, 0)); + } case -1: case '?': break; @@ -575,6 +585,12 @@ TSPluginInit(int argc, const char *argv[]) } } + // Make absolute path if not + if (!log_path.is_absolute()) { + log_path = ts::file::path(TSInstallDirGet()) / log_path; + } + TSDebug(PLUGIN_NAME, "Initialized with log directory: %s", log_path.c_str()); + if (TS_SUCCESS != TSPluginRegister(&info)) { TSError("[%s] Unable to initialize plugin (disabled). Failed to register plugin.", PLUGIN_NAME); } else if (TS_SUCCESS != TSHttpSsnArgIndexReserve(PLUGIN_NAME, "Track log related data", &s_arg_idx)) { @@ -585,7 +601,8 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_SSN_START_HOOK, ssncont); TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, ssncont); TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, ssncont); - TSDebug(PLUGIN_NAME, "Initialized with sample pool size %" PRId64 "", sample_pool_size.load()); + TSDebug(PLUGIN_NAME, "Initialized with sample pool size %" PRId64 " bytes and disk limit %" PRId64 "bytes", + sample_pool_size.load(), max_disk_usage.load()); } return; From e3789807fcedbd83a929fbd834307be992771544 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 5 Feb 2019 17:14:47 +0000 Subject: [PATCH 239/526] For tls tests look at logs until config has been reloaded --- tests/gold_tests/tls/tls_client_cert.test.py | 18 +++++++++++++++++- tests/gold_tests/tls/tls_tunnel.test.py | 20 ++++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index 394b8abc506..58bb03eafcc 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -18,6 +18,7 @@ # limitations under the License. import os +import subprocess import re Test.Summary = ''' @@ -32,6 +33,7 @@ # --clientverify: "" empty string because microserver does store_true for argparse, but options is a dictionary server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server3 = Test.MakeOriginServer("server3") server.Setup.Copy("ssl/signer.pem") server.Setup.Copy("ssl/signer2.pem") server.Setup.Copy("ssl/signed-foo.pem") @@ -171,6 +173,20 @@ tr2.Processes.Default.Env = ts.Env tr2.Processes.Default.ReturnCode = 0 +# Parking this as a ready tester on a meaningless process +# Stall the test runs until the ssl_server_name reload has completed +# At that point the new ssl_server_name settings are ready to go +def ssl_server_name_reload_done(tsenv): + def done_reload(process, hasRunFor, **kw): + cmd = "grep 'ssl_server_name.yaml done reloading!' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) + retval = subprocess.run(cmd, shell=True, env=tsenv) + if retval.returncode == 0: + cmd ="if [ -f {0}/test.out -a \"`cat {0}/test.out`\" = \"2\" ] ; then true; else false; fi".format(Test.RunDirectory) + retval = subprocess.run(cmd, shell = True, env=tsenv) + return retval.returncode == 0 + + return done_reload + tr2reload = Test.AddTestRun("Reload config") tr2reload.StillRunningAfter = ts tr2reload.StillRunningAfter = server @@ -184,7 +200,7 @@ #Should succeed tr3bar = Test.AddTestRun("Make request with other bar cert to first server") # Wait for the reload to complete -tr3bar.DelayStart = 10 +tr3bar.Processes.Default.StartBefore(server3, ready=ssl_server_name_reload_done(ts.Env)) tr3bar.StillRunningAfter = ts tr3bar.StillRunningAfter = server tr3bar.StillRunningAfter = server2 diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index 007077b7aee..5f653db20cc 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -17,6 +17,7 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' Test tunneling based on SNI ''' @@ -30,6 +31,7 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=False) server_foo = Test.MakeOriginServer("server_foo", ssl=True) server_bar = Test.MakeOriginServer("server_bar", ssl=True) +server2 = Test.MakeOriginServer("server2") request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -153,11 +155,25 @@ trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 +# Parking this as a ready tester on a meaningless process +# Stall the test runs until the ssl_server_name reload has completed +# At that point the new ssl_server_name settings are ready to go +def ssl_server_name_reload_done(tsenv): + def done_reload(process, hasRunFor, **kw): + cmd = "grep 'ssl_server_name.yaml done reloading!' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) + retval = subprocess.run(cmd, shell=True, env=tsenv) + if retval.returncode == 0: + cmd ="if [ -f {0}/test.out -a \"`cat {0}/test.out`\" = \"2\" ] ; then true; else false; fi".format(Test.RunDirectory) + retval = subprocess.run(cmd, shell = True, env=tsenv) + return retval.returncode == 0 + + return done_reload + # Should termimate on traffic_server (not tunnel) tr = Test.AddTestRun("foo.com no Tunnel-test") tr.StillRunningAfter = ts -# Wait for the reload to complete -tr.DelayStart = 5 +# Wait for the reload to complete by running the ssl_server_name_reload_done test +tr.Processes.Default.StartBefore(server2, ready=ssl_server_name_reload_done(ts.Env)) tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") From 4a9e6bee4800c9f6491d553f97e0ca38244c38a2 Mon Sep 17 00:00:00 2001 From: Pushkar Pradhan Date: Tue, 5 Feb 2019 11:40:50 -0800 Subject: [PATCH 240/526] cookie_remap plugin documentation had typo in name of substituted variable --- doc/admin-guide/plugins/cookie_remap.en.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/admin-guide/plugins/cookie_remap.en.rst b/doc/admin-guide/plugins/cookie_remap.en.rst index 24b5d94783b..897331696f7 100644 --- a/doc/admin-guide/plugins/cookie_remap.en.rst +++ b/doc/admin-guide/plugins/cookie_remap.en.rst @@ -43,7 +43,7 @@ Cookie Based Routing Inside TrafficServer Using cookie_remap * `Reserved path expressions <#reserved-path-expressions>`_ - * `$cr_request_url <#cr_request_url-v-15>`_ + * `$cr_req_url <#cr_req_url-v-15>`_ * `$cr_urlencode() <#cr_urlencode-v-15>`_ * `$path <#path>`_ * `$unmatched_path <#unmatched_path>`_ @@ -219,7 +219,7 @@ Reserved path expressions The following expressions can be used in either the sendto **or** ``else`` URLs, and will be expanded. -$cr_request_url +$cr_req_url ^^^^^^^^^^^^^^^^^^^^^^^^^ ---- @@ -235,7 +235,7 @@ Therefore, a rule like: op: cookie: K operation: exists - sendto: http://foo.com/?.done=$cr_request_url + sendto: http://foo.com/?.done=$cr_req_url @@ -275,7 +275,7 @@ Therefore, a rule like: op: cookie: B operation: exists - sendto: http://foo.com/?.done=$cr_urlencode($cr_request_url) + sendto: http://foo.com/?.done=$cr_urlencode($cr_req_url) From 9a3141b319a2f455dc04200ade76bad36361e139 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 6 Feb 2019 11:09:59 -0600 Subject: [PATCH 241/526] ats_scoped_str: c++17 cleanup. --- include/tscore/ink_memory.h | 129 +++++++++++++++++++++--------------- src/tscore/ink_memory.cc | 12 ++++ 2 files changed, 87 insertions(+), 54 deletions(-) diff --git a/include/tscore/ink_memory.h b/include/tscore/ink_memory.h index cf850c0b9be..464c10eb90d 100644 --- a/include/tscore/ink_memory.h +++ b/include/tscore/ink_memory.h @@ -280,10 +280,18 @@ class ats_scoped_resource /// Construct with contained resource. explicit ats_scoped_resource(value_type rt) : _r(rt) {} /// Destructor. - ~ats_scoped_resource() + ~ats_scoped_resource() { this->clear(); } + + /** Remove and clean up any existing resource in this object, and return to a default constructed state. + * + */ + void + clear() { - if (Traits::isValid(_r)) + if (Traits::isValid(_r)) { Traits::destroy(_r); + } + _r = Traits::initValue(); } /// Automatic conversion to resource type. @@ -326,8 +334,7 @@ class ats_scoped_resource self & operator=(value_type rt) { - if (Traits::isValid(_r)) - Traits::destroy(_r); + this->clear(); _r = rt; return *this; } @@ -473,60 +480,74 @@ struct SCOPED_OBJECT_TRAITS { class ats_scoped_str : public ats_scoped_resource> { public: - typedef ats_scoped_resource> super; ///< Super type. - typedef ats_scoped_str self; ///< Self reference type. - - /// Default constructor (no string). - ats_scoped_str() {} - /// Construct and allocate @a n bytes for a string. + using super = ats_scoped_resource>; ///< Super type. + using self_type = ats_scoped_str; ///< Self reference type. + + /** Default constructor. + * Constructs an empty container. + */ + ats_scoped_str() = default; + + /** Construct an empty buffer of @a n bytes. + * + * @param n Size of the contained buffer. + * + * The memory is left uninitialized, presumably ready for a @c memcpy. + */ explicit ats_scoped_str(size_t n) : super(static_cast(ats_malloc(n))) {} - /// Put string @a s in this container for cleanup. + + /** Construct with an existing buffer. + * + * @param s The string. + * + * This container takes ownership of the string memory. For this reason @a s must be independently + * allocated, not part of a larger memory allocation. + */ explicit ats_scoped_str(char *s) : super(s) {} - // constructor with std::string - explicit ats_scoped_str(const std::string &s) - { - if (s.empty()) - _r = nullptr; - else - _r = strdup(s.c_str()); - } - // constructor with string_view - explicit ats_scoped_str(const std::string_view &s) - { - if (s.empty()) - _r = nullptr; - else - _r = strdup(s.data()); - } - /// Assign a string @a s to this container. - self & - operator=(char *s) - { - super::operator=(s); - return *this; - } - // std::string case - self & - operator=(const std::string &s) - { - if (s.empty()) - _r = nullptr; - else - _r = strdup(s.c_str()); - return *this; - } - // string_view case - self & - operator=(const std::string_view &s) - { - if (s.empty()) - _r = nullptr; - else - _r = strdup(s.data()); - return *this; - } + + /** Construct from a @c string_view. + * + * @param s The string to copy. + * + * The string in @a s is duplicated into this container. + */ + explicit ats_scoped_str(std::string_view s); + + /** Put an existing buffer in this container. + * + * @param s The string buffer. + * @return @a this + * + * The container takes ownership of @a s. For this reason @a s must be independently allocated, + * not part of a larger memory allocation. Any currently contained resource is destroyed. + */ + self_type &operator=(char *s); + + /** Assign from a @c string_view + * + * @param s The source string. + * @return @a this + * + * The string data is duplicated into this object and guaranteed to be null terminated. Any currently + * contained resource is destroyed. + */ + self_type &operator=(std::string_view s); }; +inline ats_scoped_str::ats_scoped_str(std::string_view s) +{ + if (!s.empty()) { + *this = s; + } +} + +inline ats_scoped_str & +ats_scoped_str::operator=(char *s) +{ + super::operator=(s); + return *this; +} + /** Specialization of @c ats_scoped_resource for pointers allocated with @c ats_malloc. */ template clear(); + if (!s.empty()) { + _r = static_cast(ats_malloc(s.size() + 1)); + memcpy(_r, s.data(), s.size()); + _r[s.size()] = '\0'; + } + return *this; +} From d77e5f9e833783151cf677f7c8a57cb8a084beb7 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Sat, 2 Feb 2019 14:09:39 -0600 Subject: [PATCH 242/526] Runroot: make traffic_top use runroot --- src/traffic_top/traffic_top.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/traffic_top/traffic_top.cc b/src/traffic_top/traffic_top.cc index 4021cc77d91..e093636f065 100644 --- a/src/traffic_top/traffic_top.cc +++ b/src/traffic_top/traffic_top.cc @@ -57,6 +57,7 @@ #include "tscore/ink_args.h" #include "records/I_RecProcess.h" #include "RecordsConfig.h" +#include "tscore/runroot.h" using namespace std; @@ -407,6 +408,7 @@ main(int argc, const char **argv) process_args(&version, argument_descriptions, countof(argument_descriptions), argv, USAGE); + runroot_handler(argv); Layout::create(); RecProcessInit(RECM_STAND_ALONE, nullptr /* diags */); LibRecordsConfigInit(); From 457f4c5500eea5977dc01036963617e634a40c46 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Wed, 6 Feb 2019 12:37:22 +0900 Subject: [PATCH 243/526] Use the appropriate length field for uintmax_t --- plugins/experimental/traffic_dump/traffic_dump.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc b/plugins/experimental/traffic_dump/traffic_dump.cc index d8733ca5646..a66b90874c6 100644 --- a/plugins/experimental/traffic_dump/traffic_dump.cc +++ b/plugins/experimental/traffic_dump/traffic_dump.cc @@ -304,7 +304,7 @@ session_aio_handler(TSCont contp, TSEvent event, void *edata) ts::file::file_status st = ts::file::status(ssnData->log_name, ec); if (!ec) { disk_usage += ts::file::file_size(st); - TSDebug(PLUGIN_NAME, "Finish a session with log file of %" PRId64 "bytes", ts::file::file_size(st)); + TSDebug(PLUGIN_NAME, "Finish a session with log file of %" PRIuMAX "bytes", ts::file::file_size(st)); } delete ssnData; return TS_SUCCESS; From 72a79c0ea3225de19e10cd34c4182973133fb57a Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Sun, 3 Feb 2019 15:57:44 -0600 Subject: [PATCH 244/526] grab the lock if there is one --- src/traffic_server/traffic_server.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index e607538e2ef..712328bbf33 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -224,6 +224,7 @@ struct AutoStopCont : public Continuation { { APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_SHUTDOWN_HOOK); while (hook) { + SCOPED_MUTEX_LOCK(lock, hook->m_cont->mutex, this_ethread()); hook->invoke(TS_EVENT_LIFECYCLE_SHUTDOWN, nullptr); hook = hook->next(); } From 99f3c741b93fcb90993f8d2581c77c6b090c35ce Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 6 Feb 2019 20:52:00 +0000 Subject: [PATCH 245/526] Load client key event if only the certificate file is specified. --- iocore/net/SSLSNIConfig.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 80dc8c4b5c1..1925374e2c7 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -84,9 +84,13 @@ SNIConfigParams::loadSNIConfig() // set the next hop properties SSLConfig::scoped_config params; auto clientCTX = params->getClientSSL_CTX(); - if (!item.client_cert.empty() && !item.client_key.empty()) { + // Load if we have at least specified the client certificate + if (!item.client_cert.empty()) { std::string certFilePath = Layout::get()->relative_to(params->clientCertPathOnly, item.client_cert.data()); - std::string keyFilePath = Layout::get()->relative_to(params->clientKeyPathOnly, item.client_key.data()); + std::string keyFilePath; + if (!item.client_key.empty()) { + keyFilePath = Layout::get()->relative_to(params->clientKeyPathOnly, item.client_key.data()); + } clientCTX = params->getCTX(certFilePath.c_str(), keyFilePath.c_str(), params->clientCACertFilename, params->clientCACertPath); } From 3827153504cabcf168b3ca45f37280c65adcfd3f Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 6 Feb 2019 16:29:50 +0000 Subject: [PATCH 246/526] Fix client_hello_cb to correctly check for sufficient server_name data --- iocore/net/SSLUtils.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index e7a64447677..cf87bd41273 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -426,7 +426,8 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) const char *servername = nullptr; const unsigned char *p; size_t remaining, len; - if (SSL_client_hello_get0_ext(s, TLSEXT_TYPE_server_name, &p, &remaining) || remaining <= 2) { + // Parse the servrer name if the get extension call succeeds and there are more than 2 bytes to parse + if (SSL_client_hello_get0_ext(s, TLSEXT_TYPE_server_name, &p, &remaining) && remaining > 2) { // Parse to get to the name, originally from test/handshake_helper.c in openssl tree /* Extract the length of the supplied list of names. */ len = *(p++) << 8; @@ -457,7 +458,7 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) netvc->serverName = servername ? servername : ""; int ret = PerformAction(netvc, netvc->serverName); if (ret != SSL_TLSEXT_ERR_OK) { - return SSL_TLSEXT_ERR_ALERT_FATAL; + return SSL_CLIENT_HELLO_ERROR; } if (netvc->has_tunnel_destination() && !netvc->decrypt_tunnel()) { netvc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; @@ -465,7 +466,7 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) if (netvc->protocol_mask_set) { setTLSValidProtocols(s, netvc->protocol_mask, TLSValidProtocols::max_mask); } - return 1; + return SSL_CLIENT_HELLO_SUCCESS; } #endif From a4227747277759c27c63ba948d96fbd6204dc8db Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 6 Feb 2019 15:45:53 +0900 Subject: [PATCH 247/526] Mark H2 connection inactive only if it is NOT shutting down Prior this change, HTTP/2 connection is marked as inactive regardless it is shutting down or not. --- proxy/http2/Http2ClientSession.cc | 2 ++ proxy/http2/Http2ConnectionState.cc | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index 3cc9edd58dd..e6037fd17cd 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -272,6 +272,8 @@ Http2ClientSession::do_io_close(int alerrno) SCOPED_MUTEX_LOCK(lock, this->connection_state.mutex, this_ethread()); this->connection_state.release_stream(nullptr); } + + this->clear_session_active(); } void diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index a6843ec1a59..45689d7a262 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -1245,14 +1245,7 @@ Http2ConnectionState::release_stream(Http2Stream *stream) SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); if (this->ua_session) { ink_assert(this->mutex == ua_session->mutex); - // If the number of clients is 0 and ua_session is active, then mark the connection as inactive - if (total_client_streams_count == 0 && ua_session->is_active()) { - ua_session->clear_session_active(); - UnixNetVConnection *vc = static_cast(ua_session->get_netvc()); - if (vc && vc->active_timeout_in == 0) { - vc->add_to_keep_alive_queue(); - } - } + if (total_client_streams_count == 0) { if (fini_received) { // We were shutting down, go ahead and terminate the session @@ -1265,6 +1258,15 @@ Http2ConnectionState::release_stream(Http2Stream *stream) // ua_session = nullptr; } else if (shutdown_state == HTTP2_SHUTDOWN_IN_PROGRESS && fini_event == nullptr) { fini_event = this_ethread()->schedule_imm_local((Continuation *)this, HTTP2_SESSION_EVENT_FINI); + } else if (ua_session->is_active()) { + // If the number of clients is 0, HTTP2_SESSION_EVENT_FINI is not received or sent, and ua_session is active, + // then mark the connection as inactive + ua_session->clear_session_active(); + UnixNetVConnection *vc = static_cast(ua_session->get_netvc()); + if (vc && vc->active_timeout_in == 0) { + // With heavy traffic, ua_session could be destroyed. Do not touch ua_session after this. + vc->add_to_keep_alive_queue(); + } } else { schedule_zombie_event(); } From ef780ea0b5b212d9a5667f1160496445f2450a78 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 15 Jan 2019 12:48:27 -0600 Subject: [PATCH 248/526] DFA: Update DFA to use string_view. --- include/tscore/Regex.h | 51 ++++++++---- iocore/net/P_SSLUtils.h | 2 +- lib/records/RecCore.cc | 2 +- proxy/hdrs/HdrToken.cc | 2 +- proxy/hdrs/MIME.cc | 9 ++- src/traffic_cache_tool/CacheDefs.h | 19 ++--- src/tscore/Regex.cc | 120 +++++++++-------------------- 7 files changed, 89 insertions(+), 116 deletions(-) diff --git a/include/tscore/Regex.h b/include/tscore/Regex.h index f80121e47a6..3c95b2728bd 100644 --- a/include/tscore/Regex.h +++ b/include/tscore/Regex.h @@ -24,6 +24,9 @@ #pragma once #include +#include +#include +#include #include "tscore/ink_config.h" @@ -49,7 +52,9 @@ class Regex /// Default number of capture groups. static constexpr size_t DEFAULT_GROUP_COUNT = 10; - Regex() = default; + Regex() = default; + Regex(Regex const &) = delete; // No copying. + Regex(Regex &&that) noexcept; ~Regex(); /** Compile the @a pattern into a regular expression. @@ -60,7 +65,7 @@ class Regex * * @a flags should be the bitwise @c or of @c REFlags values. */ - bool compile(const char *pattern, const unsigned flags = 0); + bool compile(const char *pattern, unsigned flags = 0); /** Execute the regular expression. * @@ -93,27 +98,45 @@ class Regex pcre_extra *regex_extra = nullptr; }; -typedef struct __pat { - int _idx; - Regex *_re; - char *_p; - __pat *_next; -} dfa_pattern; - +/** Deterministic Finite state Automata container. + * + * This contains a set of patterns (which may be of size 1) and matches if any of the patterns + * match. + */ class DFA { public: DFA() = default; ~DFA(); - int compile(const char *pattern, unsigned flags = 0); + /// @return The number of patterns successfully compiled. + int compile(std::string_view const &pattern, unsigned flags = 0); + /// @return The number of patterns successfully compiled. + int compile(std::string_view *patterns, int npatterns, unsigned flags = 0); + /// @return The number of patterns successfully compiled. int compile(const char **patterns, int npatterns, unsigned flags = 0); - int match(const char *str) const; - int match(const char *str, int length) const; + /** Match @a str against the internal patterns. + * + * @param str String to match. + * @return Index of the matched pattern, -1 if no match. + */ + int match(std::string_view const &str) const; private: - dfa_pattern *build(const char *pattern, unsigned flags = 0); + struct Pattern { + Pattern(Regex &&rxp, std::string &&s) : _re(std::move(rxp)), _p(std::move(s)) {} + Regex _re; ///< The compile pattern. + std::string _p; ///< The original pattern. + }; + + /** Compile @a pattern and add it to the pattern set. + * + * @param pattern Regular expression to compile. + * @param flags Regular expression compilation flags. + * @return @c true if @a pattern was successfully compiled, @c false if not. + */ + bool build(std::string_view const &pattern, unsigned flags = 0); - dfa_pattern *_my_patterns = nullptr; + std::vector _patterns; }; diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index e3ec9f6d082..6990db6e4d6 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -227,7 +227,7 @@ namespace detail struct ats_wildcard_matcher { ats_wildcard_matcher() { - if (regex.compile("^\\*\\.[^\\*.]+") != 0) { + if (!regex.compile("^\\*\\.[^\\*.]+")) { Fatal("failed to compile TLS wildcard matching regex"); } } diff --git a/lib/records/RecCore.cc b/lib/records/RecCore.cc index 388a69be580..68dd6fa4f7e 100644 --- a/lib/records/RecCore.cc +++ b/lib/records/RecCore.cc @@ -497,7 +497,7 @@ RecLookupMatchingRecords(unsigned rec_type, const char *match, void (*callback)( int num_records; DFA regex; - if (regex.compile(match, RE_CASE_INSENSITIVE | RE_UNANCHORED) != 0) { + if (!regex.compile(match, RE_CASE_INSENSITIVE | RE_UNANCHORED)) { return REC_ERR_FAIL; } diff --git a/proxy/hdrs/HdrToken.cc b/proxy/hdrs/HdrToken.cc index 55fe5200a6f..77759924903 100644 --- a/proxy/hdrs/HdrToken.cc +++ b/proxy/hdrs/HdrToken.cc @@ -522,7 +522,7 @@ hdrtoken_tokenize_dfa(const char *string, int string_len, const char **wks_strin { int wks_idx; - wks_idx = hdrtoken_strs_dfa->match(string, string_len); + wks_idx = hdrtoken_strs_dfa->match({string, size_t(string_len)}); if (wks_idx < 0) { wks_idx = -1; diff --git a/proxy/hdrs/MIME.cc b/proxy/hdrs/MIME.cc index 33ab602c265..81c462223b7 100644 --- a/proxy/hdrs/MIME.cc +++ b/proxy/hdrs/MIME.cc @@ -3282,6 +3282,7 @@ int mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) { unsigned int three_char_wday, three_char_mon; + std::string_view view{buf, size_t(length)}; ink_assert(length >= 29); ink_assert(!is_ws(buf[0])); @@ -3312,7 +3313,7 @@ mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) } } if (tp->tm_wday < 0) { - tp->tm_wday = day_names_dfa->match(buf, length); + tp->tm_wday = day_names_dfa->match(view); if (tp->tm_wday < 0) { return 0; } @@ -3365,7 +3366,7 @@ mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) } } if (tp->tm_mon < 0) { - tp->tm_mon = month_names_dfa->match(buf, length); + tp->tm_mon = month_names_dfa->match(view); if (tp->tm_mon < 0) { return 0; } @@ -3510,7 +3511,7 @@ mime_parse_day(const char *&buf, const char *end, int *day) e += 1; } - *day = day_names_dfa->match(buf, e - buf); + *day = day_names_dfa->match({buf, size_t(e - buf)}); if (*day < 0) { return 0; } else { @@ -3533,7 +3534,7 @@ mime_parse_month(const char *&buf, const char *end, int *month) e += 1; } - *month = month_names_dfa->match(buf, e - buf); + *month = month_names_dfa->match({buf, size_t(e - buf)}); if (*month < 0) { return 0; } else { diff --git a/src/traffic_cache_tool/CacheDefs.h b/src/traffic_cache_tool/CacheDefs.h index 4a980827a49..db5fd146483 100644 --- a/src/traffic_cache_tool/CacheDefs.h +++ b/src/traffic_cache_tool/CacheDefs.h @@ -300,11 +300,11 @@ struct url_matcher { for (i = 0; i < count; i++) { std::cout << "regex " << patterns[i] << std::endl; } - if (regex.compile(patterns, count) != 0) { + if (regex.compile(patterns, count) != count) { std::cout << "Check your regular expression" << std::endl; } - if (port.compile(R"([0-9]+$)") != 0) { + if (!port.compile(R"([0-9]+$)")) { std::cout << "Check your regular expression" << std::endl; return; } @@ -313,11 +313,11 @@ struct url_matcher { url_matcher() { - if (regex.compile(R"(^(https?\:\/\/)") != 0) { + if (!regex.compile(R"(^(https?\:\/\/)")) { std::cout << "Check your regular expression" << std::endl; return; } - if (port.compile(R"([0-9]+$)") != 0) { + if (!port.compile(R"([0-9]+$)")) { std::cout << "Check your regular expression" << std::endl; return; } @@ -328,19 +328,12 @@ struct url_matcher { uint8_t match(const char *hostname) const { - if (regex.match(hostname) != -1) { - return 1; - } - - return 0; + return regex.match(hostname) ? 1 : 0; } uint8_t portmatch(const char *hostname, int length) const { - if (port.match(hostname, length) != -1) { - return 1; - } - return 0; + return port.match({hostname, size_t(length)}) ? 1 : 0; } private: diff --git a/src/tscore/Regex.cc b/src/tscore/Regex.cc index d067072a27f..d0a7a8bd864 100644 --- a/src/tscore/Regex.cc +++ b/src/tscore/Regex.cc @@ -50,6 +50,12 @@ get_jit_stack(void *data ATS_UNUSED) } #endif +Regex::Regex(Regex &&that) noexcept : regex(that.regex), regex_extra(that.regex_extra) +{ + that.regex = nullptr; + that.regex_extra = nullptr; +} + bool Regex::compile(const char *pattern, const unsigned flags) { @@ -114,7 +120,7 @@ Regex::exec(std::string_view const &str, int *ovector, int ovecsize) { int rv; - rv = pcre_exec(regex, regex_extra, str.data(), str.size(), 0, 0, ovector, ovecsize); + rv = pcre_exec(regex, regex_extra, str.data(), int(str.size()), 0, 0, ovector, ovecsize); return rv > 0; } @@ -132,113 +138,63 @@ Regex::~Regex() } } -DFA::~DFA() -{ - dfa_pattern *p = _my_patterns; - dfa_pattern *t; +DFA::~DFA() {} - while (p) { - if (p->_re) { - delete p->_re; - } - if (p->_p) { - ats_free(p->_p); - } - t = p->_next; - ats_free(p); - p = t; - } -} - -dfa_pattern * -DFA::build(const char *pattern, unsigned flags) +bool +DFA::build(std::string_view const &pattern, unsigned flags) { - dfa_pattern *ret; - int rv; + Regex rxp; + std::string string{pattern}; if (!(flags & RE_UNANCHORED)) { flags |= RE_ANCHORED; } - ret = (dfa_pattern *)ats_malloc(sizeof(dfa_pattern)); - ret->_p = nullptr; - - ret->_re = new Regex(); - rv = ret->_re->compile(pattern, flags); - if (rv == -1) { - delete ret->_re; - ats_free(ret); - return nullptr; + if (!rxp.compile(string.c_str(), flags)) { + return false; } - - ret->_idx = 0; - ret->_p = ats_strndup(pattern, strlen(pattern)); - ret->_next = nullptr; - return ret; + _patterns.emplace_back(std::move(rxp), std::move(string)); + return true; } int -DFA::compile(const char *pattern, unsigned flags) +DFA::compile(std::string_view const &pattern, unsigned flags) { - ink_assert(_my_patterns == nullptr); - _my_patterns = build(pattern, flags); - if (_my_patterns) { - return 0; - } else { - return -1; - } + ink_assert(_patterns.empty()); + this->build(pattern, flags); + return _patterns.size(); } int -DFA::compile(const char **patterns, int npatterns, unsigned flags) +DFA::compile(std::string_view *patterns, int npatterns, unsigned flags) { - const char *pattern; - dfa_pattern *ret = nullptr; - dfa_pattern *end = nullptr; - int i; - - for (i = 0; i < npatterns; i++) { - pattern = patterns[i]; - ret = build(pattern, flags); - if (!ret) { - continue; - } - - if (!_my_patterns) { - _my_patterns = ret; - _my_patterns->_next = nullptr; - _my_patterns->_idx = i; - } else { - end = _my_patterns; - while (end->_next) { - end = end->_next; - } - end->_next = ret; // add to end - ret->_idx = i; - } + _patterns.reserve(npatterns); // try to pre-allocate. + for (int i = 0; i < npatterns; ++i) { + this->build(patterns[i], flags); } - - return 0; + return _patterns.size(); } int -DFA::match(const char *str) const +DFA::compile(const char **patterns, int npatterns, unsigned flags) { - return match(str, strlen(str)); + _patterns.reserve(npatterns); // try to pre-allocate. + for (int i = 0; i < npatterns; ++i) { + this->build(patterns[i], flags); + } + return _patterns.size(); } int -DFA::match(const char *str, int length) const +DFA::match(std::string_view const &str) const { - int rc; - dfa_pattern *p = _my_patterns; - - while (p) { - rc = p->_re->exec({str, size_t(length)}); - if (rc > 0) { - return p->_idx; + // This is ugly, but the external interface needs to be @c const even though it's not really. + // This handles making the iterator non-const. + auto &pv{const_cast(_patterns)}; + for (auto spot = pv.begin(), limit = pv.end(); spot != limit; ++spot) { + if (spot->_re.exec(str)) { + return spot - _patterns.begin(); } - p = p->_next; } return -1; From 3a3259502090ca4bc76f27915ad91fadde782cc6 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 8 Feb 2019 10:37:19 -0700 Subject: [PATCH 249/526] Marks the YAML exceptions, which gives line number / pos info --- iocore/net/YamlSNIConfig.cc | 8 ++++---- proxy/logging/YamlLogConfig.cc | 14 +++++++------- proxy/logging/YamlLogConfigDecoders.cc | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc index 84e118ea006..6ba226e3580 100644 --- a/iocore/net/YamlSNIConfig.cc +++ b/iocore/net/YamlSNIConfig.cc @@ -115,7 +115,7 @@ template <> struct convert { for (auto &&item : node) { if (std::none_of(valid_sni_config_keys.begin(), valid_sni_config_keys.end(), [&item](std::string s) { return s == item.first.as(); })) { - throw std::runtime_error("unsupported key " + item.first.as()); + throw YAML::ParserException(item.Mark(), "unsupported key " + item.first.as()); } } @@ -133,7 +133,7 @@ template <> struct convert { auto value = node[TS_verify_client].as(); int level = LEVEL_DESCRIPTOR.get(value); if (level < 0) { - throw std::runtime_error("unknown value \"" + value + "\""); + throw YAML::ParserException(node[TS_verify_client].Mark(), "unknown value \"" + value + "\""); } item.verify_client_level = static_cast(level); } @@ -169,7 +169,7 @@ template <> struct convert { auto value = node[TS_verify_server_policy].as(); int policy = POLICY_DESCRIPTOR.get(value); if (policy < 0) { - throw std::runtime_error("unknown value \"" + value + "\""); + throw YAML::ParserException(node[TS_verify_server_policy].Mark(), "unknown value \"" + value + "\""); } item.verify_server_policy = static_cast(policy); } @@ -178,7 +178,7 @@ template <> struct convert { auto value = node[TS_verify_server_properties].as(); int properties = PROPERTIES_DESCRIPTOR.get(value); if (properties < 0) { - throw std::runtime_error("unknown value \"" + value + "\""); + throw YAML::ParserException(node[TS_verify_server_properties].Mark(), "unknown value \"" + value + "\""); } item.verify_server_properties = static_cast(properties); } diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc index cc4c6ace7b8..71bc8ef9a52 100644 --- a/proxy/logging/YamlLogConfig.cc +++ b/proxy/logging/YamlLogConfig.cc @@ -111,17 +111,17 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) for (auto &&item : node) { if (std::none_of(valid_log_object_keys.begin(), valid_log_object_keys.end(), [&item](std::string s) { return s == item.first.as(); })) { - throw std::runtime_error("log: unsupported key '" + item.first.as() + "'"); + throw YAML::ParserException(item.Mark(), "log: unsupported key '" + item.first.as() + "'"); } } if (!node["format"]) { - throw std::runtime_error("missing 'format' argument"); + throw YAML::ParserException(node.Mark(), "missing 'format' argument"); } std::string format = node["format"].as(); if (!node["filename"]) { - throw std::runtime_error("missing 'filename' argument"); + throw YAML::ParserException(node.Mark(), "missing 'filename' argument"); } std::string header; @@ -155,7 +155,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) auto value = node["rolling_enabled"].as(); obj_rolling_enabled = ROLLING_MODE.get(value); if (obj_rolling_enabled < 0) { - throw std::runtime_error("unknown value " + value); + throw YAML::ParserException(node["rolling_enabled"].Mark(), "unknown value " + value); } } if (node["rolling_interval_sec"]) { @@ -202,7 +202,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } if (!filters.IsSequence()) { - throw std::runtime_error("'filters' should be a list"); + throw YAML::ParserException(filters.Mark(), "'filters' should be a list"); } for (auto &&filter : filters) { @@ -221,7 +221,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } if (!collation_host_list.IsSequence()) { - throw std::runtime_error("'collation_hosts' should be a list of collation_host objects"); + throw YAML::ParserException(collation_host_list.Mark(), "'collation_hosts' should be a list of collation_host objects"); } for (auto &&collation_host : collation_host_list) { @@ -246,7 +246,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) if (!collation_host["failover"].IsSequence()) { delete lh; - throw std::runtime_error("'failover' should be a list of host names"); + throw YAML::ParserException(collation_host["failover"].Mark(), "'failover' should be a list of host names"); } LogHost *prev = lh; diff --git a/proxy/logging/YamlLogConfigDecoders.cc b/proxy/logging/YamlLogConfigDecoders.cc index d1853c1145d..6146aca09c6 100644 --- a/proxy/logging/YamlLogConfigDecoders.cc +++ b/proxy/logging/YamlLogConfigDecoders.cc @@ -39,12 +39,12 @@ convert>::decode(const Node &node, std::unique_ptr(); })) { - throw std::runtime_error("format: unsupported key '" + item.first.as() + "'"); + throw YAML::ParserException(node.Mark(), "format: unsupported key '" + item.first.as() + "'"); } } if (!node["format"]) { - throw std::runtime_error("missing 'format' argument"); + throw YAML::ParserException(node.Mark(), "missing 'format' argument"); } std::string format = node["format"].as(); @@ -80,14 +80,14 @@ convert>::decode(const Node &node, std::unique_ptr(); })) { - throw std::runtime_error("filter: unsupported key '" + item.first.as() + "'"); + throw YAML::ParserException(node.Mark(), "filter: unsupported key '" + item.first.as() + "'"); } } // we require all keys for LogFilter for (auto &&item : valid_log_filter_keys) { if (!node[item]) { - throw std::runtime_error("missing '" + item + "' argument"); + throw YAML::ParserException(node.Mark(), "missing '" + item + "' argument"); } } From 46c2bb55bece3183e257178dfdf2e4dc884f30e9 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 8 Feb 2019 08:05:37 -0600 Subject: [PATCH 250/526] Cleanup: Tweak for loops to be more consistently C++17 in style. --- proxy/logging/YamlLogConfig.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc index 71bc8ef9a52..948c5a630cd 100644 --- a/proxy/logging/YamlLogConfig.cc +++ b/proxy/logging/YamlLogConfig.cc @@ -60,8 +60,8 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) } auto formats = config["formats"]; - for (auto it = formats.begin(); it != formats.end(); ++it) { - auto fmt = it->as>().release(); + for (auto const &node : formats) { + auto fmt = node.as>().release(); if (fmt->valid()) { cfg->format_list.add(fmt, false); @@ -76,8 +76,8 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) } auto filters = config["filters"]; - for (auto it = filters.begin(); it != filters.end(); ++it) { - auto filter = it->as>().release(); + for (auto const &node : filters) { + auto filter = node.as>().release(); if (filter) { cfg->filter_list.add(filter, false); @@ -90,8 +90,8 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) } auto logs = config["logs"]; - for (auto it = logs.begin(); it != logs.end(); ++it) { - auto obj = decodeLogObject(*it); + for (auto const &node : logs) { + auto obj = decodeLogObject(node); if (obj) { cfg->log_object_manager.manage_object(obj); } @@ -108,7 +108,7 @@ std::set valid_log_object_keys = { LogObject * YamlLogConfig::decodeLogObject(const YAML::Node &node) { - for (auto &&item : node) { + for (auto const &item : node) { if (std::none_of(valid_log_object_keys.begin(), valid_log_object_keys.end(), [&item](std::string s) { return s == item.first.as(); })) { throw YAML::ParserException(item.Mark(), "log: unsupported key '" + item.first.as() + "'"); @@ -205,7 +205,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) throw YAML::ParserException(filters.Mark(), "'filters' should be a list"); } - for (auto &&filter : filters) { + for (auto const &filter : filters) { std::string filter_name = filter.as().c_str(); LogFilter *f = cfg->filter_list.find_by_name(filter_name.c_str()); if (!f) { @@ -224,7 +224,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) throw YAML::ParserException(collation_host_list.Mark(), "'collation_hosts' should be a list of collation_host objects"); } - for (auto &&collation_host : collation_host_list) { + for (auto const &collation_host : collation_host_list) { if (!collation_host["host"]) { Warning("no collation 'host' name; cannot add this Collation host"); continue; @@ -250,7 +250,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } LogHost *prev = lh; - for (auto &&failover_host : collation_host["failover"]) { + for (auto const &failover_host : collation_host["failover"]) { auto failover_host_name = failover_host.as(); LogHost *flh = new LogHost(logObject->get_full_filename(), logObject->get_signature()); if (!flh->set_name_or_ipstr(failover_host_name.c_str())) { From 34dcdb278b92b04f2c19f1cd3a48f94c8ae65dfb Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Sat, 2 Feb 2019 19:13:33 +0800 Subject: [PATCH 251/526] Optimize: Keep cont->mutex locked, during probe the bucket by hash object within HostDBProcessor::getby --- iocore/hostdb/HostDB.cc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index dd80c58ecc6..fbf410c2438 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -634,19 +634,18 @@ HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockad // Attempt to find the result in-line, for level 1 hits // if (!aforce_dns) { - bool loop; - do { + MUTEX_TRY_LOCK(lock, cont->mutex, thread); + bool loop = lock.is_locked(); + while (loop) { loop = false; // Only loop on explicit set for retry. // find the partition lock // - // TODO: Could we reuse the "mutex" above safely? I think so but not sure. - Ptr bmutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); - MUTEX_TRY_LOCK(lock, bmutex, thread); - MUTEX_TRY_LOCK(lock2, cont->mutex, thread); + Ptr bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + MUTEX_TRY_LOCK(lock2, bucket_mutex, thread); - if (lock.is_locked() && lock2.is_locked()) { + if (lock2.is_locked()) { // If we can get the lock and a level 1 probe succeeds, return - Ptr r = probe(bmutex, hash, aforce_dns); + Ptr r = probe(bucket_mutex, hash, aforce_dns); if (r) { if (r->is_failed() && hostname) { loop = check_for_retry(hash.db_mark, host_res_style); @@ -662,7 +661,7 @@ HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockad hash.refresh(); // only on reloop, because we've changed the family. } } - } while (loop); + } } Debug("hostdb", "delaying force %d answer for %s", aforce_dns, hostname ? hostname : ats_is_ip(ip) ? ats_ip_ntop(ip, ipb, sizeof ipb) : ""); From 7a81609082ce887e754bfc358c5654d4045077b6 Mon Sep 17 00:00:00 2001 From: Kit Chan Date: Sun, 10 Feb 2019 23:28:21 -0800 Subject: [PATCH 252/526] Add doc for using relative path to configuration directory for lua script --- doc/admin-guide/plugins/lua.en.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/admin-guide/plugins/lua.en.rst b/doc/admin-guide/plugins/lua.en.rst index 624056d5693..c955b8101f2 100644 --- a/doc/admin-guide/plugins/lua.en.rst +++ b/doc/admin-guide/plugins/lua.en.rst @@ -80,7 +80,8 @@ Configuration ============= This module acts as remap plugin of Traffic Server, so we should realize 'do_remap' or 'do_os_response' function in each -lua script. We can write this in remap.config: +lua script. The path referencing a file with the lua script can be relative to the configuration directory or an absolute +path. We can write this in remap.config: :: From 059e92a2a717a6d44fe553c82f8faf83ca7bcace Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Fri, 8 Feb 2019 14:26:32 -0600 Subject: [PATCH 253/526] ssl handshake fix --- iocore/net/SSLNetVConnection.cc | 7 +++++++ src/traffic_server/traffic_server.cc | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 86cc813a1a1..14712eee096 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -73,6 +73,8 @@ void SSL_set0_rbio(SSL *ssl, BIO *rbio); ClassAllocator sslNetVCAllocator("sslNetVCAllocator"); +bool stop_ssl_handshake = false; + namespace { /// Callback to get two locks. @@ -937,9 +939,14 @@ SSLNetVConnection::free(EThread *t) THREAD_FREE(this, sslNetVCAllocator, t); } } + int SSLNetVConnection::sslStartHandShake(int event, int &err) { + if (stop_ssl_handshake) { + Debug("ssl", "Stopping handshake due to server shutting down."); + return EVENT_ERROR; + } if (sslHandshakeBeginTime == 0) { sslHandshakeBeginTime = Thread::get_hrtime(); // net_activity will not be triggered until after the handshake diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 712328bbf33..2b9eaaac474 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -173,6 +173,9 @@ static int delay_listen_for_cache_p; // Keeps track if the server is in draining state, follows the proxy.node.config.draining metric bool ts_is_draining = false; +// Flag to stop ssl handshakes during shutdown. +extern bool stop_ssl_handshake; + AppVersionInfo appVersionInfo; // Build info for this application static ArgumentDescription argument_descriptions[] = { @@ -222,6 +225,8 @@ struct AutoStopCont : public Continuation { int mainEvent(int /* event */, Event * /* e */) { + stop_ssl_handshake = true; + APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_SHUTDOWN_HOOK); while (hook) { SCOPED_MUTEX_LOCK(lock, hook->m_cont->mutex, this_ethread()); From 7680a52ffe4afd4fd9b6c70c19cfc85ee88f49cc Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 11 Feb 2019 17:20:35 -0600 Subject: [PATCH 254/526] HdrHeap: Remove pointless code and misleading comment. By design for serialization, HdrHeap has no virtual methods and therefore no virtual function pointer. --- proxy/hdrs/HdrHeap.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index 70cf6d501d7..243d6ce8a74 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -40,7 +40,6 @@ #define MAX_LOST_STR_SPACE 1024 Allocator hdrHeapAllocator("hdrHeap", HDR_HEAP_DEFAULT_SIZE); -static HdrHeap proto_heap; Allocator strHeapAllocator("hdrStrHeap", HDR_STR_HEAP_DEFAULT_SIZE); @@ -122,11 +121,6 @@ new_HdrHeap(int size) h = (HdrHeap *)ats_malloc(size); } - // Debug("hdrs", "Allocated header heap in size %d", size); - - // Patch virtual function table ptr - *((void **)h) = *((void **)&proto_heap); - h->m_size = size; h->init(); From 84d59dc083a37a8b98d446c81fc2853bb131a11d Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Thu, 7 Feb 2019 12:38:24 +0900 Subject: [PATCH 255/526] Add history to HTTP/2 --- proxy/http2/Http2ClientSession.cc | 16 ++++++++++++++++ proxy/http2/Http2ClientSession.h | 6 ++++++ proxy/http2/Http2ConnectionState.cc | 25 ++++++++++++++++++++++--- proxy/http2/Http2Stream.cc | 12 ++++++++++++ proxy/http2/Http2Stream.h | 3 +++ 5 files changed, 59 insertions(+), 3 deletions(-) diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index e6037fd17cd..7774d014398 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -25,8 +25,14 @@ #include "HttpDebugNames.h" #include "tscore/ink_base64.h" +#define REMEMBER(e, r) \ + { \ + this->remember(MakeSourceLocation(), e, r); \ + } + #define STATE_ENTER(state_name, event) \ do { \ + REMEMBER(event, this->recursion) \ SsnDebug(this, "http2_cs", "[%" PRId64 "] [%s, %s]", this->connection_id(), #state_name, \ HttpDebugNames::get_event_name(event)); \ } while (0) @@ -35,6 +41,7 @@ #define HTTP2_SET_SESSION_HANDLER(handler) \ do { \ + REMEMBER(NO_EVENT, this->recursion); \ this->session_handler = (handler); \ } while (0) @@ -65,6 +72,7 @@ Http2ClientSession::destroy() { if (!in_destroy) { in_destroy = true; + REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session destroy"); // Let everyone know we are going down do_api_callout(TS_HTTP_SSN_CLOSE_HOOK); @@ -87,6 +95,7 @@ Http2ClientSession::free() return; } + REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session free"); HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, this->mutex->thread_holding); @@ -253,6 +262,7 @@ Http2ClientSession::do_io_shutdown(ShutdownHowTo_t howto) void Http2ClientSession::do_io_close(int alerrno) { + REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session closed"); ink_assert(this->mutex->thread_holding == this_ethread()); @@ -550,3 +560,9 @@ Http2ClientSession::decrement_current_active_client_connections_stat() { HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT, this_ethread()); } + +void +Http2ClientSession::remember(const SourceLocation &location, int event, int reentrant) +{ + this->_history.push_back(location, event, reentrant); +} diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 26636fb01cc..9a2c5a103e7 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -29,6 +29,7 @@ #include "Http2ConnectionState.h" #include #include "tscore/ink_inet.h" +#include "tscore/History.h" // Name Edata Description // HTTP2_SESSION_EVENT_INIT Http2ClientSession * HTTP/2 session is born @@ -304,6 +305,9 @@ class Http2ClientSession : public ProxySession return write_buffer->max_read_avail(); } + // Record history from Http2ConnectionState + void remember(const SourceLocation &location, int event, int reentrant = NO_REENTRANT); + // noncopyable Http2ClientSession(Http2ClientSession &) = delete; Http2ClientSession &operator=(const Http2ClientSession &) = delete; @@ -333,6 +337,8 @@ class Http2ClientSession : public ProxySession IpEndpoint cached_client_addr; IpEndpoint cached_local_addr; + History _history; + // For Upgrade: h2c Http2UpgradeContext upgrade_context; diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 45689d7a262..3da2dd7fd87 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -26,8 +26,16 @@ #include "Http2ClientSession.h" #include "Http2Stream.h" #include "Http2DebugNames.h" +#include "HttpDebugNames.h" #include +#define REMEMBER(e, r) \ + { \ + if (this->ua_session) { \ + this->ua_session->remember(MakeSourceLocation(), e, r); \ + } \ + } + #define Http2ConDebug(ua_session, fmt, ...) \ SsnDebug(ua_session, "http2_con", "[%" PRId64 "] " fmt, ua_session->connection_id(), ##__VA_ARGS__); @@ -877,6 +885,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) case HTTP2_SESSION_EVENT_INIT: { ink_assert(this->ua_session == nullptr); this->ua_session = (Http2ClientSession *)edata; + REMEMBER(event, this->recursion); // [RFC 7540] 3.5. HTTP/2 Connection Preface. Upon establishment of a TCP connection and // determination that HTTP/2 will be used by both peers, each endpoint MUST @@ -902,6 +911,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Finalize HTTP/2 Connection case HTTP2_SESSION_EVENT_FINI: { SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + REMEMBER(event, this->recursion); ink_assert(this->fini_received == false); this->fini_received = true; @@ -911,6 +921,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) } break; case HTTP2_SESSION_EVENT_XMIT: { + REMEMBER(event, this->recursion); SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); send_data_frames_depends_on_priority(); _scheduled = false; @@ -918,6 +929,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Parse received HTTP/2 frames case HTTP2_SESSION_EVENT_RECV: { + REMEMBER(event, this->recursion); const Http2Frame *frame = (Http2Frame *)edata; const Http2StreamId stream_id = frame->header().streamid; Http2Error error; @@ -964,6 +976,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Initiate a gracefull shutdown case HTTP2_SESSION_EVENT_SHUTDOWN_INIT: { + REMEMBER(event, this->recursion); ink_assert(shutdown_state == HTTP2_SHUTDOWN_NOT_INITIATED); shutdown_state = HTTP2_SHUTDOWN_INITIATED; // [RFC 7540] 6.8. GOAWAY @@ -977,6 +990,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Continue a gracefull shutdown case HTTP2_SESSION_EVENT_SHUTDOWN_CONT: { + REMEMBER(event, this->recursion); ink_assert(shutdown_state == HTTP2_SHUTDOWN_INITIATED); shutdown_cont_event = nullptr; shutdown_state = HTTP2_SHUTDOWN_IN_PROGRESS; @@ -1010,8 +1024,10 @@ Http2ConnectionState::main_event_handler(int event, void *edata) } int -Http2ConnectionState::state_closed(int /* event */, void *edata) +Http2ConnectionState::state_closed(int event, void *edata) { + REMEMBER(event, this->recursion); + if (edata == zombie_event) { // Zombie session is still around. Assert! ink_release_assert(zombie_event == nullptr); @@ -1195,6 +1211,7 @@ Http2ConnectionState::delete_stream(Http2Stream *stream) } Http2StreamDebug(ua_session, stream->get_id(), "Delete stream"); + REMEMBER(NO_EVENT, this->recursion); if (Http2::stream_priority_enabled) { Http2DependencyTree::Node *node = stream->priority_node; @@ -1236,6 +1253,8 @@ Http2ConnectionState::delete_stream(Http2Stream *stream) void Http2ConnectionState::release_stream(Http2Stream *stream) { + REMEMBER(NO_EVENT, this->recursion) + if (stream) { // Decrement total_client_streams_count here, because it's a counter include streams in the process of shutting down. // Other counters (client_streams_in_count/client_streams_out_count) are already decremented in delete_stream(). @@ -1770,6 +1789,8 @@ Http2ConnectionState::send_ping_frame(Http2StreamId id, uint8_t flag, const uint void Http2ConnectionState::send_goaway_frame(Http2StreamId id, Http2ErrorCode ec) { + ink_assert(this->ua_session != nullptr); + Http2ConDebug(ua_session, "Send GOAWAY frame, last_stream_id: %d", id); if (ec != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { @@ -1779,8 +1800,6 @@ Http2ConnectionState::send_goaway_frame(Http2StreamId id, Http2ErrorCode ec) Http2Frame frame(HTTP2_FRAME_TYPE_GOAWAY, 0, 0); Http2Goaway goaway; - ink_assert(this->ua_session != nullptr); - goaway.last_streamid = id; goaway.error_code = ec; diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 429b55fdc5f..fc6e3b40738 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -26,6 +26,11 @@ #include "Http2ClientSession.h" #include "../http/HttpSM.h" +#define REMEMBER(e, r) \ + { \ + this->_history.push_back(MakeSourceLocation(), e, r); \ + } + #define Http2StreamDebug(fmt, ...) \ SsnDebug(parent, "http2_stream", "[%" PRId64 "] [%u] " fmt, parent->connection_id(), this->get_id(), ##__VA_ARGS__); @@ -35,6 +40,7 @@ int Http2Stream::main_event_handler(int event, void *edata) { SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + REMEMBER(event, this->reentrancy_count); if (!this->_switch_thread_if_not_on_right_thread(event, edata)) { // Not on the right thread @@ -320,6 +326,7 @@ Http2Stream::do_io_close(int /* flags */) super::release(nullptr); if (!closed) { + REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("do_io_close"); // When we get here, the SM has initiated the shutdown. Either it received a WRITE_COMPLETE, or it is shutting down. Any @@ -371,6 +378,8 @@ void Http2Stream::terminate_if_possible() { if (terminate_stream && reentrancy_count == 0) { + REMEMBER(NO_EVENT, this->reentrancy_count); + Http2ClientSession *h2_parent = static_cast(parent); SCOPED_MUTEX_LOCK(lock, h2_parent->connection_state.mutex, this_ethread()); h2_parent->connection_state.delete_stream(this); @@ -384,6 +393,7 @@ Http2Stream::initiating_close() { if (!closed) { SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("initiating_close"); // Set the state of the connection to closed @@ -452,6 +462,7 @@ Http2Stream::send_tracked_event(Event *event, int send_event, VIO *vio) } if (event == nullptr) { + REMEMBER(send_event, this->reentrancy_count); event = this_ethread()->schedule_imm(this, send_event, vio); } @@ -705,6 +716,7 @@ Http2Stream::reenable(VIO *vio) void Http2Stream::destroy() { + REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("Destroy stream, sent %" PRIu64 " bytes", this->bytes_sent); SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); // Clean up after yourself if this was an EOS diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index e766ced28df..22030f4bff6 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -28,6 +28,7 @@ #include "Http2DebugNames.h" #include "../http/HttpTunnel.h" // To get ChunkedHandler #include "Http2DependencyTree.h" +#include "tscore/History.h" class Http2Stream; class Http2ConnectionState; @@ -245,6 +246,8 @@ class Http2Stream : public ProxyTransaction VIO read_vio; VIO write_vio; + History _history; + bool trailing_header = false; bool body_done = false; bool chunked = false; From f5534ba5581e09ebe3e759dc04c9dda23b75a741 Mon Sep 17 00:00:00 2001 From: scw00 Date: Thu, 24 Jan 2019 17:43:37 +0800 Subject: [PATCH 256/526] Assert when sm callback to wrong thread --- src/traffic_server/InkAPI.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index cbe88ce9397..5b63750b6e6 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -5928,13 +5928,14 @@ TSHttpTxnReenable(TSHttpTxn txnp, TSEvent event) // If this function is being executed on a thread created by the API // which is DEDICATED, the continuation needs to be called back on a // REGULAR thread. - if (eth == nullptr || eth->tt != REGULAR) { + if (eth == nullptr || eth->tt != REGULAR || !eth->is_event_type(ET_NET)) { eventProcessor.schedule_imm(new TSHttpSMCallback(sm, event), ET_NET); } else { MUTEX_TRY_LOCK(trylock, sm->mutex, eth); if (!trylock.is_locked()) { eventProcessor.schedule_imm(new TSHttpSMCallback(sm, event), ET_NET); } else { + ink_assert(eth->is_event_type(ET_NET)); sm->state_api_callback((int)event, nullptr); } } From e9888197392a261ac6305255fb3a67b7c515b5b3 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 12 Feb 2019 14:49:43 +0900 Subject: [PATCH 257/526] Fix client connections of traffic_top Prior this change, traffic_top used HTTP/1.x specific metrics for Req/Conn, New Conn, Curr Conn, and Active Con in CLIENT table. --- src/traffic_top/stats.h | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/traffic_top/stats.h b/src/traffic_top/stats.h index 7dc3d9773c7..ffec58c0780 100644 --- a/src/traffic_top/stats.h +++ b/src/traffic_top/stats.h @@ -111,12 +111,33 @@ class Stats lookup_table.insert(make_pair("dns_entry", LookupItem("DNS Entry", "proxy.process.hostdb.cache.current_items", 1))); lookup_table.insert(make_pair("dns_hits", LookupItem("DNS Hits", "proxy.process.hostdb.total_hits", 2))); lookup_table.insert(make_pair("dns_lookups", LookupItem("DNS Lookups", "proxy.process.hostdb.total_lookups", 2))); + + // Incoming HTTP/1.1 and HTTP/2 connections - some metrics are HTTP version specific lookup_table.insert(make_pair("client_req", LookupItem("Requests", "proxy.process.http.incoming_requests", 2))); - lookup_table.insert(make_pair("client_conn", LookupItem("New Conn", "proxy.process.http.total_client_connections", 2))); + + // total_client_connections + lookup_table.insert( + make_pair("client_conn_h1", LookupItem("New Conn HTTP/1.x", "proxy.process.http.total_client_connections", 2))); + lookup_table.insert( + make_pair("client_conn_h2", LookupItem("New Conn HTTP/2", "proxy.process.http2.total_client_connections", 2))); + lookup_table.insert(make_pair("client_conn", LookupItem("New Conn", "client_conn_h1", "client_conn_h2", 6))); + + // requests / connections lookup_table.insert(make_pair("client_req_conn", LookupItem("Req/Conn", "client_req", "client_conn", 3))); - lookup_table.insert(make_pair("client_curr_conn", LookupItem("Curr Conn", "proxy.process.http.current_client_connections", 1))); + + // current_client_connections + lookup_table.insert( + make_pair("client_curr_conn_h1", LookupItem("Curr Conn HTTP/1.x", "proxy.process.http.current_client_connections", 1))); lookup_table.insert( - make_pair("client_actv_conn", LookupItem("Active Con", "proxy.process.http.current_active_client_connections", 1))); + make_pair("client_curr_conn_h2", LookupItem("Curr Conn HTTP/2", "proxy.process.http2.current_client_connections", 1))); + lookup_table.insert(make_pair("client_curr_conn", LookupItem("Curr Conn", "client_curr_conn_h1", "client_curr_conn_h2", 6))); + + // current_active_client_connections + lookup_table.insert(make_pair("client_actv_conn_h1", + LookupItem("Active Con HTTP/1.x", "proxy.process.http.current_active_client_connections", 1))); + lookup_table.insert(make_pair("client_actv_conn_h2", + LookupItem("Active Con HTTP/2", "proxy.process.http2.current_active_client_connections", 1))); + lookup_table.insert(make_pair("client_actv_conn", LookupItem("Active Con", "client_actv_conn_h1", "client_actv_conn_h2", 6))); lookup_table.insert(make_pair("server_req", LookupItem("Requests", "proxy.process.http.outgoing_requests", 2))); lookup_table.insert(make_pair("server_conn", LookupItem("New Conn", "proxy.process.http.total_server_connections", 2))); From 840f4b8412d3ac55d47e5f1b3a5836e37810a730 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 8 Feb 2019 09:16:42 -0600 Subject: [PATCH 258/526] Scalar: add generic rounding. --- include/tscore/Scalar.h | 38 ++++++++++++++++++++-------- src/tscore/unit_tests/test_Scalar.cc | 18 +++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/include/tscore/Scalar.h b/include/tscore/Scalar.h index e71a0824da0..06c339832e0 100644 --- a/include/tscore/Scalar.h +++ b/include/tscore/Scalar.h @@ -44,19 +44,19 @@ template class Scalar; namespace detail { - // @internal - althought these conversion methods look bulky, in practice they compile down to - // very small amounts of code due to dead code elimination and that all of the conditions are - // compile time constants. + // @internal - although these conversion methods look bulky, in practice they compile down to + // very small amounts of code due to the conditions being compile time constant - the non-taken + // clauses are dead code and eliminated by the compiler. // The general case where neither N nor S are a multiple of the other seems a bit long but this // minimizes the risk of integer overflow. I need to validate that under -O2 the compiler will // only do 1 division to get both the quotient and remainder for (n/N) and (n%N). In cases where // N,S are powers of 2 I have verified recent GNU compilers will optimize to bit operations. - /// Convert a count @a c that is scale @s S to scale @c N - template - intmax_t - scale_conversion_round_up(intmax_t c) + /// Convert a count @a c that is scale @s S to the corresponding count for scale @c N + template + C + scale_conversion_round_up(C c) { typedef std::ratio R; if (N == S) { @@ -71,9 +71,9 @@ namespace detail } /// Convert a count @a c that is scale @s S to scale @c N - template - intmax_t - scale_conversion_round_down(intmax_t c) + template + C + scale_conversion_round_down(C c) { typedef std::ratio R; if (N == S) { @@ -108,6 +108,10 @@ namespace detail Much of this is driven by the fact that the assignment operator, in some cases, can not be templated and therefore to have a nice interface for assignment this split is needed. + + Note - the key point is the actual conversion is not done when the wrapper instance is created + but when the wrapper instance is assigned. That is what enables the conversion to be done in + the context of the destination, which is not otherwise possible. */ // Unit value, to be rounded up. @@ -893,6 +897,20 @@ Scalar::minus(Counter n) const -> self return {_n - n}; } +template +C +round_up(C value) +{ + return N * detail::scale_conversion_round_up(value); +} + +template +C +round_down(C value) +{ + return N * detail::scale_conversion_round_down(value); +} + namespace detail { // These classes exist only to create distinguishable overloads. diff --git a/src/tscore/unit_tests/test_Scalar.cc b/src/tscore/unit_tests/test_Scalar.cc index 3f8ac69ba9b..6e8956054ac 100644 --- a/src/tscore/unit_tests/test_Scalar.cc +++ b/src/tscore/unit_tests/test_Scalar.cc @@ -76,6 +76,24 @@ TEST_CASE("Scalar", "[libts][Scalar]") sz_b = sz; // Should be OK because SCALE_1 is an integer multiple of SCALE_2 // sz = sz_b; // Should not compile. REQUIRE(sz_b.count() == 119 * (SCALE_1 / SCALE_2)); + + // Test generic rounding. + REQUIRE(120 == ts::round_up<10>(118)); + REQUIRE(120 == ts::round_up<10>(120)); + REQUIRE(130 == ts::round_up<10>(121)); + + REQUIRE(110 == ts::round_down<10>(118)); + REQUIRE(120 == ts::round_down<10>(120)); + REQUIRE(120 == ts::round_down<10>(121)); + + REQUIRE(1200 == ts::round_up<100>(1108)); + REQUIRE(1200 == ts::round_up<100>(1200)); + REQUIRE(1300 == ts::round_up<100>(1201)); + + REQUIRE(100 == ts::round_down<100>(118)); + REQUIRE(1100 == ts::round_down<100>(1108)); + REQUIRE(1200 == ts::round_down<100>(1200)); + REQUIRE(1200 == ts::round_down<100>(1208)); } TEST_CASE("Scalar Factors", "[libts][Scalar][factors]") From 0193bd11c162ce6289ff2437c558ea8cf3593092 Mon Sep 17 00:00:00 2001 From: dyrock Date: Tue, 12 Feb 2019 20:43:37 +0000 Subject: [PATCH 259/526] Added relative path to tls_client_verify test curl commands to avoid confusion with NSS database nickname. --- .../gold_tests/tls/tls_client_verify.test.py | 16 ++++++------- .../gold_tests/tls/tls_client_verify2.test.py | 24 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py index 9860955df65..e09ddabbdee 100644 --- a/tests/gold_tests/tls/tls_client_verify.test.py +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -89,7 +89,7 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) # Should fail with badly signed certs tr.Processes.Default.ReturnCode = 35 @@ -98,7 +98,7 @@ tr.Setup.Copy("ssl/signed-foo.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-foo.pem --key signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -114,7 +114,7 @@ tr.Setup.Copy("ssl/signed-bar.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-bar.pem --key signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-bar.pem --key ./signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -123,7 +123,7 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -139,7 +139,7 @@ tr.Setup.Copy("ssl/signed-foo.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-foo.pem --key signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-foo.pem --key ./signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -148,7 +148,7 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -163,7 +163,7 @@ tr.Setup.Copy("ssl/signed-bar.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") @@ -172,7 +172,7 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 diff --git a/tests/gold_tests/tls/tls_client_verify2.test.py b/tests/gold_tests/tls/tls_client_verify2.test.py index 885e75f7ae5..cf0286c598d 100644 --- a/tests/gold_tests/tls/tls_client_verify2.test.py +++ b/tests/gold_tests/tls/tls_client_verify2.test.py @@ -90,7 +90,7 @@ tr.Setup.Copy("ssl/signed-foo.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-foo.pem --key signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -98,14 +98,14 @@ tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) -tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bob.bar.com with cert") tr.Setup.Copy("ssl/signed-bob-bar.pem") tr.Setup.Copy("ssl/signed-bar.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-bar.pem --key signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-bar.pem --key ./signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") @@ -114,21 +114,21 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) -tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bob.foo.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) -tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bob.foo.com with cert") tr.Setup.Copy("ssl/signed-bob-foo.pem") tr.Setup.Copy("ssl/signed-foo.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bob-foo.pem --key signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-foo.pem --key ./signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") @@ -137,14 +137,14 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) -tr.Processes.Default.ReturnCode = 35 +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) -tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") tr = Test.AddTestRun("Connect to bar.com with cert") @@ -152,7 +152,7 @@ tr.Setup.Copy("ssl/signed-bar.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert signed-bar.pem --key signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -161,7 +161,7 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert server.pem --key server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert unknown ca", "TLS handshake should succeed") From 19e9776877e005191b3349b0fe1ea1380f850388 Mon Sep 17 00:00:00 2001 From: dyrock Date: Tue, 12 Feb 2019 21:01:39 +0000 Subject: [PATCH 260/526] Adjust tls_client_versions to be more resilient for different curl versions --- tests/gold_tests/tls/tls_client_versions.test.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/gold_tests/tls/tls_client_versions.test.py b/tests/gold_tests/tls/tls_client_versions.test.py index 54de7086628..91eea68fef1 100644 --- a/tests/gold_tests/tls/tls_client_versions.test.py +++ b/tests/gold_tests/tls/tls_client_versions.test.py @@ -34,7 +34,7 @@ ts = Test.MakeATSProcess("ts", select_ports=False) server = Test.MakeOriginServer("server", ssl=True) -request_foo_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_foo_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "foo ok"} server.addResponse("sessionlog.json", request_foo_header, response_foo_header) @@ -66,7 +66,7 @@ }) # foo.com should only offer the older TLS protocols -# bar.com should terminate. +# bar.com should terminate. # empty SNI should tunnel to server_bar ts.Disk.ssl_server_name_yaml.AddLines([ '- fqdn: foo.com', @@ -78,9 +78,8 @@ tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) -tr.ReturnCode = 35 +tr.ReturnCode = 35 tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("ssl_choose_client_version:unsupported protocol", "Should not allow TLSv1_2") # Target foo.com for TLSv1. Should succeed tr = Test.AddTestRun("foo.com TLSv1") @@ -91,9 +90,8 @@ # Target bar.com for TLSv1. Should fail tr = Test.AddTestRun("bar.com TLSv1") tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) -tr.ReturnCode = 35 +tr.ReturnCode = 35 tr.StillRunningAfter = ts -tr.Processes.Default.Streams.All += Testers.ContainsExpression("alert protocol version", "Should not allow TLSv1_0") # Target bar.com for TLSv1_2. Should succeed tr = Test.AddTestRun("bar.com TLSv1_2") From 338a04fa60c25b89ea0584bed8d371e899ff310f Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 4 Feb 2019 09:10:30 -0600 Subject: [PATCH 261/526] Doc: TSHttpTxnServerReqGet --- .../functions/TSHttpTxnServerReqGet.en.rst | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst index ad222513d2a..f9cb31e6a85 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst @@ -26,7 +26,28 @@ Synopsis `#include ` -.. function:: TSReturnCode TSHttpTxnServerReqGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * offset) +.. function:: TSReturnCode TSHttpTxnServerReqGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * obj) Description =========== + +Get the request |TS| is sending to the upstream (server) for the transaction :arg:`txnp`. +:arg:`bufp` and :arg:`obj` should be valid pointers to use as return values. The call site could +look something like :: + + TSMBuffer mbuffer; + TSMLoc mloc; + if (TS_SUCCESS == TSHttpTxnServerReqGet(&mbuffer, &mloc)) { + /* Can use safely mbuffer, mloc for subsequent API calls */ + } else { + /* mbuffer, mloc in an undefined state */ + } + +This call returns :c:macro:`TS_SUCCESS` on success, and :c:macro:`TS_ERROR` on failure. It is the +caller's responsibility to see that :arg:`txnp` is a valid transaction. + +Once the request object is obtained, it can be used to access all of the elements of the request, +such as the URL, the header fields, etc. This is also the mechanism by which a plugin can change the +upstream request, if done before the request is sent (in or before +:c:macro:`TS_HTTP_SEND_REQUEST_HDR_HOOK`). Note that for earlier hooks, the request may not yet +exist, in which case an error is returned. From 40ee180af44d49e3eb0cc76577455d883df6c89e Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 12 Feb 2019 13:13:46 -0700 Subject: [PATCH 262/526] Fixed a typo in the background_fetch plugin section --- doc/admin-guide/plugins/background_fetch.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin-guide/plugins/background_fetch.en.rst b/doc/admin-guide/plugins/background_fetch.en.rst index 3277e1fcc8f..2f62c19f7f4 100644 --- a/doc/admin-guide/plugins/background_fetch.en.rst +++ b/doc/admin-guide/plugins/background_fetch.en.rst @@ -54,7 +54,7 @@ of the original (Client) request under these conditions: - The response is a ``206`` response - The original client request, and the Origin server response, is clearly indicating that the response is cacheable. This uses the new API - c:func:`TSHttpTxnIsCacheable()`, which also implies honoring current + :c:func:`TSHttpTxnIsCacheable()`, which also implies honoring current Traffic Server configurations. From 42ca94353919aa1f45b0577a97dc3997e39c4818 Mon Sep 17 00:00:00 2001 From: scw00 Date: Mon, 28 Jan 2019 15:35:08 +0800 Subject: [PATCH 263/526] Avoid ats_malloc in unmarshal --- proxy/hdrs/HTTP.cc | 44 +++++++++----------------------------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 6c4187785af..4704bfd4925 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -1991,7 +1991,6 @@ HTTPInfo::marshal_length() } if (m_alt->m_frag_offset_count > HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS) { - len -= sizeof(m_alt->m_integral_frag_offsets); len += sizeof(FragOffset) * m_alt->m_frag_offset_count; } @@ -2006,23 +2005,11 @@ HTTPInfo::marshal(char *buf, int len) HTTPCacheAlt *marshal_alt = (HTTPCacheAlt *)buf; // non-zero only if the offsets are external. Otherwise they get // marshalled along with the alt struct. - int frag_len = (0 == m_alt->m_frag_offset_count || m_alt->m_frag_offsets == m_alt->m_integral_frag_offsets) ? - 0 : - sizeof(HTTPCacheAlt::FragOffset) * m_alt->m_frag_offset_count; - ink_assert(m_alt->m_magic == CACHE_ALT_MAGIC_ALIVE); // Make sure the buffer is aligned // ink_assert(((intptr_t)buf) & 0x3 == 0); - // If we have external fragment offsets, copy the initial ones - // into the integral data. - if (frag_len) { - memcpy(m_alt->m_integral_frag_offsets, m_alt->m_frag_offsets, sizeof(m_alt->m_integral_frag_offsets)); - frag_len -= sizeof(m_alt->m_integral_frag_offsets); - // frag_len should never be non-zero at this point, as the offsets - // should be external only if too big for the internal table. - } // Memcpy the whole object so that we can use it // live later. This involves copying a few // extra bytes now but will save copying any @@ -2035,13 +2022,14 @@ HTTPInfo::marshal(char *buf, int len) buf += HTTP_ALT_MARSHAL_SIZE; used += HTTP_ALT_MARSHAL_SIZE; - if (frag_len > 0) { + if (m_alt->m_frag_offset_count > HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS) { marshal_alt->m_frag_offsets = static_cast(reinterpret_cast(used)); - memcpy(buf, m_alt->m_frag_offsets + HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS, frag_len); - buf += frag_len; - used += frag_len; + memcpy(buf, m_alt->m_frag_offsets, m_alt->m_frag_offset_count * sizeof(FragOffset)); + buf += m_alt->m_frag_offset_count * sizeof(FragOffset); + used += m_alt->m_frag_offset_count * sizeof(FragOffset); } else { - marshal_alt->m_frag_offsets = nullptr; + // the data stored in intergral buffer + m_alt->m_frag_offsets = nullptr; } // The m_{request,response}_hdr->m_heap pointers are converted @@ -2098,23 +2086,9 @@ HTTPInfo::unmarshal(char *buf, int len, RefCountObj *block_ref) len -= HTTP_ALT_MARSHAL_SIZE; if (alt->m_frag_offset_count > HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS) { - // stuff that didn't fit in the integral slots. - int extra = sizeof(FragOffset) * alt->m_frag_offset_count - sizeof(alt->m_integral_frag_offsets); - char *extra_src = buf + reinterpret_cast(alt->m_frag_offsets); - // Actual buffer size, which must be a power of two. - // Well, technically not, because we never modify an unmarshalled fragment - // offset table, but it would be a nasty bug should that be done in the - // future. - int bcount = HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS * 2; - - while (bcount < alt->m_frag_offset_count) { - bcount *= 2; - } - alt->m_frag_offsets = - static_cast(ats_malloc(bcount * sizeof(FragOffset))); // WRONG - must round up to next power of 2. - memcpy(alt->m_frag_offsets, alt->m_integral_frag_offsets, sizeof(alt->m_integral_frag_offsets)); - memcpy(alt->m_frag_offsets + HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS, extra_src, extra); - len -= extra; + alt->m_frag_offsets = reinterpret_cast(buf + reinterpret_cast(alt->m_frag_offsets)); + len -= sizeof(FragOffset) * alt->m_frag_offset_count; + ink_assert(len >= 0); } else if (alt->m_frag_offset_count > 0) { alt->m_frag_offsets = alt->m_integral_frag_offsets; } else { From acde206524104c5aa19aded347923f4f513ad87a Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 6 Feb 2019 20:48:03 +0000 Subject: [PATCH 264/526] Add a case to the tls_client_cert2 to exercise one file with cert and key --- tests/gold_tests/tls/ssl/combo-signed-foo.pem | 47 +++++++++++++++++++ tests/gold_tests/tls/tls_client_cert2.test.py | 4 +- 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 tests/gold_tests/tls/ssl/combo-signed-foo.pem diff --git a/tests/gold_tests/tls/ssl/combo-signed-foo.pem b/tests/gold_tests/tls/ssl/combo-signed-foo.pem new file mode 100644 index 00000000000..e3bf4cd29f1 --- /dev/null +++ b/tests/gold_tests/tls/ssl/combo-signed-foo.pem @@ -0,0 +1,47 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAnQCCQC81MtBCwmQtzANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTgxMDE1MTU1NjMzWhcNMjgxMDEyMTU1NjMzWjByMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxEjAQBgNVBAcMCUNoYW1wYWlnbjEQMA4GA1UECgwHRXhh +bXBsZTEQMA4GA1UEAwwHZm9vLmNvbTEeMBwGCSqGSIb3DQEJARYPYm9iQGV4YW1w +bGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyYXjrK9KOAtE +FXpaPL4mJfyI1r5I+GOmKI5zvxPU+R0n1sZEYpEU/F+8qrPQpa4zXGKDzTUD+b4J +dagCHosVK22WZJXGd5BxfzF3c7mFf/7k92H1q7Dk3X23LumnR7Qa/0HNMPRkmwaa +eNmQC8c42doWWaudV5ir3M+ef4Jv/WL5RhK877D85Ho3R+lNRini6hmmTqpFezdi +eMypicoj88K5kf/Mu5PvYwx0F/gNsGuYGogDBnSDPGk1fl7DTQL3rKb18l+1IcMQ +7MNHq9Bi7+LLGAq7uYRfrVHI3jgh8UpwqBAuOW9sw2RwEyy46+wCCedk3EqNZE2k +4qwDgIh0SwIDAQABMA0GCSqGSIb3DQEBCwUAA4GBALr2gm+KgveEcTXwURM0wxJC +m0yOR8w6MX8fxHKaekhJH1U84G64Ub0gbn2beOdLBQkG+4czLiOOOgyeukPaJJ81 +od2ooE7DrGUPGnbHYxW/70EtVF5nQEctcqpKNF/d04mVKrqI90919MJSxJ5KedHK +2H11+gUPwDWy/mAwJzEJ +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDJheOsr0o4C0QV +elo8viYl/IjWvkj4Y6YojnO/E9T5HSfWxkRikRT8X7yqs9ClrjNcYoPNNQP5vgl1 +qAIeixUrbZZklcZ3kHF/MXdzuYV//uT3YfWrsOTdfbcu6adHtBr/Qc0w9GSbBpp4 +2ZALxzjZ2hZZq51XmKvcz55/gm/9YvlGErzvsPzkejdH6U1GKeLqGaZOqkV7N2J4 +zKmJyiPzwrmR/8y7k+9jDHQX+A2wa5gaiAMGdIM8aTV+XsNNAvespvXyX7UhwxDs +w0er0GLv4ssYCru5hF+tUcjeOCHxSnCoEC45b2zDZHATLLjr7AIJ52TcSo1kTaTi +rAOAiHRLAgMBAAECggEAB7fXBnAYOZlE3EW5WwY1U9MeMotLJCg83uTFzhWmXHwf +YHxrdhL0aM4J3cfRP+cyFGG5hox3QINkvVrX6e+NugISdnu+BCpGDocIeigq0sIi +Zs8bp524xjrgXy2XuIlPV2NfxnY1vDI+jE5Y0/qnVMCjhn+qIQa53lUdTujh/SRR +3U7di+QMK4mdGwRnInos++ENy33A+2LqtUK8i0ERkzPFa1yMQEE4DOFPzZcW+jhK +arvzBwPIn37PZmL5oyiQB1YiGPGt4XNfPBwACTMYM8LlYBfEBHG77k3bMtUf0WqE +GctoT5SIe5+YbyrWkpfHgoKPxggH3I3TrFnVvqrKQQKBgQDmcV1YbuNEQLeif521 +iGqMgPQYmnpO6k27RsZrM9ikhIgm9bVJsOqnaYzQFeSfJ3eNLXYUL6IF8g46xddw +fDBtrEjDAA9OUkNRcizbeKF+GJRMtX11d4ZNbnG1wyMZYkArZGfraZBLHPEF1pya +2iFdVfokQCBpLmX7BMQEPePyuwKBgQDf33H9njf9oO0l9GfuWDvSoaV8GwqV9x55 +sFjggQYD/xqwEprrzr524X5Y2ZiTUpBu+kqqM8GYfm3bzBKkZU1rnjwxADUwBw8U +L2U/Z7Id3om8tAdzHOSI9d7mxWA8uTsScMm0IFv2l/XBQo1+AAJSD03pcsabr4Lf +SuJGmoFTsQKBgQDVzPASEC+DL5gwh75Gop5YZXwTJ5+6f+BGlM+avquNV/kKTIU6 +LY5IbMFcfjNzBicBMOCQsfDdG0rgdJYBovc7idCoOvH4dJJIimnb5fvPBfbxhKE1 +zwMn7ARL4xQ5hNKMb8eKvpJFXkCwbgE2GpNCCXbfEy/+5jFvx2gll1ZZ6QKBgG3J +OzJ/w796irHBQLKOzI+HvAq3jCJs9KICjCNUwql1EhZkmVqooZjVDkvuMbeVlsUF +s1XyWa852RAf7Mh38VakW6pACtVJsOhaMdG9PYkOWAeVVc3qzlwoDy6mfoJo6AIs +E45lDBRLAzbKN28h/AFYBgJEygcRNCHirEKphGCRAoGAEhcaxbmMo2fHBYuvOR1Q +ZAIq1EPvysDROUBHhdTJqN1wHsuJsmVJxX42+YHcZdjtgeCdjU3HMoyCnTaRxDee +K3VeB4PobN1WpQwFklFoqcvAhW6eicdZXme7ktK120NPQsXrmjgN6Lfg3PNjosn0 +tqSxQhQ4DrSf60fxx0/M/rw= +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py index a2411362cf5..05ee893b28e 100644 --- a/tests/gold_tests/tls/tls_client_cert2.test.py +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -55,6 +55,7 @@ ts.addSSLfile("ssl/server.pem") ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/combo-signed-foo.pem") ts.addSSLfile("ssl/signed-foo.pem") ts.addSSLfile("ssl/signed-foo.key") ts.addSSLfile("ssl/signed2-foo.pem") @@ -91,8 +92,7 @@ ' client_cert: signed-bar.pem', ' client_key: signed-bar.key', '- fqdn: bob.*.com', - ' client_cert: {0}/signed-foo.pem'.format(ts.Variables.SSLDir), - ' client_key: {0}/signed-foo.key'.format(ts.Variables.SSLDir), + ' client_cert: {0}/combo-signed-foo.pem'.format(ts.Variables.SSLDir), '- fqdn: "*bar.com"', ' client_cert: {0}/signed2-bar.pem'.format(ts.Variables.SSLDir), ' client_key: {0}/signed-bar.key'.format(ts.Variables.SSLDir), From d3a47a0f8b5d484dbdb58bfedfde6227c950d7e7 Mon Sep 17 00:00:00 2001 From: dyrock Date: Tue, 12 Feb 2019 21:35:29 +0000 Subject: [PATCH 265/526] tls_check_cert_selection test: remove HTTP2 checking; remove extra spaces; use relative path --- .../tls/tls_check_cert_selection.test.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/gold_tests/tls/tls_check_cert_selection.test.py b/tests/gold_tests/tls/tls_check_cert_selection.test.py index f1eaa8fde94..5f2174bd26e 100644 --- a/tests/gold_tests/tls/tls_check_cert_selection.test.py +++ b/tests/gold_tests/tls/tls_check_cert_selection.test.py @@ -76,7 +76,7 @@ tr = Test.AddTestRun("bar.com cert") tr.Setup.Copy("ssl/signer.pem") tr.Setup.Copy("ssl/signer2.pem") -tr.Processes.Default.Command = "curl -v --cacert signer2.pem --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v --cacert ./signer2.pem --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(dns) @@ -84,20 +84,20 @@ tr.StillRunningAfter = server tr.StillRunningAfter = ts tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=bar.com", "Cert should contain bar.com") -tr.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=foo.com", "Cert should not contain foo.com") -tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN=bar.com", "Cert should contain bar.com") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Cert should not contain foo.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("404", "Should make an exchange") # Should receive a foo.com cert tr2 = Test.AddTestRun("foo.com cert") -tr2.Processes.Default.Command = "curl -v --cacert signer.pem --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v --cacert ./signer.pem --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=foo.com", "Cert should contain foo.com") -tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=bar.com", "Cert should not contain bar.com") -tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Cert should contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("404", "Should make an exchange") # Should receive random.server.com tr2 = Test.AddTestRun("random.server.com cert") @@ -106,20 +106,20 @@ tr2.StillRunningAfter = server tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=random.server.com", "Cert should contain random.server.com") -tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=foo.com", "Cert should not contain foo.com") -tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=bar.com", "Cert should not contain bar.com") -tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=random.server.com", "Cert should contain random.server.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Cert should not contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("404", "Should make an exchange") # No SNI match should match specific IP address, foo.com # SNI name and returned cert name will not match, so must use -k to avoid cert verification tr2 = Test.AddTestRun("Bad SNI") -tr2.Processes.Default.Command = "curl -v -k --cacert signer.pem --resolve 'bad.sni.com:{0}:127.0.0.1' https://bad.sni.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k --cacert ./signer.pem --resolve 'bad.sni.com:{0}:127.0.0.1' https://bad.sni.com:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.Processes.Default.Streams.All += Testers.ContainsExpression(" CN=foo.com", "Cert should contain foo.com") -tr2.Processes.Default.Streams.All += Testers.ExcludesExpression(" CN=bar.com", "Cert should not contain bar.com") -tr.Processes.Default.Streams.All += Testers.ContainsExpression(" HTTP/2 404", "Should make an exchange") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Cert should contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("404", "Should make an exchange") From 985894ddae2377e3f0178d01e82948eb807987a5 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 11 Feb 2019 18:48:16 +0000 Subject: [PATCH 266/526] Remove header set for uninitialized response. --- proxy/http/HttpSM.cc | 8 ++++---- proxy/http/HttpSM.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index b2c9cb9e405..462febb33af 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -5592,7 +5592,7 @@ HttpSM::setup_transform_to_server_transfer() } void -HttpSM::do_drain_request_body() +HttpSM::do_drain_request_body(HTTPHdr &response) { int64_t content_length = t_state.hdr_info.client_request.get_content_length(); int64_t avail = ua_buffer_reader->read_avail(); @@ -5617,7 +5617,7 @@ HttpSM::do_drain_request_body() close_connection: t_state.client_info.keep_alive = HTTP_NO_KEEPALIVE; - t_state.hdr_info.client_response.value_set(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION, "close", 5); + response.value_set(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION, "close", 5); } void @@ -7417,11 +7417,10 @@ HttpSM::set_next_state() release_server_session(true); t_state.source = HttpTransact::SOURCE_CACHE; - do_drain_request_body(); - if (transform_info.vc) { ink_assert(t_state.hdr_info.client_response.valid() == 0); ink_assert((t_state.hdr_info.transform_response.valid() ? true : false) == true); + do_drain_request_body(t_state.hdr_info.transform_response); t_state.hdr_info.cache_response.create(HTTP_TYPE_RESPONSE); t_state.hdr_info.cache_response.copy(&t_state.hdr_info.transform_response); @@ -7430,6 +7429,7 @@ HttpSM::set_next_state() tunnel.tunnel_run(p); } else { ink_assert((t_state.hdr_info.client_response.valid() ? true : false) == true); + do_drain_request_body(t_state.hdr_info.client_response); t_state.hdr_info.cache_response.create(HTTP_TYPE_RESPONSE); t_state.hdr_info.cache_response.copy(&t_state.hdr_info.client_response); diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index f2144b96c30..11a40e20a4f 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -479,7 +479,7 @@ class HttpSM : public Continuation void do_api_callout_internal(); void do_redirect(); void redirect_request(const char *redirect_url, const int redirect_len); - void do_drain_request_body(); + void do_drain_request_body(HTTPHdr &response); void wait_for_full_body(); From 8213ca0293f5b47b0291c0eb3ae083afe1457004 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 7 Feb 2019 18:00:18 -0600 Subject: [PATCH 267/526] HdrHeap: refresh for C++17. --- .../core-architecture/heap.en.rst | 13 ++- iocore/cache/Cache.cc | 12 +-- proxy/hdrs/HTTP.cc | 2 +- proxy/hdrs/HdrHeap.cc | 80 +++++++++---------- proxy/hdrs/HdrHeap.h | 56 ++++++------- proxy/hdrs/HdrTSOnly.cc | 25 +++--- proxy/hdrs/unit_tests/test_HdrUtils.cc | 2 +- src/traffic_cache_tool/CacheScan.cc | 5 +- src/traffic_server/CoreUtils.cc | 12 +-- 9 files changed, 103 insertions(+), 104 deletions(-) diff --git a/doc/developer-guide/core-architecture/heap.en.rst b/doc/developer-guide/core-architecture/heap.en.rst index f05fbbb1f20..e03b5ad77a7 100644 --- a/doc/developer-guide/core-architecture/heap.en.rst +++ b/doc/developer-guide/core-architecture/heap.en.rst @@ -159,10 +159,17 @@ collection operation on the writeable string heap in the header heap by calling * An external string heap is being added and all current read only string heap slots are used. +The mechanism is simple in design - the size of the live string data in the current string heaps is +calculated and a new heap is allocated sufficient to contain all existing strings, with additional +space for new string data. Each heap object is required to provide a :code:`strings_length` method +which returns the size of the live string data for that object (recursively as needed). The strings +are copied to the new string heap, all of the previous string heaps are discarded, and the new heap +becomes the writable string heap for the header heap. + Each heap object is responsible for providing a :code:`move_strings` method which copies its strings -to a new string heap. This is a source of pointer invalidation for other parts of the core and the -plugin API. For the latter, insulating from such string movement is the point of the -:c:type:`TSMLoc` type. +to a new string heap, passed as an argument. This is a source of pointer invalidation for other +parts of the core and the plugin API. For the latter, insulating from such string movement is the +point of the :c:type:`TSMLoc` type. String Allocation ----------------- diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 28bace51a27..49337278902 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -3289,18 +3289,18 @@ CacheProcessor::find_by_path(const char *path, int len) namespace cache_bc { -static size_t const HTTP_ALT_MARSHAL_SIZE = ROUND(sizeof(HTTPCacheAlt), HDR_PTR_SIZE); // current size. +static size_t const HTTP_ALT_MARSHAL_SIZE = HdrHeapMarshalBlocks{ts::round_up(sizeof(HTTPCacheAlt))}; // current size. size_t HTTPInfo_v21::marshalled_length(void *data) { - size_t zret = ROUND(sizeof(HTTPCacheAlt_v21), HDR_PTR_SIZE); + size_t zret = HdrHeapMarshalBlocks{ts::round_up(sizeof(HTTPCacheAlt_v21))}; HTTPCacheAlt_v21 *alt = static_cast(data); HdrHeap *hdr; hdr = reinterpret_cast(reinterpret_cast(alt) + reinterpret_cast(alt->m_request_hdr.m_heap)); - zret += ROUND(hdr->unmarshal_size(), HDR_PTR_SIZE); + zret += HdrHeapMarshalBlocks{ts::round_up(hdr->unmarshal_size())}; hdr = reinterpret_cast(reinterpret_cast(alt) + reinterpret_cast(alt->m_response_hdr.m_heap)); - zret += ROUND(hdr->unmarshal_size(), HDR_PTR_SIZE); + zret += HdrHeapMarshalBlocks{ts::round_up(hdr->unmarshal_size())}; return zret; } @@ -3363,7 +3363,7 @@ HTTPInfo_v21::copy_and_upgrade_unmarshalled_to_v23(char *&dst, char *&src, size_ s_hdr = reinterpret_cast(reinterpret_cast(s_alt) + reinterpret_cast(s_alt->m_request_hdr.m_heap)); d_hdr = reinterpret_cast(dst); - hdr_size = ROUND(s_hdr->unmarshal_size(), HDR_PTR_SIZE); + hdr_size = HdrHeapMarshalBlocks{ts::round_up(s_hdr->unmarshal_size())}; if (hdr_size > length) { return false; } @@ -3375,7 +3375,7 @@ HTTPInfo_v21::copy_and_upgrade_unmarshalled_to_v23(char *&dst, char *&src, size_ s_hdr = reinterpret_cast(reinterpret_cast(s_alt) + reinterpret_cast(s_alt->m_response_hdr.m_heap)); d_hdr = reinterpret_cast(dst); - hdr_size = ROUND(s_hdr->unmarshal_size(), HDR_PTR_SIZE); + hdr_size = HdrHeapMarshalBlocks{ts::round_up(s_hdr->unmarshal_size())}; if (hdr_size > length) { return false; } diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 4704bfd4925..41bf41b2284 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -1950,7 +1950,7 @@ HTTPCacheAlt::copy_frag_offsets_from(HTTPCacheAlt *src) } } -const int HTTP_ALT_MARSHAL_SIZE = ROUND(sizeof(HTTPCacheAlt), HDR_PTR_SIZE); +const int HTTP_ALT_MARSHAL_SIZE = HdrHeapMarshalBlocks{ts::round_up(sizeof(HTTPCacheAlt))}; void HTTPInfo::create() diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index 243d6ce8a74..cde0e9f2362 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -37,11 +37,11 @@ #include "HTTP.h" #include "I_EventSystem.h" -#define MAX_LOST_STR_SPACE 1024 +constexpr size_t MAX_LOST_STR_SPACE = 1024; -Allocator hdrHeapAllocator("hdrHeap", HDR_HEAP_DEFAULT_SIZE); +Allocator hdrHeapAllocator("hdrHeap", HdrHeap::DEFAULT_SIZE); -Allocator strHeapAllocator("hdrStrHeap", HDR_STR_HEAP_DEFAULT_SIZE); +Allocator strHeapAllocator("hdrStrHeap", HdrStrHeap::DEFAULT_SIZE); /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ @@ -114,8 +114,8 @@ HdrHeap * new_HdrHeap(int size) { HdrHeap *h; - if (size <= HDR_HEAP_DEFAULT_SIZE) { - size = HDR_HEAP_DEFAULT_SIZE; + if (size <= HdrHeap::DEFAULT_SIZE) { + size = HdrHeap::DEFAULT_SIZE; h = (HdrHeap *)(THREAD_ALLOC(hdrHeapAllocator, this_ethread())); } else { h = (HdrHeap *)ats_malloc(size); @@ -138,12 +138,12 @@ new_HdrStrHeap(int requested_size) int alloc_size = requested_size + sizeof(HdrStrHeap); HdrStrHeap *sh; - if (alloc_size <= HDR_STR_HEAP_DEFAULT_SIZE) { - alloc_size = HDR_STR_HEAP_DEFAULT_SIZE; + if (alloc_size <= HdrStrHeap::DEFAULT_SIZE) { + alloc_size = HdrStrHeap::DEFAULT_SIZE; sh = (HdrStrHeap *)(THREAD_ALLOC(strHeapAllocator, this_ethread())); } else { - alloc_size = ROUND(alloc_size, HDR_STR_HEAP_DEFAULT_SIZE * 2); - sh = (HdrStrHeap *)ats_malloc(alloc_size); + alloc_size = ts::round_up(alloc_size); + sh = static_cast(ats_malloc(alloc_size)); } // Debug("hdrs", "Allocated string heap in size %d", alloc_size); @@ -152,8 +152,8 @@ new_HdrStrHeap(int requested_size) sh = new (sh) HdrStrHeap(); sh->m_heap_size = alloc_size; - sh->m_free_size = alloc_size - STR_HEAP_HDR_SIZE; - sh->m_free_start = ((char *)sh) + STR_HEAP_HDR_SIZE; + sh->m_free_size = alloc_size - sizeof(HdrStrHeap); + sh->m_free_start = reinterpret_cast(sh + 1); ink_assert(sh->refcount() == 0); @@ -174,7 +174,7 @@ HdrHeap::destroy() i.m_ref_count_ptr = nullptr; } - if (m_size == HDR_HEAP_DEFAULT_SIZE) { + if (m_size == HdrHeap::DEFAULT_SIZE) { THREAD_FREE(this, hdrHeapAllocator, this_thread()); } else { ats_free(this); @@ -189,9 +189,9 @@ HdrHeap::allocate_obj(int nbytes, int type) ink_assert(m_writeable); - nbytes = ROUND(nbytes, HDR_PTR_SIZE); + nbytes = HdrHeapMarshalBlocks{ts::round_up(nbytes)}; - if (nbytes > (int)HDR_MAX_ALLOC_SIZE) { + if (nbytes > static_cast(HDR_MAX_ALLOC_SIZE)) { ink_assert(!"alloc too big"); return nullptr; } @@ -253,7 +253,7 @@ HdrHeap::allocate_str(int nbytes) // First check to see if we have a read/write // string heap if (!m_read_write_heap) { - int next_size = (last_size * 2) - STR_HEAP_HDR_SIZE; + int next_size = (last_size * 2) - sizeof(HdrStrHeap); next_size = next_size > nbytes ? next_size : nbytes; m_read_write_heap = new_HdrStrHeap(next_size); } @@ -562,7 +562,7 @@ HdrHeap::marshal_length() } } - len = ROUND(len, HDR_PTR_SIZE); + len = HdrHeapMarshalBlocks(ts::round_up(len)); return len; } @@ -624,7 +624,6 @@ HdrHeap::marshal(char *buf, int len) // Variables used later on. Sunpro doesn't like // bypassing initializations with gotos int used; - int i; HdrHeap *unmarshal_hdr = this; @@ -660,7 +659,7 @@ HdrHeap::marshal(char *buf, int len) // Now that we've got the pointer blocks marshaled // we can fill in the header on marshalled block marshal_hdr->m_free_start = nullptr; - marshal_hdr->m_data_start = (char *)HDR_HEAP_HDR_SIZE; // offset + marshal_hdr->m_data_start = reinterpret_cast(HDR_HEAP_HDR_SIZE.value()); // offset marshal_hdr->m_magic = HDR_BUF_MAGIC_MARSHALED; marshal_hdr->m_writeable = false; marshal_hdr->m_size = ptr_heap_size + HDR_HEAP_HDR_SIZE; @@ -673,7 +672,7 @@ HdrHeap::marshal(char *buf, int len) marshal_hdr->m_ronly_heap[0].m_heap_start = (char *)(intptr_t)marshal_hdr->m_size; // offset marshal_hdr->m_ronly_heap[0].m_ref_count_ptr.detach(); - for (int i = 1; i < HDR_BUF_RONLY_HEAPS; i++) { + for (unsigned i = 1; i < HDR_BUF_RONLY_HEAPS; ++i) { marshal_hdr->m_ronly_heap[i].m_heap_start = nullptr; } @@ -708,7 +707,7 @@ HdrHeap::marshal(char *buf, int len) str_heaps++; } - for (i = 0; i < HDR_BUF_RONLY_HEAPS; i++) { + for (unsigned i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { if (m_ronly_heap[i].m_heap_start != nullptr) { if (m_ronly_heap[i].m_heap_len > len) { goto Failed; @@ -785,7 +784,7 @@ HdrHeap::marshal(char *buf, int len) // Add up the total bytes used used = ptr_heap_size + str_size + HDR_HEAP_HDR_SIZE; - used = ROUND(used, HDR_PTR_SIZE); + used = HdrHeapMarshalBlocks(ts::round_up(used)); #ifdef HDR_HEAP_CHECKSUMS { @@ -956,14 +955,14 @@ HdrHeap::unmarshal(int buf_length, int obj_type, HdrHeapObjImpl **found_obj, Ref m_magic = HDR_BUF_MAGIC_ALIVE; - unmarshal_size = ROUND(unmarshal_size, HDR_PTR_SIZE); + unmarshal_size = HdrHeapMarshalBlocks(ts::round_up(unmarshal_size)); return unmarshal_size; } inline bool -HdrHeap::attach_str_heap(char *h_start, int h_len, RefCountObj *h_ref_obj, int *index) +HdrHeap::attach_str_heap(char const *h_start, int h_len, RefCountObj *h_ref_obj, int *index) { - if (*index >= HDR_BUF_RONLY_HEAPS) { + if (static_cast(*index) >= HDR_BUF_RONLY_HEAPS) { return false; } @@ -1005,14 +1004,13 @@ HdrHeap::inherit_string_heaps(const HdrHeap *inherit_from) return; } - int index; int first_free = HDR_BUF_RONLY_HEAPS; // default is out of array bounds int free_slots = 0; int inherit_str_size = 0; ink_assert(m_writeable); // Find the number of free heap slots & the first open index - for (index = 0; index < HDR_BUF_RONLY_HEAPS; index++) { + for (unsigned index = 0; index < HDR_BUF_RONLY_HEAPS; ++index) { if (m_ronly_heap[index].m_heap_start == nullptr) { if (first_free == HDR_BUF_RONLY_HEAPS) { first_free = index; @@ -1026,7 +1024,7 @@ HdrHeap::inherit_string_heaps(const HdrHeap *inherit_from) free_slots--; inherit_str_size = inherit_from->m_read_write_heap->m_heap_size; } - for (index = 0; index < HDR_BUF_RONLY_HEAPS; index++) { + for (unsigned index = 0; index < HDR_BUF_RONLY_HEAPS; ++index) { if (inherit_from->m_ronly_heap[index].m_heap_start != nullptr) { free_slots--; inherit_str_size += inherit_from->m_ronly_heap[index].m_heap_len; @@ -1054,8 +1052,8 @@ HdrHeap::inherit_string_heaps(const HdrHeap *inherit_from) // Copy over read/write string heap if it exists if (inherit_from->m_read_write_heap) { int str_size = - inherit_from->m_read_write_heap->m_heap_size - STR_HEAP_HDR_SIZE - inherit_from->m_read_write_heap->m_free_size; - ink_release_assert(attach_str_heap(((char *)inherit_from->m_read_write_heap.get()) + STR_HEAP_HDR_SIZE, str_size, + inherit_from->m_read_write_heap->m_heap_size - sizeof(HdrStrHeap) - inherit_from->m_read_write_heap->m_free_size; + ink_release_assert(attach_str_heap(reinterpret_cast(inherit_from->m_read_write_heap.get() + 1), str_size, inherit_from->m_read_write_heap.get(), &first_free)); } // Copy over read only string heaps @@ -1115,7 +1113,7 @@ HdrHeap::dump_heap(int len) void HdrStrHeap::free() { - if (m_heap_size == HDR_STR_HEAP_DEFAULT_SIZE) { + if (m_heap_size == HdrStrHeap::DEFAULT_SIZE) { THREAD_FREE(this, strHeapAllocator, this_thread()); } else { ats_free(this); @@ -1152,8 +1150,8 @@ HdrStrHeap::expand(char *ptr, int old_size, int new_size) { unsigned int expand_size = new_size - old_size; - ink_assert(ptr >= ((char *)this) + STR_HEAP_HDR_SIZE); - ink_assert(ptr < ((char *)this) + m_heap_size); + ink_assert(ptr >= reinterpret_cast(this + 1)); + ink_assert(ptr < reinterpret_cast(this) + m_heap_size); if (ptr + old_size == m_free_start && expand_size <= m_free_size) { m_free_start += expand_size; @@ -1174,9 +1172,9 @@ REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, * demotion of rw heaps to ronly heaps, and finally the coalesce and evacuate behaviours. */ - // The amount of space we will need to overflow the StrHdrHeap is HDR_STR_HEAP_DEFAULT_SIZE - STR_HEAP_HDR_SIZE - size_t next_rw_heap_size = HDR_STR_HEAP_DEFAULT_SIZE; - size_t next_required_overflow_size = next_rw_heap_size - STR_HEAP_HDR_SIZE; + // The amount of space we will need to overflow the StrHdrHeap is HdrStrHeap::DEFAULT_SIZE - sizeof(HdrStrHeap) + size_t next_rw_heap_size = HdrStrHeap::DEFAULT_SIZE; + size_t next_required_overflow_size = next_rw_heap_size - sizeof(HdrStrHeap); char buf[next_required_overflow_size]; for (unsigned int i = 0; i < sizeof(buf); ++i) { buf[i] = ('a' + (i % 26)); @@ -1189,15 +1187,15 @@ REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, tb.check(heap->m_read_write_heap.get() == nullptr, "Checking that we have no rw heap."); url_path_set(heap, url, buf, next_required_overflow_size, true); tb.check(heap->m_read_write_heap->m_free_size == 0, "Checking that we've completely consumed the rw heap"); - for (int i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { + for (unsigned i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { tb.check(heap->m_ronly_heap[i].m_heap_start == (char *)nullptr, "Checking ronly_heap[%d] is NULL", i); } // Now we have no ronly heaps in use and a completely full rwheap, so we will test that // we demote to ronly heaps HDR_BUF_RONLY_HEAPS times. - for (int ronly_heap = 0; ronly_heap < HDR_BUF_RONLY_HEAPS; ++ronly_heap) { + for (unsigned ronly_heap = 0; ronly_heap < HDR_BUF_RONLY_HEAPS; ++ronly_heap) { next_rw_heap_size = 2 * heap->m_read_write_heap->m_heap_size; - next_required_overflow_size = next_rw_heap_size - STR_HEAP_HDR_SIZE; + next_required_overflow_size = next_rw_heap_size - sizeof(HdrStrHeap); char buf2[next_required_overflow_size]; for (unsigned int i = 0; i < sizeof(buf2); ++i) { buf2[i] = ('a' + (i % 26)); @@ -1210,13 +1208,13 @@ REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, (int)next_rw_heap_size); tb.check(heap->m_read_write_heap->m_free_size == 0, "Checking that we've completely consumed the rw heap"); tb.check(heap->m_ronly_heap[ronly_heap].m_heap_start != nullptr, "Checking that we properly demoted the previous rw heap"); - for (int i = ronly_heap + 1; i < HDR_BUF_RONLY_HEAPS; ++i) { + for (unsigned i = ronly_heap + 1; i < HDR_BUF_RONLY_HEAPS; ++i) { tb.check(heap->m_ronly_heap[i].m_heap_start == nullptr, "Checking ronly_heap[%d] is NULL", i); } } // We will rerun these checks after we introduce a non-copied string to make sure we didn't already coalesce - for (int i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { + for (unsigned i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { tb.check(heap->m_ronly_heap[i].m_heap_start != (char *)nullptr, "Pre non-copied string: Checking ronly_heap[%d] is NOT NULL", i); } @@ -1234,7 +1232,7 @@ REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, "Checking that the aliased string shows having proper length"); tb.check(aliased_str_url->m_ptr_path == buf3, "Checking that the aliased string is correctly pointing at buf"); - for (int i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { + for (unsigned i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { tb.check(heap->m_ronly_heap[i].m_heap_start != (char *)nullptr, "Post non-copied string: Checking ronly_heap[%d] is NOT NULL", i); } diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index 9737782f0bb..6dd7deb3a14 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -36,28 +36,21 @@ #include "tscore/ink_defs.h" #include "tscore/ink_assert.h" #include "tscore/Arena.h" +#include "tscore/Scalar.h" #include "HdrToken.h" // Objects in the heap must currently be aligned to 8 byte boundaries, // so their (address & HDR_PTR_ALIGNMENT_MASK) == 0 -#define HDR_PTR_SIZE (sizeof(uint64_t)) -#define HDR_PTR_ALIGNMENT_MASK ((HDR_PTR_SIZE)-1L) - -#define ROUND(x, l) (((x) + ((l)-1L)) & ~((l)-1L)) +static constexpr size_t HDR_PTR_SIZE = sizeof(uint64_t); +static constexpr size_t HDR_PTR_ALIGNMENT_MASK = HDR_PTR_SIZE - 1L; +using HdrHeapMarshalBlocks = ts::Scalar; // A many of the operations regarding read-only str // heaps are hand unrolled in the code. Chaning // this value requires a full pass through HdrBuf.cc // to fix the unrolled operations -#define HDR_BUF_RONLY_HEAPS 3 - -#define HDR_HEAP_DEFAULT_SIZE 2048 -#define HDR_STR_HEAP_DEFAULT_SIZE 2048 - -#define HDR_MAX_ALLOC_SIZE (HDR_HEAP_DEFAULT_SIZE - sizeof(HdrHeap)) -#define HDR_HEAP_HDR_SIZE ROUND(sizeof(HdrHeap), HDR_PTR_SIZE) -#define STR_HEAP_HDR_SIZE sizeof(HdrStrHeap) +static constexpr unsigned HDR_BUF_RONLY_HEAPS = 3; class CoreUtils; class IOBufferBlock; @@ -140,6 +133,8 @@ enum { class HdrStrHeap : public RefCountObj { public: + static constexpr int DEFAULT_SIZE = 2048; + void free() override; char *allocate(int nbytes); @@ -150,20 +145,22 @@ class HdrStrHeap : public RefCountObj char *m_free_start; uint32_t m_free_size; - bool - contains(const char *str) const - { - return (str >= ((const char *)this + STR_HEAP_HDR_SIZE) && str < ((const char *)this + m_heap_size)); - } + bool contains(const char *str) const; }; +inline bool +HdrStrHeap::contains(const char *str) const +{ + return reinterpret_cast(this + 1) <= str && str < reinterpret_cast(this) + m_heap_size; +} + struct StrHeapDesc { StrHeapDesc() = default; Ptr m_ref_count_ptr; - char *m_heap_start = nullptr; - int32_t m_heap_len = 0; - bool m_locked = false; + char const *m_heap_start = nullptr; + int32_t m_heap_len = 0; + bool m_locked = false; bool contains(const char *str) const @@ -177,6 +174,8 @@ class HdrHeap friend class CoreUtils; public: + static constexpr int DEFAULT_SIZE = 2048; + void init(); inkcoreapi void destroy(); @@ -265,7 +264,7 @@ class HdrHeap void coalesce_str_heaps(int incoming_size = 0); void evacuate_from_str_heaps(HdrStrHeap *new_heap); size_t required_space_for_evacuation(); - bool attach_str_heap(char *h_start, int h_len, RefCountObj *h_ref_obj, int *index); + bool attach_str_heap(char const *h_start, int h_len, RefCountObj *h_ref_obj, int *index); /** Struct to prevent garbage collection on heaps. This bumps the reference count to the heap containing the pointer @@ -302,6 +301,9 @@ class HdrHeap int m_lost_string_space; }; +static constexpr HdrHeapMarshalBlocks HDR_HEAP_HDR_SIZE{ts::round_up(sizeof(HdrHeap))}; +static constexpr size_t HDR_MAX_ALLOC_SIZE = HdrHeap::DEFAULT_SIZE - HDR_HEAP_HDR_SIZE; + inline void HdrHeap::free_string(const char *s, int len) { @@ -318,15 +320,15 @@ HdrHeap::unmarshal_size() const // struct MarshalXlate { - char *start; - char *end; - char *offset; + char const *start; + char const *end; + char const *offset; MarshalXlate() : start(nullptr), end(nullptr), offset(nullptr) {} }; struct HeapCheck { - char *start; - char *end; + char const *start; + char const *end; }; // Nasty macro to do string marshalling @@ -479,6 +481,6 @@ HdrHeapSDKHandle::set(const HdrHeapSDKHandle *from) } HdrStrHeap *new_HdrStrHeap(int requested_size); -inkcoreapi HdrHeap *new_HdrHeap(int size = HDR_HEAP_DEFAULT_SIZE); +inkcoreapi HdrHeap *new_HdrHeap(int size = HdrHeap::DEFAULT_SIZE); void hdr_heap_test(); diff --git a/proxy/hdrs/HdrTSOnly.cc b/proxy/hdrs/HdrTSOnly.cc index 6841b1e9837..55e2d9351fb 100644 --- a/proxy/hdrs/HdrTSOnly.cc +++ b/proxy/hdrs/HdrTSOnly.cc @@ -170,30 +170,29 @@ HdrHeap::attach_block(IOBufferBlock *b, const char *use_start) RETRY: - // It's my contention that since heaps are add to the - // first available slot, one you find an empty slot - // it's not possible that a heap ptr for this block - // exists in a later slot - for (int i = 0; i < HDR_BUF_RONLY_HEAPS; i++) { - if (m_ronly_heap[i].m_heap_start == nullptr) { + // It's my contention that since heaps are add to the first available slot, one you find an empty + // slot it's not possible that a heap ptr for this block exists in a later slot + + for (auto &heap : m_ronly_heap) { + if (heap.m_heap_start == nullptr) { // Add block to heap in this slot - m_ronly_heap[i].m_heap_start = (char *)use_start; - m_ronly_heap[i].m_heap_len = (int)(b->end() - b->start()); - m_ronly_heap[i].m_ref_count_ptr = b->data.object(); + heap.m_heap_start = static_cast(use_start); + heap.m_heap_len = static_cast(b->end() - b->start()); + heap.m_ref_count_ptr = b->data.object(); // printf("Attaching block at %X for %d in slot %d\n", // m_ronly_heap[i].m_heap_start, // m_ronly_heap[i].m_heap_len, // i); - return i; - } else if (m_ronly_heap[i].m_heap_start == b->buf()) { + return &heap - m_ronly_heap; + } else if (heap.m_heap_start == b->buf()) { // This block is already on the heap so just extend // it's range - m_ronly_heap[i].m_heap_len = (int)(b->end() - b->buf()); + heap.m_heap_len = static_cast(b->end() - b->buf()); // printf("Extending block at %X to %d in slot %d\n", // m_ronly_heap[i].m_heap_start, // m_ronly_heap[i].m_heap_len, // i); - return i; + return &heap - m_ronly_heap; } } diff --git a/proxy/hdrs/unit_tests/test_HdrUtils.cc b/proxy/hdrs/unit_tests/test_HdrUtils.cc index fc178a256f9..fe085c98757 100644 --- a/proxy/hdrs/unit_tests/test_HdrUtils.cc +++ b/proxy/hdrs/unit_tests/test_HdrUtils.cc @@ -47,7 +47,7 @@ TEST_CASE("HdrUtils", "[proxy][hdrutils]") static constexpr std::string_view FOUR_TAG{"Four"}; static constexpr std::string_view FIVE_TAG{"Five"}; - HdrHeap *heap = new_HdrHeap(HDR_HEAP_DEFAULT_SIZE + 64); + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); MIMEParser parser; char const *real_s = text.data(); char const *real_e = text.data_end(); diff --git a/src/traffic_cache_tool/CacheScan.cc b/src/traffic_cache_tool/CacheScan.cc index 06be5355217..e52b256db7d 100644 --- a/src/traffic_cache_tool/CacheScan.cc +++ b/src/traffic_cache_tool/CacheScan.cc @@ -29,7 +29,7 @@ // using namespace ct; -const int HTTP_ALT_MARSHAL_SIZE = ROUND(sizeof(HTTPCacheAlt), HDR_PTR_SIZE); +constexpr HdrHeapMarshalBlocks HTTP_ALT_MARSHAL_SIZE = ts::round_up(sizeof(HTTPCacheAlt)); namespace ct { @@ -248,8 +248,7 @@ CacheScan::unmarshal(HdrHeap *hh, int buf_length, int obj_type, HdrHeapObjImpl * hh->m_magic = HDR_BUF_MAGIC_ALIVE; - int unmarshal_length = ROUND(hh->unmarshal_size(), HDR_PTR_SIZE); - return unmarshal_length; + return HdrHeapMarshalBlocks(ts::round_up(hh->unmarshal_size())); } Errata diff --git a/src/traffic_server/CoreUtils.cc b/src/traffic_server/CoreUtils.cc index eac83b0d1a3..15ddb38febc 100644 --- a/src/traffic_server/CoreUtils.cc +++ b/src/traffic_server/CoreUtils.cc @@ -469,10 +469,7 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) intptr_t ptr_heap_size = 0; intptr_t str_size = 0; intptr_t str_heaps = 0; - std::vector ptr_xlation(2); - // MarshalXlate static_table[2]; - // MarshalXlate* ptr_xlation = static_table; - intptr_t used; + std::vector ptr_xlation(2); intptr_t i; intptr_t copy_size; @@ -544,7 +541,7 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) swizzle_heap->m_ronly_heap[0].m_heap_start = (char *)(intptr_t)swizzle_heap->m_size; // offset swizzle_heap->m_ronly_heap[0].m_ref_count_ptr.m_ptr = nullptr; - for (int i = 1; i < HDR_BUF_RONLY_HEAPS; i++) { + for (unsigned i = 1; i < HDR_BUF_RONLY_HEAPS; ++i) { swizzle_heap->m_ronly_heap[i].m_heap_start = nullptr; } @@ -650,10 +647,7 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) } // Add up the total bytes used - used = ptr_heap_size + str_size + HDR_HEAP_HDR_SIZE; - used = ROUND(used, HDR_PTR_SIZE); - - return used; + return HdrHeapMarshalBlocks{ts::round_up(ptr_heap_size + str_size + HDR_HEAP_HDR_SIZE)}; Failed: swizzle_heap->m_magic = HDR_BUF_MAGIC_CORRUPT; From 57d74d8ae301ab6b1ccb0242f730618a4b84c4f8 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 12 Feb 2019 15:38:02 -0600 Subject: [PATCH 268/526] URL: remove undefined function declaration. --- proxy/hdrs/URL.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/proxy/hdrs/URL.h b/proxy/hdrs/URL.h index d1fa5822d11..31abaa1b40e 100644 --- a/proxy/hdrs/URL.h +++ b/proxy/hdrs/URL.h @@ -151,9 +151,6 @@ extern int URL_LEN_MMS; extern int URL_LEN_MMSU; extern int URL_LEN_MMST; -/* Private */ -void url_adjust(MarshalXlate *str_xlate, int num_xlate); - /* Public */ bool validate_host_name(std::string_view addr); void url_init(); From c33b904571c777365dbcd308ea651c8d7b3c82ec Mon Sep 17 00:00:00 2001 From: dyrock Date: Tue, 12 Feb 2019 21:44:23 +0000 Subject: [PATCH 269/526] tls_forward_nonhttp: Skip if nc is not installed (instead of curl) --- tests/gold_tests/tls/tls_forward_nonhttp.test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gold_tests/tls/tls_forward_nonhttp.test.py b/tests/gold_tests/tls/tls_forward_nonhttp.test.py index 90c947732dc..949f77458a2 100644 --- a/tests/gold_tests/tls/tls_forward_nonhttp.test.py +++ b/tests/gold_tests/tls/tls_forward_nonhttp.test.py @@ -21,9 +21,9 @@ Forwarding a non-HTTP protocol out of TLS ''' -# need Curl +# need nc Test.SkipUnless( - Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") + Condition.HasProgram("nc", "nc need to be installed on system for this test to work") ) # Define default ATS From 693b3cb465d2867379a80e882899aca8ea98f389 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Tue, 12 Feb 2019 10:01:44 +0900 Subject: [PATCH 270/526] Close a H2 connection if its stream error rate is high This adds a new setting for H2: proxy.config.http2.stream_error_rate_threshold This adds a new metric for a number of session closes because of the error rate proxy.process.http2.session_die_high_error_rate A conection starts graceful close when its stream error rate (error per ssn) exceeds the configured threshold. --- doc/admin-guide/files/records.config.en.rst | 7 +++ mgmt/RecordsConfig.cc | 2 + proxy/http2/HTTP2.cc | 5 ++ proxy/http2/HTTP2.h | 2 + proxy/http2/Http2ClientSession.cc | 70 ++++++++++++++------- proxy/http2/Http2ClientSession.h | 16 +++-- proxy/http2/Http2ConnectionState.cc | 6 +- proxy/http2/Http2ConnectionState.h | 26 +++++++- proxy/http2/Http2Stream.cc | 2 +- 9 files changed, 104 insertions(+), 32 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 7a3236c83d0..01e6d110d50 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3712,6 +3712,13 @@ HTTP/2 Configuration HTTP/2 connection to avoid duplicate pushes on the same connection. If the maximum number is reached, new entries are not remembered. +.. ts:cv:: CONFIG proxy.config.http2.stream_error_rate_threshold FLOAT 0.1 + :reloadable: + + This is the maximum stream error rate |TS| allows on an HTTP/2 connection. + |TS| gracefully closes connections that have stream error rates above this + setting by sending GOAWAY frames. + Plug-in Configuration ===================== diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 8bd392fa0db..3e93abf2c62 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1331,6 +1331,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.zombie_debug_timeout_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.stream_error_rate_threshold", RECD_FLOAT, "0.1", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , //# Add LOCAL Records Here {RECT_LOCAL, "proxy.local.incoming_ip_to_bind", RECD_STRING, nullptr, RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index 55bd94bde85..082c7b97ebb 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -62,6 +62,7 @@ static const char *const HTTP2_STAT_SESSION_DIE_ACTIVE_NAME = "pro static const char *const HTTP2_STAT_SESSION_DIE_INACTIVE_NAME = "proxy.process.http2.session_die_inactive"; static const char *const HTTP2_STAT_SESSION_DIE_EOS_NAME = "proxy.process.http2.session_die_eos"; static const char *const HTTP2_STAT_SESSION_DIE_ERROR_NAME = "proxy.process.http2.session_die_error"; +static const char *const HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE_NAME = "proxy.process.http2.session_die_high_error_rate"; union byte_pointer { byte_pointer(void *p) : ptr(p) {} @@ -731,6 +732,7 @@ uint32_t Http2::no_activity_timeout_in = 120; uint32_t Http2::active_timeout_in = 0; uint32_t Http2::push_diary_size = 256; uint32_t Http2::zombie_timeout_in = 0; +float Http2::stream_error_rate_threshold = 0.1; void Http2::init() @@ -748,6 +750,7 @@ Http2::init() REC_EstablishStaticConfigInt32U(active_timeout_in, "proxy.config.http2.active_timeout_in"); REC_EstablishStaticConfigInt32U(push_diary_size, "proxy.config.http2.push_diary_size"); REC_EstablishStaticConfigInt32U(zombie_timeout_in, "proxy.config.http2.zombie_debug_timeout_in"); + REC_EstablishStaticConfigFloat(stream_error_rate_threshold, "proxy.config.http2.stream_error_rate_threshold"); // If any settings is broken, ATS should not start ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, max_concurrent_streams_in})); @@ -796,6 +799,8 @@ Http2::init() static_cast(HTTP2_STAT_SESSION_DIE_INACTIVE), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_SESSION_DIE_ERROR_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_SESSION_DIE_ERROR), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE), RecRawStatSyncSum); } #if TS_HAS_TESTS diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index b34474344d2..9289b4090c0 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -83,6 +83,7 @@ enum { HTTP2_STAT_SESSION_DIE_INACTIVE, HTTP2_STAT_SESSION_DIE_EOS, HTTP2_STAT_SESSION_DIE_ERROR, + HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE, HTTP2_N_STATS // Terminal counter, NOT A STAT INDEX. }; @@ -377,6 +378,7 @@ class Http2 static uint32_t active_timeout_in; static uint32_t push_diary_size; static uint32_t zombie_timeout_in; + static float stream_error_rate_threshold; static void init(); }; diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index 7774d014398..dafa43ee97c 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -103,25 +103,37 @@ Http2ClientSession::free() // Update stats on how we died. May want to eliminate this. Was useful for // tracking down which cases we were having problems cleaning up. But for general // use probably not worth the effort - switch (dying_event) { - case VC_EVENT_NONE: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_DEFAULT, this_ethread()); - break; - case VC_EVENT_ACTIVE_TIMEOUT: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_ACTIVE, this_ethread()); - break; - case VC_EVENT_INACTIVITY_TIMEOUT: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_INACTIVE, this_ethread()); - break; - case VC_EVENT_ERROR: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_ERROR, this_ethread()); - break; - case VC_EVENT_EOS: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_EOS, this_ethread()); - break; - default: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_OTHER, this_ethread()); - break; + if (cause_of_death != Http2SessionCod::NOT_PROVIDED) { + switch (cause_of_death) { + case Http2SessionCod::HIGH_ERROR_RATE: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE, this_ethread()); + break; + case Http2SessionCod::NOT_PROVIDED: + // Can't happen but this case is here to not have default case. + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_OTHER, this_ethread()); + break; + } + } else { + switch (dying_event) { + case VC_EVENT_NONE: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_DEFAULT, this_ethread()); + break; + case VC_EVENT_ACTIVE_TIMEOUT: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_ACTIVE, this_ethread()); + break; + case VC_EVENT_INACTIVITY_TIMEOUT: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_INACTIVE, this_ethread()); + break; + case VC_EVENT_ERROR: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_ERROR, this_ethread()); + break; + case VC_EVENT_EOS: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_EOS, this_ethread()); + break; + default: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_OTHER, this_ethread()); + break; + } } ink_release_assert(this->client_vc == nullptr); @@ -360,13 +372,25 @@ Http2ClientSession::main_event_handler(int event, void *edata) break; } - if (!this->is_draining()) { + if (!this->is_draining() && this->connection_state.get_shutdown_reason() == Http2ErrorCode::HTTP2_ERROR_MAX) { this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NONE); } - // For a case we already checked Connection header and it didn't exist - if (this->is_draining() && this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { - this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED); + if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { + if (this->is_draining()) { // For a case we already checked Connection header and it didn't exist + Http2SsnDebug("Preparing for graceful shutdown because of draining state"); + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED); + } else if (this->connection_state.get_stream_error_rate() > + Http2::stream_error_rate_threshold) { // For a case many stream errors happened + ip_port_text_buffer ipb; + const char *client_ip = ats_ip_ntop(get_client_addr(), ipb, sizeof(ipb)); + Error("HTTP/2 session error client_ip=%s session_id=%" PRId64 + " closing a connection, because its stream error rate (%f) exceeded the threshold (%f)", + client_ip, connection_id(), this->connection_state.get_stream_error_rate(), Http2::stream_error_rate_threshold); + Http2SsnDebug("Preparing for graceful shutdown because of a high stream error rate"); + cause_of_death = Http2SessionCod::HIGH_ERROR_RATE; + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM); + } } if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NOT_INITIATED) { diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 9a2c5a103e7..06e62622bb8 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -44,6 +44,11 @@ #define HTTP2_SESSION_EVENT_SHUTDOWN_INIT (HTTP2_SESSION_EVENTS_START + 5) #define HTTP2_SESSION_EVENT_SHUTDOWN_CONT (HTTP2_SESSION_EVENTS_START + 6) +enum class Http2SessionCod : int { + NOT_PROVIDED, + HIGH_ERROR_RATE, +}; + size_t const HTTP2_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; // To support Upgrade: h2c @@ -342,11 +347,12 @@ class Http2ClientSession : public ProxySession // For Upgrade: h2c Http2UpgradeContext upgrade_context; - VIO *write_vio = nullptr; - int dying_event = 0; - bool kill_me = false; - bool half_close_local = false; - int recursion = 0; + VIO *write_vio = nullptr; + int dying_event = 0; + bool kill_me = false; + Http2SessionCod cause_of_death = Http2SessionCod::NOT_PROVIDED; + bool half_close_local = false; + int recursion = 0; std::unordered_set h2_pushed_urls; }; diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 3da2dd7fd87..9db9cafad4e 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -996,7 +996,10 @@ Http2ConnectionState::main_event_handler(int event, void *edata) shutdown_state = HTTP2_SHUTDOWN_IN_PROGRESS; // [RFC 7540] 6.8. GOAWAY // ..., the server can send another GOAWAY frame with an updated last stream identifier - send_goaway_frame(latest_streamid_in, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + if (shutdown_reason == Http2ErrorCode::HTTP2_ERROR_MAX) { + shutdown_reason = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; + } + send_goaway_frame(latest_streamid_in, shutdown_reason); // Stop creating new streams SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); this->ua_session->set_half_close_local_flag(true); @@ -1693,6 +1696,7 @@ Http2ConnectionState::send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec) if (ec != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_STREAM_ERRORS_COUNT, this_ethread()); + ++stream_error_count; } Http2Frame rst_stream(HTTP2_FRAME_TYPE_RST_STREAM, id, 0); diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index c2eea39025f..50c2b395b50 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -221,6 +221,23 @@ class Http2ConnectionState : public Continuation return client_streams_in_count; } + double + get_stream_error_rate() const + { + int total = get_stream_requests(); + if (total > 0) { + return (double)stream_error_count / (double)total; + } else { + return 0; + } + } + + Http2ErrorCode + get_shutdown_reason() const + { + return shutdown_reason; + } + // Connection level window size ssize_t client_rwnd = HTTP2_INITIAL_WINDOW_SIZE; ssize_t server_rwnd = Http2::initial_window_size; @@ -267,9 +284,10 @@ class Http2ConnectionState : public Continuation } void - set_shutdown_state(Http2ShutdownState state) + set_shutdown_state(Http2ShutdownState state, Http2ErrorCode reason = Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { - shutdown_state = state; + shutdown_state = state; + shutdown_reason = reason; } // noncopyable @@ -316,6 +334,9 @@ class Http2ConnectionState : public Continuation // Counter for current active streams and streams in the process of shutting down std::atomic total_client_streams_count = 0; + // Counter for stream errors ATS sent + uint32_t stream_error_count = 0; + // NOTE: Id of stream which MUST receive CONTINUATION frame. // - [RFC 7540] 6.2 HEADERS // "A HEADERS frame without the END_HEADERS flag set MUST be followed by a @@ -328,6 +349,7 @@ class Http2ConnectionState : public Continuation bool fini_received = false; int recursion = 0; Http2ShutdownState shutdown_state = HTTP2_SHUTDOWN_NONE; + Http2ErrorCode shutdown_reason = Http2ErrorCode::HTTP2_ERROR_MAX; Event *shutdown_cont_event = nullptr; Event *fini_event = nullptr; Event *zombie_event = nullptr; diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index fc6e3b40738..83da4e1375a 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -603,7 +603,7 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, if (memcmp(HTTP_VALUE_CLOSE, value, HTTP_LEN_CLOSE) == 0) { SCOPED_MUTEX_LOCK(lock, parent->connection_state.mutex, this_ethread()); if (parent->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { - parent->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED); + parent->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); } } } From d1e15c176936440fb6bc13cef983081a573e9174 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Wed, 13 Feb 2019 16:24:27 -0800 Subject: [PATCH 271/526] Doc: centering images is not in view When we center images/figures, it frequently extends into the full browser width, not the more constrained view. Figures have a surrounding div that allows for better styling than images. In Sphinx, an image is a simple picture, whereas a figure is a image and surrounding data (caption, legend, etc). So, they are effectively interchangeable. --- doc/admin-guide/layer-4-routing.en.rst | 4 ++-- .../monitoring/monitoring/third-party/circonus.en.rst | 10 +++++----- .../cache-architecture/architecture.en.rst | 9 +++++---- doc/developer-guide/core-architecture/rpc.en.rst | 4 ++-- .../plugins/example-plugins/tls_bridge.en.rst | 4 ++-- doc/static/override.css | 5 +++++ 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/doc/admin-guide/layer-4-routing.en.rst b/doc/admin-guide/layer-4-routing.en.rst index f81a56f7e00..2355ecd2feb 100644 --- a/doc/admin-guide/layer-4-routing.en.rst +++ b/doc/admin-guide/layer-4-routing.en.rst @@ -28,7 +28,7 @@ accomplished by examining the initial data from the inbound connection to decide destination. The initial data is then sent to the destination and subsequently |TS| forwards all data read on one connection to the other and vice versa. -.. image:: ../uml/images/l4-basic-sequence.svg +.. figure:: ../uml/images/l4-basic-sequence.svg :align: center In this way it acts similary to `nc `__. @@ -116,7 +116,7 @@ In addition to this, in the :file:`records.config` file, edit the following vari The sequence of network activity for a Client connecting to ``service-2`` is -.. image:: ../uml/images/l4-sni-routing-seq.svg +.. figure:: ../uml/images/l4-sni-routing-seq.svg :align: center Note the destination for the outbound TCP connection and the HTTP ``CONNECT`` is the same. If this diff --git a/doc/admin-guide/monitoring/monitoring/third-party/circonus.en.rst b/doc/admin-guide/monitoring/monitoring/third-party/circonus.en.rst index 27ae6d2d7c9..fde5e9676fa 100644 --- a/doc/admin-guide/monitoring/monitoring/third-party/circonus.en.rst +++ b/doc/admin-guide/monitoring/monitoring/third-party/circonus.en.rst @@ -62,7 +62,7 @@ environment. #. Begin the new check creation process from within your Circonus account by clicking the *New Check* button near the top-right of the checks screen. - .. image:: ../../../../static/images/admin/monitor/circonus/new-check-button.png + .. figure:: ../../../../static/images/admin/monitor/circonus/new-check-button.png :alt: Circonus New Check button :align: center @@ -71,7 +71,7 @@ environment. you and will depend largely on whether you are using on-site Circonus or the hosted service, as well as the geographic location of your |TS| instance(s). - .. image:: ../../../../static/images/admin/monitor/circonus/check-config-1.png + .. figure:: ../../../../static/images/admin/monitor/circonus/check-config-1.png :alt: Choosing a check type :align: center @@ -83,7 +83,7 @@ environment. any of the other options to match your environment if necessary (for this guide, only *Host* and *URI* will need to be entered). - .. image:: ../../../../static/images/admin/monitor/circonus/check-config-2.png + .. figure:: ../../../../static/images/admin/monitor/circonus/check-config-2.png :alt: Advanced check configuration :align: center @@ -94,7 +94,7 @@ environment. running and use ``curl`` to manually fetch the statistics data from your server. - .. image:: ../../../../static/images/admin/monitor/circonus/check-config-3.png + .. figure:: ../../../../static/images/admin/monitor/circonus/check-config-3.png :alt: Check test :align: center @@ -110,7 +110,7 @@ environment. clicking on *Metrics Grid* for an overview visualization of all the data being collected. - .. image:: ../../../../static/images/admin/monitor/circonus/metric-grid.png + .. figure:: ../../../../static/images/admin/monitor/circonus/metric-grid.png :alt: Circonus metric grid :align: center diff --git a/doc/developer-guide/cache-architecture/architecture.en.rst b/doc/developer-guide/cache-architecture/architecture.en.rst index 6442c0e9579..84a04084140 100644 --- a/doc/developer-guide/cache-architecture/architecture.en.rst +++ b/doc/developer-guide/cache-architecture/architecture.en.rst @@ -80,12 +80,12 @@ stripe is in a single cache span and part of a single cache volume. If the cache volumes for the example cache spans were defined as: -.. image:: images/ats-cache-volume-definition.png +.. figure:: images/ats-cache-volume-definition.png :align: center Then the actual layout would look like: -.. image:: images/cache-span-layout.png +.. figure:: images/cache-span-layout.png :align: center Cache stripes are the fundamental unit of cache for the implementation. A @@ -115,7 +115,7 @@ relationship between a span block and a cache stripe is the same as between a di file system. A cache stripe is structured data contained in a span block and always occupies the entire span block. -.. image:: images/span-header.svg +.. figure:: images/span-header.svg :align: center Stripe Structure @@ -292,7 +292,8 @@ The header for a stripe is a variably sized instance of :class:`VolHeaderFooter` The variable trailing section contains the head indices of the directory entry free lists for the segments. -.. image:: images/stripe-header.svg +.. figure:: images/stripe-header.svg + :align: center The trailing :member:`VolHeaderFooter::freelist` array overlays the disk storage with an entry for every segment, even though the array is declared to have length `1`. diff --git a/doc/developer-guide/core-architecture/rpc.en.rst b/doc/developer-guide/core-architecture/rpc.en.rst index 13c0a2fe7f3..0b5cdeaa3f6 100644 --- a/doc/developer-guide/core-architecture/rpc.en.rst +++ b/doc/developer-guide/core-architecture/rpc.en.rst @@ -65,7 +65,7 @@ Message Passing Sequence diagram for a command sent from |TCtl| to when it is recieved by a plugin. -.. image:: ../../uml/images/RPC-sequence-diagram.svg +.. figure:: ../../uml/images/RPC-sequence-diagram.svg .. note:: @@ -231,7 +231,7 @@ Now, using this macro, messages can easily be sent. For example: RPC API for |TServer| and |TManager| ==================================== -.. image:: ../../uml/images/RPC-states.svg +.. figure:: ../../uml/images/RPC-states.svg :align: center |LM| and |PM| follow similar workflows. A manager will poll the socket for any messages. If it is able to read a message, it will handle it based on the :arg:`msg_id` from the :class:`MgmtMessageHdr` and select a callback to run asynchoronously. The async callback will add a response, if any, to an outgoing event queue within the class. A manager will continue to poll and read on the socket as long as there are messages avaliable. Two things can stop a manager from polling. diff --git a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst index 29821e65d5b..8cf444fd5ab 100644 --- a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst +++ b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst @@ -272,7 +272,7 @@ The overall exchange looks like the following: A detailed view of the plugin operation. -.. image:: ../../../uml/images/TLS-Bridge-Plugin.svg +.. figure:: ../../../uml/images/TLS-Bridge-Plugin.svg :align: center A sequence diagram focusing on the request / response data flow. There is a :code:`NetVConn` for the @@ -287,7 +287,7 @@ The :code:`200 OK` sent from the Peer |TS| is parsed and consumed by the plugin. means there was an error and the tunnel is shut down. To deal with the Client response clean up the response code is stored and used later during cleanup. -.. image:: ../../../uml/images/TLS-Bridge-Messages.svg +.. figure:: ../../../uml/images/TLS-Bridge-Messages.svg :align: center A restartable state machine is used to recognize the end of the Peer |TS| response. The initial part diff --git a/doc/static/override.css b/doc/static/override.css index 57cfd51d9a0..d29fb0d16ce 100644 --- a/doc/static/override.css +++ b/doc/static/override.css @@ -19,6 +19,11 @@ dd { overflow: visible; } +div.figure.align-center{ + margin: initial; + max-width: 800px; +} + /* pre inside paragraphs gets weird spacing */ span.pre { line-height: initial; From 707ccf346c3e97d5b0b37fe5250f1cef839f5e5d Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 11 Feb 2019 15:45:30 +0000 Subject: [PATCH 272/526] Remove unnecessary storing of redirect_url in redirect_info --- proxy/http/HttpSM.cc | 11 ++++------- proxy/http/HttpTransact.cc | 8 +------- proxy/http/HttpTransact.h | 2 -- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 462febb33af..4450247a744 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -7727,13 +7727,8 @@ HttpSM::redirect_request(const char *arg_redirect_url, const int arg_redirect_le t_state.redirect_info.redirect_in_process = true; // set the passed in location url and parse it - URL &redirectUrl = t_state.redirect_info.redirect_url; - if (!redirectUrl.valid()) { - redirectUrl.create(nullptr); - } - - // reset the path from previous redirects (if any) - t_state.redirect_info.redirect_url.path_set(nullptr, 0); + URL redirectUrl; + redirectUrl.create(nullptr); redirectUrl.parse(arg_redirect_url, arg_redirect_len); { @@ -7763,6 +7758,8 @@ HttpSM::redirect_request(const char *arg_redirect_url, const int arg_redirect_le // copy the redirect url to the client url clientUrl.copy(&redirectUrl); + redirectUrl.destroy(); + //(bug 2540703) Clear the previous response if we will attempt the redirect if (t_state.hdr_info.client_response.valid()) { // XXX - doing a destroy() for now, we can do a fileds_clear() if we have performance issue diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 604db29a9f5..a85263d3959 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -7551,13 +7551,7 @@ HttpTransact::build_request(State *s, HTTPHdr *base_request, HTTPHdr *outgoing_r // // notice that currently, based_request IS client_request if (base_request == &s->hdr_info.client_request) { - if (s->redirect_info.redirect_in_process) { - // this is for auto redirect - URL *r_url = &s->redirect_info.redirect_url; - - ink_assert(r_url->valid()); - base_request->url_set(r_url); - } else { + if (!s->redirect_info.redirect_in_process) { // this is for multiple cache lookup URL *o_url = &s->cache_info.original_url; diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index d76f0ec0398..5315cdba690 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -530,7 +530,6 @@ class HttpTransact typedef struct _RedirectInfo { bool redirect_in_process = false; URL original_url; - URL redirect_url; _RedirectInfo() {} } RedirectInfo; @@ -882,7 +881,6 @@ class HttpTransact cache_info.object_store.destroy(); cache_info.transform_store.destroy(); redirect_info.original_url.destroy(); - redirect_info.redirect_url.destroy(); url_map.clear(); arena.reset(); From c4cf32f04be3325170c0fb85d786dbb0e4a47c57 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Wed, 13 Feb 2019 16:26:08 -0800 Subject: [PATCH 273/526] Doc: money trace plugin link incorrect --- doc/admin-guide/plugins/index.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index de8e8841eb6..d27e931560b 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -201,7 +201,7 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`Metalink ` Implements the Metalink download description format in order to try not to download the same file twice. -:doc:`Money Trace ` +:doc:`Money Trace ` Allows Trafficserver to participate in a distributed tracing system based upon the Comcast Money library. :doc:`MP4 ` From dc503e87ed127dc74e60f6fa0e15857b3b5efcdd Mon Sep 17 00:00:00 2001 From: Kit Chan Date: Wed, 13 Feb 2019 00:24:54 -0800 Subject: [PATCH 274/526] Documentation updates to the lua plugin --- doc/admin-guide/plugins/lua.en.rst | 372 ++++++++++++++++------------- 1 file changed, 201 insertions(+), 171 deletions(-) diff --git a/doc/admin-guide/plugins/lua.en.rst b/doc/admin-guide/plugins/lua.en.rst index c955b8101f2..67fc36c574a 100644 --- a/doc/admin-guide/plugins/lua.en.rst +++ b/doc/admin-guide/plugins/lua.en.rst @@ -141,7 +141,7 @@ is always available within lua script. This package can be introduced into Lua l ts.say('Hello World') ts.sleep(10) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.process.uuid --------------- @@ -157,7 +157,7 @@ Here is an example: local pid = ts.process.uuid() -- a436bae6-082c-4805-86af-78a5916c4a91 -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.now ------ @@ -174,7 +174,7 @@ Here is an example: local nt = ts.now() -- 1395221053.123 -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.debug -------- @@ -195,7 +195,7 @@ We should write this TAG in records.config(If TAG is missing, default TAG will b ``CONFIG proxy.config.diags.debug.tags STRING TAG`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.error -------- @@ -211,7 +211,7 @@ Here is an example: ts.error('This is an error message') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ TS Basic Internal Information ----------------------------- @@ -232,7 +232,7 @@ Here is an example: local config_dir = ts.get_config_dir() -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Remap status constants ---------------------- @@ -248,7 +248,7 @@ Remap status constants These constants are usually used as return value of do_remap function. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_url_host ------------------------ @@ -268,7 +268,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_url_port ------------------------ @@ -278,7 +278,7 @@ ts.remap.get_to_url_port **description**: retrieve the "to" port of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_url_scheme -------------------------- @@ -288,7 +288,7 @@ ts.remap.get_to_url_scheme **description**: retrieve the "to" scheme of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_uri ------------------- @@ -298,7 +298,7 @@ ts.remap.get_to_uri **description**: retrieve the "to" path of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_url ------------------- @@ -308,7 +308,7 @@ ts.remap.get_to_url **description**: retrieve the "to" url of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_url_host -------------------------- @@ -318,7 +318,7 @@ ts.remap.get_from_url_host **description**: retrieve the "from" host of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_url_port -------------------------- @@ -328,7 +328,7 @@ ts.remap.get_from_url_port **description**: retrieve the "from" port of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_url_scheme ---------------------------- @@ -338,7 +338,7 @@ ts.remap.get_from_url_scheme **description**: retrieve the "from" scheme of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_uri --------------------- @@ -348,7 +348,7 @@ ts.remap.get_from_uri **description**: retrieve the "from" path of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_url --------------------- @@ -358,7 +358,7 @@ ts.remap.get_from_url **description**: retrieve the "from" url of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.hook ------- @@ -403,7 +403,7 @@ You can create global hook as well ts.hook(TS_LUA_HOOK_READ_REQUEST_HDR, do_some_work) - Or you can do it this way +Or you can do it this way :: @@ -418,7 +418,7 @@ Also the return value of the function will control how the transaction will be r the transaction to be re-enabled normally (TS_EVENT_HTTP_CONTINUE). Return value of 1 will be using TS_EVENT_HTTP_ERROR instead. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Hook point constants -------------------- @@ -489,7 +489,7 @@ Additional Information: +-----------------------+---------------------------+----------------------+--------------------+----------------------+ -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.ctx ------ @@ -528,7 +528,7 @@ Then the client will get the response like this: Connection: Keep-Alive ... -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_method ---------------------------- @@ -539,7 +539,7 @@ ts.client_request.get_method **description:** This function can be used to retrieve the current client request's method name. String like "GET" or "POST" is returned. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_method ---------------------------- @@ -559,7 +559,7 @@ ts.client_request.get_version Current possible values are 1.0, 1.1, and 0.9. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_version ----------------------------- @@ -573,7 +573,7 @@ ts.client_request.set_version ts.client_request.set_version('1.0') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_uri ------------------------- @@ -597,7 +597,7 @@ Then ``GET /st?a=1`` will yield the output: ``/st`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_uri ------------------------- @@ -610,7 +610,7 @@ ts.client_request.set_uri The PATH argument must be a Lua string and starts with ``/`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_uri_args ------------------------------ @@ -634,7 +634,7 @@ Then ``GET /st?a=1&b=2`` will yield the output: ``a=1&b=2`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_uri_args ------------------------------ @@ -649,7 +649,7 @@ ts.client_request.set_uri_args ts.client_request.set_uri_args('n=6&p=7') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_uri_params -------------------------------- @@ -673,7 +673,7 @@ Then ``GET /st;a=1`` will yield the output: ``a=1`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_uri_params -------------------------------- @@ -688,7 +688,7 @@ ts.client_request.set_uri_params ts.client_request.set_uri_params('n=6') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_url ------------------------- @@ -711,7 +711,7 @@ Then ``GET /st?a=1&b=2 HTTP/1.1\r\nHost: a.tbcdn.cn\r\n...`` will yield the outp ``http://a.tbcdn.cn/st?a=1&b=2`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.header.HEADER ------------------------------- @@ -740,7 +740,7 @@ Then ``GET /st HTTP/1.1\r\nHost: b.tb.cn\r\nUser-Agent: Mozilla/5.0\r\n...`` wil ``Mozilla/5.0`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_headers ----------------------------- @@ -768,7 +768,7 @@ Then ``GET /st HTTP/1.1\r\nHost: b.tb.cn\r\nUser-Aget: Mozilla/5.0\r\nAccept: */ Accept: */* -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.client_addr.get_addr -------------------------------------- @@ -792,7 +792,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.client_addr.get_incoming_port ----------------------------------------------- @@ -813,7 +813,7 @@ Here is an example: ts.debug(port) -- 80 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_url_host ------------------------------ @@ -836,7 +836,7 @@ Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` will yi ``192.168.231.129`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_url_host ------------------------------ @@ -866,7 +866,7 @@ remap.config like this: Then server request will connect to ``192.168.231.130:80`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_url_port ------------------------------ @@ -890,7 +890,7 @@ Then Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` wi ``8080`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_url_port ------------------------------ @@ -902,7 +902,7 @@ ts.client_request.set_url_port the origin server, and we should return TS_LUA_REMAP_DID_REMAP(_STOP) in do_remap. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_url_scheme -------------------------------- @@ -926,7 +926,7 @@ Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` will yi ``http`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_url_scheme -------------------------------- @@ -938,7 +938,7 @@ ts.client_request.set_url_scheme server request, and we should return TS_LUA_REMAP_DID_REMAP(_STOP) in do_remap. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_cache_url --------------------- @@ -957,7 +957,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_cache_lookup_url ---------------------------- @@ -982,7 +982,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_cache_lookup_url ---------------------------- @@ -992,7 +992,7 @@ ts.http.set_cache_lookup_url **description:** This function can be used to set the cache lookup url for the client request. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_parent_proxy ------------------------ @@ -1018,7 +1018,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_parent_proxy ------------------------ @@ -1028,7 +1028,7 @@ ts.http.set_parent_proxy **description:** This function can be used to set the parent proxy host and name. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_parent_selection_url -------------------------------- @@ -1053,7 +1053,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_parent_selection_url -------------------------------- @@ -1063,7 +1063,7 @@ ts.http.set_parent_selection_url **description:** This function can be used to set the parent selection url for the client request. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_server_resp_no_store -------------------------------- @@ -1082,7 +1082,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_resp ---------------- @@ -1117,7 +1117,7 @@ We will get the response like this: Document access failed :) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_cache_lookup_status ------------------------------- @@ -1146,7 +1146,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_cache_lookup_status ------------------------------- @@ -1176,7 +1176,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Http cache lookup status constants ---------------------------------- @@ -1190,7 +1190,7 @@ Http cache lookup status constants TS_LUA_CACHE_LOOKUP_SKIPPED (3) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.cached_response.get_status ----------------------------- @@ -1218,7 +1218,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.cached_response.get_version ------------------------------ @@ -1231,7 +1231,7 @@ ts.cached_response.get_version Current possible values are 1.0, 1.1, and 0.9. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.cached_response.header.HEADER -------------------------------- @@ -1261,7 +1261,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.cached_response.get_headers ------------------------------ @@ -1304,7 +1304,7 @@ We will get the output: Server: ATS/5.0.0 -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_uri @@ -1334,7 +1334,7 @@ Then ``GET /am.txt?a=1`` will yield the output: ``/am.txt`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_uri ------------------------- @@ -1347,7 +1347,7 @@ ts.server_request.set_uri The PATH argument must be a Lua string and starts with ``/`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_uri_args ------------------------------ @@ -1376,7 +1376,7 @@ Then ``GET /st?a=1&b=2`` will yield the output: ``a=1&b=2`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_uri_args ------------------------------ @@ -1391,7 +1391,7 @@ ts.server_request.set_uri_args ts.server_request.set_uri_args('n=6&p=7') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_uri_params -------------------------------- @@ -1420,7 +1420,7 @@ Then ``GET /st;a=1`` will yield the output: ``a=1`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_uri_params -------------------------------- @@ -1435,7 +1435,7 @@ ts.server_request.set_uri_params ts.server_request.set_uri_params('n=6') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.header.HEADER ------------------------------- @@ -1469,7 +1469,7 @@ Then ``GET /st HTTP/1.1\r\nHost: b.tb.cn\r\nUser-Agent: Mozilla/5.0\r\n...`` wil ``Mozilla/5.0`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_headers ----------------------------- @@ -1504,7 +1504,7 @@ We will get the output: Accept: */* -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.set_addr -------------------------------------- @@ -1524,7 +1524,7 @@ Here is an example: ts.server_request.server_addr.set_addr("192.168.231.17", 80, TS_LUA_AF_INET) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Socket address family --------------------- @@ -1536,7 +1536,7 @@ Socket address family TS_LUA_AF_INET6 (10) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_addr -------------------------------------- @@ -1559,7 +1559,7 @@ Here is an example: ts.debug(family) -- 2(AF_INET) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_nexthop_addr ---------------------------------------------- @@ -1582,7 +1582,7 @@ Here is an example: ts.debug(family) -- 2(AF_INET) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_ip ------------------------------------ @@ -1603,7 +1603,7 @@ Here is an example: ts.debug(ip) -- 192.168.231.17 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_port -------------------------------------- @@ -1624,7 +1624,7 @@ Here is an example: ts.debug(port) -- 80 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_outgoing_port ----------------------------------------------- @@ -1645,7 +1645,7 @@ Here is an example: ts.debug(port) -- 50880 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.set_outgoing_addr ----------------------------------------------- @@ -1665,7 +1665,7 @@ Here is an example: ts.server_request.server_addr.set_outgoing_addr("192.168.231.17", 80, TS_LUA_AF_INET) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_url_host ------------------------------ @@ -1693,7 +1693,7 @@ Then ``GET http://abc.com/p2/a.txt HTTP/1.1`` will yield the output: ``abc.com`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_url_host ------------------------------ @@ -1743,7 +1743,7 @@ Will be changed to: Client-ip: 135.xx.xx.xx X-Forwarded-For: 135.xx.xx.xx -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_url_scheme -------------------------------- @@ -1771,7 +1771,7 @@ Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` will yi ``http`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_url_scheme -------------------------------- @@ -1781,7 +1781,7 @@ ts.server_request.set_url_scheme **description:** Set ``scheme`` field of the request url with ``str``. This function is used to change the scheme of the server request. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_response.get_status ----------------------------- @@ -1807,7 +1807,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_' +`TOP <#lua-plugin>`_' ts.server_response.set_status ----------------------------- @@ -1831,7 +1831,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_' +`TOP <#lua-plugin>`_' ts.server_response.get_version ------------------------------ @@ -1843,7 +1843,7 @@ ts.server_response.get_version Current possible values are 1.0, 1.1, and 0.9. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_response.set_version ------------------------------ @@ -1857,7 +1857,7 @@ ts.server_response.set_version ts.server_response.set_version('1.0') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_response.header.HEADER -------------------------------- @@ -1891,7 +1891,7 @@ We will get the output: ``text/html`` -`TOP <#ts-lua-plugin>`_' +`TOP <#lua-plugin>`_' ts.server_response.get_headers ------------------------------ @@ -1932,7 +1932,7 @@ We will get the output: Accept-Ranges: bytes -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.get_status ----------------------------- @@ -1958,7 +1958,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.set_status ----------------------------- @@ -1982,7 +1982,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.get_version ------------------------------ @@ -1994,7 +1994,7 @@ ts.client_response.get_version Current possible values are 1.0, 1.1, and 0.9. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.set_version ------------------------------ @@ -2008,7 +2008,7 @@ ts.client_response.set_version ts.client_response.set_version('1.0') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.header.HEADER -------------------------------- @@ -2042,7 +2042,7 @@ We will get the output: ``text/html`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.get_headers ------------------------------ @@ -2083,7 +2083,7 @@ We will get the output: Accept-Ranges: bytes -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.set_error_resp --------------------------------- @@ -2127,7 +2127,7 @@ We will get the response like this: bad luck :( -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Number constants ---------------------- @@ -2140,7 +2140,7 @@ Number constants These constants are usually used in transform handler. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_cache_transformed ------------------------------ @@ -2167,7 +2167,7 @@ Here is an example: This function is usually called after we hook TS_LUA_RESPONSE_TRANSFORM. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_cache_untransformed -------------------------------- @@ -2194,7 +2194,7 @@ Here is an example: This function is usually called after we hook TS_LUA_RESPONSE_TRANSFORM. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_transform.get_upstream_bytes ----------------------------------------- @@ -2235,7 +2235,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_transform.get_upstream_watermark_bytes --------------------------------------------------- @@ -2246,7 +2246,7 @@ ts.http.resp_transform.get_upstream_watermark_bytes **description**: This function can be used to retrive the current watermark bytes for the upstream transform buffer. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_transform.set_upstream_watermark_bytes --------------------------------------------------- @@ -2259,7 +2259,7 @@ ts.http.resp_transform.set_upstream_watermark_bytes Setting the watermark bytes above 32kb may improve the performance of the transform handler. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_transform.set_downstream_bytes ------------------------------------------- @@ -2273,7 +2273,7 @@ Sometimes we want to set Content-Length header in client_response, and this func data is returned from the transform handler. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.skip_remapping_set -------------------------- @@ -2295,7 +2295,7 @@ Here is an example: This function is usually called in do_global_read_request function -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_client_protocol_stack --------------------------------- @@ -2317,7 +2317,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_push ------------------- @@ -2336,7 +2336,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.is_websocket -------------------- @@ -2356,7 +2356,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_plugin_tag ---------------------- @@ -2376,7 +2376,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.id ---------- @@ -2396,7 +2396,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.ssn_id -------------- @@ -2416,7 +2416,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.is_internal_request --------------------------- @@ -2436,7 +2436,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.is_aborted ------------------ @@ -2456,7 +2456,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.transaction_count ------------------------- @@ -2476,7 +2476,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.redirect_url_set ------------------------ @@ -2495,7 +2495,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_server_state ------------------------ @@ -2516,7 +2516,7 @@ Here is an example end end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Server state constants ---------------------- @@ -2534,10 +2534,9 @@ Server state constants TS_LUA_SRVSTATE_OPEN_RAW_ERROR (7) TS_LUA_SRVSTATE_PARSE_ERROR (8) TS_LUA_SRVSTATE_TRANSACTION_COMPLETE (9) - TS_LUA_SRVSTATE_CONGEST_CONTROL_CONGESTED_ON_F (10) - TS_LUA_SRVSTATE_CONGEST_CONTROL_CONGESTED_ON_M (11) + TS_LUA_SRVSTATE_PARENT_RETRY (10) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_remap_from_url -------------------------- @@ -2556,7 +2555,7 @@ Here is an example ts.debug(from_url) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_remap_to_url ------------------------ @@ -2575,7 +2574,7 @@ Here is an example ts.debug(to_url) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_client_fd --------------------- @@ -2594,7 +2593,7 @@ Here is an example ts.debug(fd) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_server_fd --------------------- @@ -2613,7 +2612,7 @@ Here is an example ts.debug(fd) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.add_package_path ------------------- @@ -2636,7 +2635,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.add_package_cpath -------------------- @@ -2660,7 +2659,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.md5 @@ -2682,7 +2681,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.md5_bin ---------- @@ -2702,7 +2701,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.sha1 ------- @@ -2723,7 +2722,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.sha1_bin ----------- @@ -2743,7 +2742,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.base64_encode ---------------- @@ -2763,7 +2762,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.base64_decode ---------------- @@ -2784,7 +2783,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.escape_uri ------------- @@ -2803,7 +2802,7 @@ Here is an example: value = ts.escape_uri(test) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.unescape_uri --------------- @@ -2824,7 +2823,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.fetch ----------- @@ -2879,7 +2878,7 @@ Issuing a post request: res = ts.fetch('http://xx.com/foo', {method = 'POST', body = 'hello world'}) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.fetch_multi -------------- @@ -2907,7 +2906,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.intercept @@ -2968,7 +2967,7 @@ Then we will get the response like this: 1395145392 Zheng. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_intercept ------------------------ @@ -3034,7 +3033,7 @@ Here is an example: ts.http.server_intercept(process_combo, h) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.say ------ @@ -3045,7 +3044,7 @@ ts.say **description:** Write response to ATS within intercept or server_intercept. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.flush -------- @@ -3108,7 +3107,7 @@ We will get the response like this: wo ai yu ye hua wo ai yu ye hua -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.sleep -------- @@ -3137,7 +3136,7 @@ Here is an example: ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.host_lookup -------------- @@ -3161,7 +3160,7 @@ Here is an example: ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.schedule ----------- @@ -3191,7 +3190,7 @@ Here is an example: ts.hook(TS_LUA_HOOK_CACHE_LOOKUP_COMPLETE, cache_lookup) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_int_get ---------------------- @@ -3206,7 +3205,7 @@ ts.http.config_int_get val = ts.http.config_int_get(TS_LUA_CONFIG_HTTP_CACHE_HTTP) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_int_set ---------------------- @@ -3226,7 +3225,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_float_get ------------------------ @@ -3237,7 +3236,7 @@ ts.http.config_float_get **description:** Configuration option which has a float value can be retrieved with this function. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_float_set ------------------------ @@ -3248,7 +3247,7 @@ ts.http.config_float_set **description:** This function can be used to overwrite the configuration options. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_string_get ------------------------- @@ -3259,7 +3258,7 @@ ts.http.config_string_get **description:** Configuration option which has a string value can be retrieved with this function. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_string_set ------------------------- @@ -3270,7 +3269,7 @@ ts.http.config_string_set **description:** This function can be used to overwrite the configuration options. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Http config constants --------------------- @@ -3323,7 +3322,8 @@ Http config constants TS_LUA_CONFIG_HTTP_TRANSACTION_NO_ACTIVITY_TIMEOUT_IN TS_LUA_CONFIG_HTTP_TRANSACTION_NO_ACTIVITY_TIMEOUT_OUT TS_LUA_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_OUT - TS_LUA_CONFIG_HTTP_ORIGIN_MAX_CONNECTIONS + TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX + TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DEAD_SERVER TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES @@ -3331,13 +3331,10 @@ Http config constants TS_LUA_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT TS_LUA_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME TS_LUA_CONFIG_HTTP_DOWN_SERVER_ABORT_THRESHOLD - TS_LUA_CONFIG_HTTP_CACHE_FUZZ_TIME - TS_LUA_CONFIG_HTTP_CACHE_FUZZ_MIN_TIME TS_LUA_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS TS_LUA_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT TS_LUA_CONFIG_HTTP_RESPONSE_SERVER_STR TS_LUA_CONFIG_HTTP_CACHE_HEURISTIC_LM_FACTOR - TS_LUA_CONFIG_HTTP_CACHE_FUZZ_PROBABILITY TS_LUA_CONFIG_HTTP_BACKGROUND_FILL_COMPLETED_THRESHOLD TS_LUA_CONFIG_NET_SOCK_PACKET_MARK_OUT TS_LUA_CONFIG_NET_SOCK_PACKET_TOS_OUT @@ -3347,6 +3344,7 @@ Http config constants TS_LUA_CONFIG_HTTP_FLOW_CONTROL_LOW_WATER_MARK TS_LUA_CONFIG_HTTP_FLOW_CONTROL_HIGH_WATER_MARK TS_LUA_CONFIG_HTTP_CACHE_RANGE_LOOKUP + TS_LUA_CONFIG_HTTP_NORMALIZE_AE TS_LUA_CONFIG_HTTP_DEFAULT_BUFFER_SIZE TS_LUA_CONFIG_HTTP_DEFAULT_BUFFER_WATER_MARK TS_LUA_CONFIG_HTTP_REQUEST_HEADER_MAX_SIZE @@ -3367,10 +3365,42 @@ Http config constants TS_LUA_CONFIG_HTTP_CACHE_OPEN_WRITE_FAIL_ACTION TS_LUA_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS TS_LUA_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES - TS_LUA_CONFIG_HTTP_NORMALIZE_AE + TS_LUA_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY + TS_LUA_CONFIG_HTTP_ATTACH_SERVER_SESSION_TO_CLIENT + TS_LUA_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT + TS_LUA_CONFIG_WEBSOCKET_ACTIVE_TIMEOUT + TS_LUA_CONFIG_HTTP_UNCACHEABLE_REQUESTS_BYPASS_PARENT + TS_LUA_CONFIG_HTTP_PARENT_PROXY_TOTAL_CONNECT_ATTEMPTS + TS_LUA_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_IN + TS_LUA_CONFIG_SRV_ENABLED + TS_LUA_CONFIG_HTTP_FORWARD_CONNECT_METHOD + TS_LUA_CONFIG_SSL_CERT_FILENAME + TS_LUA_CONFIG_SSL_CERT_FILEPATH + TS_LUA_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB + TS_LUA_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER + TS_LUA_CONFIG_HTTP_CACHE_VARY_DEFAULT_TEXT + TS_LUA_CONFIG_HTTP_CACHE_VARY_DEFAULT_IMAGES + TS_LUA_CONFIG_HTTP_CACHE_VARY_DEFAULT_OTHER + TS_LUA_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH + TS_LUA_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_LANGUAGE_MISMATCH + TS_LUA_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_ENCODING_MISMATCH + TS_LUA_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_CHARSET_MISMATCH + TS_LUA_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD + TS_LUA_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME + TS_LUA_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS + TS_LUA_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT + TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE + TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED + TS_LUA_CONFIG_HTTP_ALLOW_HALF_OPEN + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES + TS_LUA_CONFIG_SSL_CLIENT_SNI_POLICY + TS_LUA_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME + TS_LUA_CONFIG_SSL_CLIENT_CA_CERT_FILENAME TS_LUA_CONFIG_LAST_ENTRY -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.timeout_set ------------------- @@ -3390,7 +3420,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Timeout constants ----------------- @@ -3404,7 +3434,7 @@ Timeout constants TS_LUA_TIMEOUT_NO_ACTIVITY -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.client_packet_mark_set ------------------------------ @@ -3423,7 +3453,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_packet_mark_set ------------------------------ @@ -3434,7 +3464,7 @@ ts.http.server_packet_mark_set **description:** This function can be used to set packet mark for server connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.client_packet_tos_set ----------------------------- @@ -3445,7 +3475,7 @@ ts.http.client_packet_tos_set **description:** This function can be used to set packet tos for client connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_packet_tos_set ----------------------------- @@ -3456,7 +3486,7 @@ ts.http.server_packet_tos_set **description:** This function can be used to set packet tos for server connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.client_packet_dscp_set ------------------------------ @@ -3467,7 +3497,7 @@ ts.http.client_packet_dscp_set **description:** This function can be used to set packet dscp for client connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_packet_dscp_set ------------------------------ @@ -3478,7 +3508,7 @@ ts.http.server_packet_dscp_set **description:** This function can be used to set packet dscp for server connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.enable_redirect ----------------------- @@ -3498,7 +3528,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_debug ----------------- @@ -3518,7 +3548,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.cntl_get ---------------- @@ -3533,7 +3563,7 @@ ts.http.cntl_get val = ts.http.cntl_get(TS_LUA_HTTP_CNTL_GET_LOGGING_MODE) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.cntl_set ---------------- @@ -3553,7 +3583,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Http control channel constants ------------------------------ @@ -3567,7 +3597,7 @@ Http control channel constants TS_LUA_HTTP_CNTL_SET_INTERCEPT_RETRY_MODE -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.milestone_get --------------------- @@ -3583,7 +3613,7 @@ of seconds since the beginning of the transaction. val = ts.http.milestone_get(TS_LUA_MILESTONE_SM_START) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Milestone constants ------------------------------ @@ -3617,7 +3647,7 @@ Milestone constants TS_LUA_MILESTONE_TLS_HANDSHAKE_END -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.mgmt.get_counter ------------------- @@ -3631,7 +3661,7 @@ ts.mgmt.get_counter n = ts.mgmt.get_counter('proxy.process.http.incoming_requests') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.mgmt.get_int --------------- @@ -3641,7 +3671,7 @@ ts.mgmt.get_int **description:** This function can be used to retrieve the record value which has a int type. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.mgmt.get_float ----------------- @@ -3651,7 +3681,7 @@ ts.mgmt.get_float **description:** This function can be used to retrieve the record value which has a float type. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.mgmt.get_string ------------------ @@ -3665,7 +3695,7 @@ ts.mgmt.get_string name = ts.mgmt.get_string('proxy.config.product_name') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.stat_create -------------- @@ -3704,7 +3734,7 @@ Here is an example. return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.stat_find ------------ @@ -3715,7 +3745,7 @@ ts.stat_find **description:** This function can be used to find a statistics record given the name. A statistics record table will be returned with 4 functions to increment, decrement, get and set the value. That is similar to ts.stat_create() -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Todo ==== @@ -3729,7 +3759,7 @@ as transaction hook instead. But this will have problem down the road when we ne together in some proper orderings. In the future, we should consider different approach, such as creating and maintaining the lua state in the ATS core. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Notes on Unit Testing Lua scripts for ATS Lua Plugin ==================================================== @@ -3759,11 +3789,11 @@ Reference for further information * luacov - https://luarocks.org/modules/hisham/luacov -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ More docs ========= * https://github.com/portl4t/ts-lua -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ From a6b1ee90fc40e1361e8c91be3d6e5ac89f9319fd Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 13 Feb 2019 14:37:26 +0900 Subject: [PATCH 275/526] Ignore config.cache file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 361ce6d9158..ec01fe126c4 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ build/lt~obsolete.m4 Makefile Makefile-pl +config.cache config.status config.nice config.h From a37f0d91ec0bbc5cd1f0700a69d76df54b9484fc Mon Sep 17 00:00:00 2001 From: scw00 Date: Sat, 26 Jan 2019 17:21:42 +0800 Subject: [PATCH 276/526] XXxClean up ua_entry when client_vc is closed --- proxy/http/HttpSM.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 4450247a744..3aa0cee28dd 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -3287,7 +3287,9 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) ua_txn->set_half_close_flag(true); } + vc_table.remove_entry(this->ua_entry); ua_txn->do_io_close(); + ua_txn = nullptr; } else { ink_assert(ua_buffer_reader != nullptr); ua_txn->release(ua_buffer_reader); From 37f34050616a7a9400fd5bdb713257d2f0e8964a Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Fri, 15 Feb 2019 15:29:17 +0900 Subject: [PATCH 277/526] Fix typo of SNI --- configs/ssl_multicert.config.default | 2 +- doc/admin-guide/files/ssl_multicert.config.en.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/ssl_multicert.config.default b/configs/ssl_multicert.config.default index b3452d0b316..f4fab20638d 100644 --- a/configs/ssl_multicert.config.default +++ b/configs/ssl_multicert.config.default @@ -21,7 +21,7 @@ # dest_ip=ADDRESS # The IP (v4 or v6) address that the certificate should be presented # on. This is now only used as a fallback in the case that the TLS -# SubjectNameIndication extension is not supported. If ADDRESS is +# ServerNameIndication extension is not supported. If ADDRESS is # '*', the certificate will be used as the default fallback if no # other match can be made. # diff --git a/doc/admin-guide/files/ssl_multicert.config.en.rst b/doc/admin-guide/files/ssl_multicert.config.en.rst index e8112a78cb7..e79ae467965 100644 --- a/doc/admin-guide/files/ssl_multicert.config.en.rst +++ b/doc/admin-guide/files/ssl_multicert.config.en.rst @@ -73,7 +73,7 @@ ssl_cert_name=FILENAME[,FILENAME ...] dest_ip=ADDRESS (optional) The IP (v4 or v6) address that the certificate should be presented on. This is now only used as a fallback in the case that the TLS - SubjectNameIndication extension is not supported. If *ADDRESS* is + ServerNameIndication extension is not supported. If *ADDRESS* is `*`, the corresponding certificate will be used as the global default fallback if no other match can be made. The address may contain a port specifier, in which case the corresponding certificate From 953cde65b015e0f914fabdc79153fb9d694b18aa Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Sat, 16 Feb 2019 09:32:07 -0800 Subject: [PATCH 278/526] s3_auth:fixed uncaught exception on invalid input Instead of fixing the URI decoding functionality to handle the exception (re)implemented a check for percent encoding which was what was needed. During signature calculation AWS avoids URI encoding of already encoded query parameters (rfc3986#section-2.4 says "implementations must not percent-encode or decode the same string more than once ...") --- plugins/s3_auth/aws_auth_v4.cc | 63 +++++++++------ .../s3_auth/unit_tests/test_aws_auth_v4.cc | 79 ++++++++++++++----- plugins/s3_auth/unit_tests/test_aws_auth_v4.h | 2 +- 3 files changed, 101 insertions(+), 43 deletions(-) diff --git a/plugins/s3_auth/aws_auth_v4.cc b/plugins/s3_auth/aws_auth_v4.cc index 61f595de412..1df1a6175ec 100644 --- a/plugins/s3_auth/aws_auth_v4.cc +++ b/plugins/s3_auth/aws_auth_v4.cc @@ -103,32 +103,52 @@ uriEncode(const String &in, bool isObjectName) } /** - * @brief URI-decode a character string (AWS specific version, see spec) + * @brief checks if the string is URI-encoded (AWS specific encoding version, see spec) * * @see AWS spec: http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html * - * @todo Consider reusing / converting to TSStringPercentDecode() - * Currently we don't build a library/archive so we could link with the unit-test binary. Also using - * different sets of encode/decode functions during runtime and unit-testing did not seem as a good idea. - * @param in string to be URI decoded - * @return encoded string. + * @note According to the following RFC if the string is encoded and contains '%' it should + * be followed by 2 hexadecimal symbols otherwise '%' should be encoded with %25: + * https://tools.ietf.org/html/rfc3986#section-2.1 + * + * @param in string to be URI checked + * @param isObjectName if true encoding didn't encode '/', kept it as it is. + * @return true if encoded, false not encoded. */ -String -uriDecode(const String &in) +bool +isUriEncoded(const String &in, bool isObjectName) { - std::string result; - result.reserve(in.length()); - size_t i = 0; - while (i < in.length()) { - if (in[i] == '%') { - result += static_cast(std::stoi(in.substr(i + 1, 2), nullptr, 16)); - i += 3; - } else { - result += in[i]; - i++; + for (size_t pos = 0; pos < in.length(); pos++) { + char c = in[pos]; + + if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') { + /* found a unreserved character which should not have been be encoded regardless + * 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', and '~'. */ + continue; + } + + if (' ' == c) { + /* space should have been encoded with %20 if the string was encoded */ + return false; + } + + if ('/' == c && !isObjectName) { + /* if this is not an object name '/' should have been encoded */ + return false; + } + + if ('%' == c) { + if (pos + 2 < in.length() && std::isxdigit(in[pos + 1]) && std::isxdigit(in[pos + 2])) { + /* if string was encoded we should have exactly 2 hexadecimal chars following it */ + return true; + } else { + /* lonely '%' should have been encoded with %25 according to the RFC so likely not encoded */ + return false; + } } } - return result; + + return false; } /** @@ -290,10 +310,7 @@ getCanonicalRequestSha256Hash(TsInterface &api, bool signPayload, const StringSe paramNames.insert(encodedParam); - /* Look for '%' first trying to avoid as many uri-decode calls as possible. - * it is hard to estimate which is more likely use-case - (1) URIs with uri-encoded query parameter - * values or (2) with unencoded which defines the success of this optimization */ - if (nullptr == memchr(value.c_str(), '%', value.length()) || 0 == uriDecode(value).compare(value)) { + if (!isUriEncoded(value, /* isObjectName */ false)) { /* Not URI-encoded */ paramsMap[encodedParam] = uriEncode(value, /* isObjectName */ false); } else { diff --git a/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc b/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc index 7dd4f600880..025cc2bb976 100644 --- a/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc +++ b/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc @@ -69,37 +69,78 @@ TEST_CASE("uriEncode(): encode reserved chars in an object name", "[AWS][auth][u CHECK_FALSE(encoded.compare("%20/%21%22%23%24%25%26%27%28%29%2A%2B%2C%3A%3B%3C%3D%3E%3F%40%5B%5C%5D%5E%60%7B%7C%7D")); } -TEST_CASE("uriDecode(): decode empty input", "[AWS][auth][utility]") +TEST_CASE("isUriEncoded(): check an empty input", "[AWS][auth][utility]") { - String encoded(""); - String decoded = uriDecode(encoded); - CHECK(0 == decoded.length()); /* 0 encoded because of the invalid input */ + CHECK(false == isUriEncoded("")); } -TEST_CASE("uriDecode(): decode unreserved chars", "[AWS][auth][utility]") +TEST_CASE("isUriEncoded(): '%' and nothing else", "[AWS][auth][utility]") +{ + CHECK(false == isUriEncoded("%")); +} + +TEST_CASE("isUriEncoded(): '%' but no hex digits", "[AWS][auth][utility]") +{ + CHECK(false == isUriEncoded("XXX%XXX")); +} + +TEST_CASE("isUriEncoded(): '%' but only one hex digit", "[AWS][auth][utility]") +{ + CHECK(false == isUriEncoded("XXXXX%1XXXXXX")); + CHECK(false == isUriEncoded("XXX%1")); // test end of string case +} + +TEST_CASE("isUriEncoded(): '%' and 2 hex digit", "[AWS][auth][utility]") +{ + CHECK(true == isUriEncoded("XXX%12XXX")); + CHECK(true == isUriEncoded("XXX%12")); // test end of string case +} + +TEST_CASE("isUriEncoded(): space not encoded", "[AWS][auth][utility]") +{ + // Having a space always means it was not encoded. + CHECK(false == isUriEncoded("XXXXX XXXXXX")); +} + +TEST_CASE("isUriEncoded(): '/' in strings which are not object names", "[AWS][auth][utility]") +{ + // This is not an object name so if we have '/' => the string was not encoded. + CHECK(false == isUriEncoded("XXXXX/XXXXXX", /* isObjectName */ false)); + + // There is no '/' and '%2F' shows that it was encoded. + CHECK(true == isUriEncoded("XXXXX%2FXXXXXX", /* isObjectName */ false)); + + // This is not an object name so if we have '/' => the string was not encoded despite '%20' in it. + CHECK(false == isUriEncoded("XXXXX/%20XXXXX", /* isObjectName */ false)); +} + +TEST_CASE("isUriEncoded(): '/' in strings that are object names", "[AWS][auth][utility]") +{ + // This is an object name so having '/' is normal but not enough to conclude if it is encoded or not. + CHECK(false == isUriEncoded("XXXXX/XXXXXX", /* isObjectName */ true)); + + // There is no '/' and '%2F' shows it is encoded. + CHECK(true == isUriEncoded("XXXXX%2FXXXXXX", /* isObjectName */ true)); + + // This is an object name so having '/' is normal and because of '%20' we can conclude it was encoded. + CHECK(true == isUriEncoded("XXXXX/%20XXXXX", /* isObjectName */ true)); +} + +TEST_CASE("isUriEncoded(): no reserved chars in the input", "[AWS][auth][utility]") { const String encoded = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789" "-._~"; - String decoded = uriDecode(encoded); - - CHECK(encoded.length() == encoded.length()); - CHECK_FALSE(decoded.compare("ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789" - "-._~")); + CHECK(false == isUriEncoded(encoded)); } -TEST_CASE("uriDecode(): decode reserved chars", "[AWS][auth][utility]") +TEST_CASE("isUriEncoded(): reserved chars in the input", "[AWS][auth][utility]") { - const String encoded = - "%20%2F%21%22%23%24%25%26%27%28%29%2A%2B%2C%3A%3B%3C%3D%3E%3F%40%5B%5C%5D%5E%60%7B%7C%7D"; /* some printable but - reserved chars */ - String decoded = uriDecode(encoded); + // some printable but reserved chars " /!\"#$%&'()*+,:;<=>?@[\\]^`{|}" + const String encoded = "%20%2F%21%22%23%24%25%26%27%28%29%2A%2B%2C%3A%3B%3C%3D%3E%3F%40%5B%5C%5D%5E%60%7B%7C%7D"; - CHECK(3 * decoded.length() == encoded.length()); /* size of "%NN" = 3 */ - CHECK_FALSE(decoded.compare(" /!\"#$%&'()*+,:;<=>?@[\\]^`{|}")); + CHECK(true == isUriEncoded(encoded)); } /* base16Encode() ************************************************************************************************************** */ diff --git a/plugins/s3_auth/unit_tests/test_aws_auth_v4.h b/plugins/s3_auth/unit_tests/test_aws_auth_v4.h index 5b0ea30dd96..134151797b4 100644 --- a/plugins/s3_auth/unit_tests/test_aws_auth_v4.h +++ b/plugins/s3_auth/unit_tests/test_aws_auth_v4.h @@ -121,7 +121,7 @@ class MockTsInterface : public TsInterface /* Expose the following methods only to the unit tests */ String base16Encode(const char *in, size_t inLen); String uriEncode(const String &in, bool isObjectName = false); -String uriDecode(const String &in); +bool isUriEncoded(const String &in, bool isObjectName = false); String lowercase(const char *in, size_t inLen); const char *trimWhiteSpaces(const char *in, size_t inLen, size_t &newLen); From c2a91894585bd84ec14ae367762dcadb55e0f645 Mon Sep 17 00:00:00 2001 From: scw00 Date: Mon, 18 Feb 2019 16:55:54 +0800 Subject: [PATCH 279/526] Frees disk_vols when volume create failed --- iocore/cache/Cache.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 49337278902..e1416fa7fe4 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -2865,6 +2865,8 @@ cplist_reconfigure() new_cp->disk_vols = (DiskVol **)ats_malloc(gndisks * sizeof(DiskVol *)); memset(new_cp->disk_vols, 0, gndisks * sizeof(DiskVol *)); if (create_volume(config_vol->number, size_in_blocks, config_vol->scheme, new_cp)) { + ats_free(new_cp->disk_vols); + new_cp->disk_vols = nullptr; delete new_cp; return -1; } From 5ad8eec303b5f9c38da0de3775e0aadb7186fc38 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 14 Feb 2019 15:06:58 -0600 Subject: [PATCH 280/526] RecHttp: Convert protocol session pool to use views and MemArena. --- lib/records/I_RecHttp.h | 72 ++++++++++++++++++++--------------------- lib/records/RecHttp.cc | 58 +++++++++++++-------------------- 2 files changed, 58 insertions(+), 72 deletions(-) diff --git a/lib/records/I_RecHttp.h b/lib/records/I_RecHttp.h index 1c95ef72100..c3db5a71000 100644 --- a/lib/records/I_RecHttp.h +++ b/lib/records/I_RecHttp.h @@ -29,8 +29,9 @@ #include "ts/apidefs.h" #include "tscore/ink_assert.h" #include "tscore/IpMap.h" +#include "tscore/MemArena.h" #include -#include +#include /// Load default inbound IP addresses from the configuration file. void RecHttpLoadIp(const char *name, ///< Name of value in configuration file. @@ -48,18 +49,17 @@ void RecHttpLoadIpMap(const char *name, ///< Name of value in configuration file */ class SessionProtocolSet { - typedef SessionProtocolSet self; ///< Self reference type. + using self_type = SessionProtocolSet; ///< Self reference type. /// Storage for the set - a bit vector. - uint32_t m_bits; + uint32_t m_bits = 0; public: - // The right way. - // static int const MAX = sizeof(m_bits) * CHAR_BIT; - // The RHEL5/gcc 4.1.2 way - static int const MAX = sizeof(uint32_t) * 8; + static constexpr int MAX = sizeof(m_bits) * CHAR_BIT; + /// Default constructor. /// Constructs and empty set. - SessionProtocolSet() : m_bits(0) {} + SessionProtocolSet() = default; + uint32_t indexToMask(int idx) const { @@ -72,9 +72,10 @@ class SessionProtocolSet { m_bits |= this->indexToMask(idx); } + /// Mark all the protocols in @a that as present in @a this. void - markIn(self const &that) + markIn(self_type const &that) { m_bits |= that.m_bits; } @@ -86,7 +87,7 @@ class SessionProtocolSet } /// Mark the protocols in @a that as not in @a this. void - markOut(self const &that) + markOut(self_type const &that) { m_bits &= ~(that.m_bits); } @@ -98,7 +99,7 @@ class SessionProtocolSet } /// Test if all the protocols in @a that are in @a this protocol set. bool - contains(self const &that) const + contains(self_type const &that) const { return that.m_bits == (that.m_bits & m_bits); } @@ -117,7 +118,7 @@ class SessionProtocolSet /// Check for intersection. bool - intersects(self const &that) + intersects(self_type const &that) { return 0 != (m_bits & that.m_bits); } @@ -131,7 +132,7 @@ class SessionProtocolSet /// Equality (identical sets). bool - operator==(self const &that) const + operator==(self_type const &that) const { return m_bits == that.m_bits; } @@ -147,52 +148,51 @@ const char *RecNormalizeProtoTag(const char *tag); /** Registered session protocol names. - We do this to avoid lots of string compares. By normalizing the - string names we can just compare their indices in this table. + We do this to avoid lots of string compares. By normalizing the string names we can just compare + their indices in this table. + + @internal To simplify the implementation we limit the maximum number of strings to 32. That will + be sufficient for the forseeable future. We can come back to this if it ever becomes a problem. - @internal To simplify the implementation we limit the maximum - number of strings to 32. That will be sufficient for the forseeable - future. We can come back to this if it ever becomes a problem. + @internal Because we have so few strings we just use a linear search. If the size gets much + larger we should consider doing something more clever. - @internal Because we have so few strings we just use a linear search. - If the size gets much larger we should consider doing something more - clever. + @internal This supports providing constant strings because those strings are exported to the + C API and this logic @b must return exactly those pointers. */ class SessionProtocolNameRegistry { public: - static int const MAX = SessionProtocolSet::MAX; ///< Maximum # of registered names. - static int const INVALID = -1; ///< Normalized invalid index value. + static int constexpr MAX = SessionProtocolSet::MAX; ///< Maximum # of registered names. + static int constexpr INVALID = -1; ///< Normalized invalid index value. + + using TextView = ts::TextView; /// Default constructor. /// Creates empty registry with no names. - SessionProtocolNameRegistry(); - - /// Destructor. - /// Cleans up strings. - ~SessionProtocolNameRegistry(); + SessionProtocolNameRegistry() = default; /** Get the index for @a name, registering it if needed. The name is copied internally. @return The index for the registered @a name. */ - int toIndex(const char *name); + int toIndex(TextView name); /** Get the index for @a name, registering it if needed. The caller @b guarantees @a name is persistent and immutable. @return The index for the registered @a name. */ - int toIndexConst(const char *name); + int toIndexConst(TextView name); /** Convert a @a name to an index. @return The index for @a name or @c INVALID if it is not registered. */ - int indexFor(const char *name) const; + int indexFor(TextView name) const; /** Convert an @a index to the corresponding name. @return A pointer to the name or @c nullptr if the index isn't registered. */ - const char *nameFor(int index) const; + TextView nameFor(int index) const; /// Mark protocols as present in @a sp_set based on the names in @a value. /// The names can be separated by ;/|,: and space. @@ -201,11 +201,9 @@ class SessionProtocolNameRegistry void markIn(const char *value, SessionProtocolSet &sp_set); protected: - unsigned int m_n; ///< Index of first unused slot. - const char *m_names[MAX]; ///< Pointers to registered names. - uint8_t m_flags[MAX]; ///< Flags for each name. - - static uint8_t const F_ALLOCATED = 0x1; ///< Flag for allocated by this instance. + int m_n = 0; ///< Index of first unused slot. + std::array m_names; + ts::MemArena m_arena; ///< Storage for non-constant strings. }; extern SessionProtocolNameRegistry globalSessionProtocolNameRegistry; diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index 64193b8043f..d20ab3aacd3 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -500,7 +500,7 @@ SessionProtocolNameRegistry::markIn(const char *value, SessionProtocolSet &sp_se } else if (0 == strcasecmp(elt, TS_ALPN_PROTOCOL_GROUP_HTTP2)) { sp_set.markIn(HTTP2_PROTOCOL_SET); } else { // user defined - register and mark. - int idx = globalSessionProtocolNameRegistry.toIndex(elt); + int idx = globalSessionProtocolNameRegistry.toIndex(TextView{elt, strlen(elt)}); sp_set.markIn(idx); } } @@ -651,7 +651,8 @@ HttpProxyPort::print(char *out, size_t n) bool sep_p = !need_colon_p; for (int k = 0; k < SessionProtocolSet::MAX; ++k) { if (sp_set.contains(k)) { - zret += snprintf(out + zret, n - zret, "%s%s", sep_p ? ";" : "", globalSessionProtocolNameRegistry.nameFor(k)); + auto name{globalSessionProtocolNameRegistry.nameFor(k)}; + zret += snprintf(out + zret, n - zret, "%s%.*s", sep_p ? ";" : "", static_cast(name.size()), name.data()); sep_p = true; } } @@ -678,10 +679,10 @@ void ts_session_protocol_well_known_name_indices_init() { // register all the well known protocols and get the indices set. - TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 = globalSessionProtocolNameRegistry.toIndexConst(TS_ALPN_PROTOCOL_HTTP_0_9); - TS_ALPN_PROTOCOL_INDEX_HTTP_1_0 = globalSessionProtocolNameRegistry.toIndexConst(TS_ALPN_PROTOCOL_HTTP_1_0); - TS_ALPN_PROTOCOL_INDEX_HTTP_1_1 = globalSessionProtocolNameRegistry.toIndexConst(TS_ALPN_PROTOCOL_HTTP_1_1); - TS_ALPN_PROTOCOL_INDEX_HTTP_2_0 = globalSessionProtocolNameRegistry.toIndexConst(TS_ALPN_PROTOCOL_HTTP_2_0); + TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_0_9}); + TS_ALPN_PROTOCOL_INDEX_HTTP_1_0 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_1_0}); + TS_ALPN_PROTOCOL_INDEX_HTTP_1_1 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_1_1}); + TS_ALPN_PROTOCOL_INDEX_HTTP_2_0 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_2_0}); // Now do the predefined protocol sets. HTTP_PROTOCOL_SET.markIn(TS_ALPN_PROTOCOL_INDEX_HTTP_0_9); @@ -713,30 +714,18 @@ RecNormalizeProtoTag(const char *tag) return findResult == TSProtoTags.end() ? nullptr : findResult->data(); } -SessionProtocolNameRegistry::SessionProtocolNameRegistry() : m_n(0) -{ - memset(m_names, 0, sizeof(m_names)); - memset(&m_flags, 0, sizeof(m_flags)); -} - -SessionProtocolNameRegistry::~SessionProtocolNameRegistry() -{ - for (size_t i = 0; i < m_n; ++i) { - if (m_flags[i] & F_ALLOCATED) { - ats_free(const_cast(m_names[i])); // blech - ats_free won't take a const char* - } - } -} - int -SessionProtocolNameRegistry::toIndex(const char *name) +SessionProtocolNameRegistry::toIndex(ts::TextView name) { int zret = this->indexFor(name); if (INVALID == zret) { - if (m_n < static_cast(MAX)) { - m_names[m_n] = ats_strdup(name); - m_flags[m_n] = F_ALLOCATED; - zret = m_n++; + if (m_n < MAX) { + // Localize the name by copying it in to the arena. + auto text = m_arena.alloc(name.size() + 1); + memcpy(text.data(), name.data(), name.size()); + text.end()[-1] = '\0'; + m_names[m_n] = text.view(); + zret = m_n++; } else { ink_release_assert(!"Session protocol name registry overflow"); } @@ -745,11 +734,11 @@ SessionProtocolNameRegistry::toIndex(const char *name) } int -SessionProtocolNameRegistry::toIndexConst(const char *name) +SessionProtocolNameRegistry::toIndexConst(TextView name) { int zret = this->indexFor(name); if (INVALID == zret) { - if (m_n < static_cast(MAX)) { + if (m_n < MAX) { m_names[m_n] = name; zret = m_n++; } else { @@ -760,18 +749,17 @@ SessionProtocolNameRegistry::toIndexConst(const char *name) } int -SessionProtocolNameRegistry::indexFor(const char *name) const +SessionProtocolNameRegistry::indexFor(TextView name) const { - for (size_t i = 0; i < m_n; ++i) { - if (0 == strcasecmp(name, m_names[i])) { - return i; - } + auto spot = std::find(m_names.begin(), m_names.begin() + m_n, name); + if (spot != m_names.end()) { + return static_cast(spot - m_names.begin()); } return INVALID; } -const char * +ts::TextView SessionProtocolNameRegistry::nameFor(int idx) const { - return 0 <= idx && idx < static_cast(m_n) ? m_names[idx] : nullptr; + return 0 <= idx && idx < m_n ? m_names[idx] : TextView{}; } From 4e05b248b59be2c3ece1a13a725468eb5a760afb Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Mon, 18 Feb 2019 13:51:17 -0600 Subject: [PATCH 281/526] Adds websocket note for remap config --- doc/admin-guide/files/remap.config.en.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/admin-guide/files/remap.config.en.rst b/doc/admin-guide/files/remap.config.en.rst index bd7ee3d21da..6f246e66d09 100644 --- a/doc/admin-guide/files/remap.config.en.rst +++ b/doc/admin-guide/files/remap.config.en.rst @@ -107,6 +107,8 @@ Traffic Server recognizes three space-delimited fields: ``type``, where ``scheme`` is ``http``, ``https``, ``ws`` or ``wss``. + .. note:: A remap rule for requests that upgrade from HTTP to WebSocket still require a remap rule with the ``ws`` or ``wss`` scheme. + .. _remap-config-precedence: From 0b39c1db4fbfa7b9cdc928429852c3bd67c55f52 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Mon, 18 Feb 2019 13:56:03 -0600 Subject: [PATCH 282/526] Fixes formatting --- doc/admin-guide/files/remap.config.en.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/admin-guide/files/remap.config.en.rst b/doc/admin-guide/files/remap.config.en.rst index 6f246e66d09..a37aaff3c54 100644 --- a/doc/admin-guide/files/remap.config.en.rst +++ b/doc/admin-guide/files/remap.config.en.rst @@ -87,7 +87,7 @@ Traffic Server recognizes three space-delimited fields: ``type``, notify the browser of the URL change for the current request only (by returning an HTTP status code 307). - .. note: use the ``regex_`` prefix to indicate that the line has a regular expression (regex). + .. note:: use the ``regex_`` prefix to indicate that the line has a regular expression (regex). .. _remap-config-format-target: @@ -119,7 +119,7 @@ Remap rules are not processed top-down, but based on an internal priority. Once these rules are executed we pick the first match based on configuration file parse order. -1. ``map_with_recv_port`` and ```regex_map_with_recv_port``` +1. ``map_with_recv_port`` and ``regex_map_with_recv_port`` #. ``map`` and ``regex_map`` and ``reverse_map`` #. ``redirect`` and ``redirect_temporary`` #. ``regex_redirect`` and ``regex_redirect_temporary`` From 0781e91fc7af2ba626154382b2e2a6853906517b Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 15 Feb 2019 11:21:35 -0600 Subject: [PATCH 283/526] Build fix: Fix build issue for -O3 on CI. --- lib/tsconfig/TsValue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tsconfig/TsValue.h b/lib/tsconfig/TsValue.h index 59d7d4e30b8..fbbf473e1c6 100644 --- a/lib/tsconfig/TsValue.h +++ b/lib/tsconfig/TsValue.h @@ -217,7 +217,7 @@ namespace detail { ValueType getType() const; protected: ValueType _type; ///< Type of value. - ValueIndex _parent; ///< Table index of parent value. + ValueIndex _parent = 0; ///< Table index of parent value. ConstBuffer _text; ///< Text of value (if scalar). ConstBuffer _name; ///< Local name of value, if available. size_t _local_index; ///< Index among siblings. From 390ebe55712cce687a07427311a933fbaf073438 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 15 Feb 2019 11:35:24 -0600 Subject: [PATCH 284/526] Build: Fix array bounds error under -O3. --- proxy/hdrs/HdrHeap.cc | 2 +- proxy/hdrs/HdrHeap.h | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index cde0e9f2362..edbc5781254 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -962,7 +962,7 @@ HdrHeap::unmarshal(int buf_length, int obj_type, HdrHeapObjImpl **found_obj, Ref inline bool HdrHeap::attach_str_heap(char const *h_start, int h_len, RefCountObj *h_ref_obj, int *index) { - if (static_cast(*index) >= HDR_BUF_RONLY_HEAPS) { + if (*index >= static_cast(HDR_BUF_RONLY_HEAPS)) { return false; } diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index 6dd7deb3a14..98113626bc4 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -206,20 +206,20 @@ class HdrHeap // by a heap consolidation. Does NOT lock for Multi-Threaed // access! void - lock_ronly_str_heap(int i) + lock_ronly_str_heap(unsigned i) { m_ronly_heap[i].m_locked = true; } void - unlock_ronly_str_heap(int i) + unlock_ronly_str_heap(unsigned i) { m_ronly_heap[i].m_locked = false; // INKqa11238 // Move slot i to the first available slot in m_ronly_heap[]. // The move is necessary because the rest of the code assumes // heaps are always allocated in order. - for (int j = 0; j < i; j++) { + for (unsigned j = 0; j < i; j++) { if (m_ronly_heap[j].m_heap_start == nullptr) { // move slot i to slot j m_ronly_heap[j].m_ref_count_ptr = m_ronly_heap[i].m_ref_count_ptr; @@ -230,6 +230,7 @@ class HdrHeap m_ronly_heap[i].m_heap_start = nullptr; m_ronly_heap[i].m_heap_len = 0; m_ronly_heap[i].m_locked = false; + break; // Did the move, time to go. } } } From 9f5396ee87c48e645ffc712f18de1d32849341dd Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 4 Jun 2018 10:09:41 -0700 Subject: [PATCH 285/526] Adds logging around various config file loads Also changes the messaging of all config file loads to be consistent Addresses issue #1348 --- iocore/cache/CacheHosting.cc | 11 +++++++++++ iocore/cache/Store.cc | 6 ++++++ iocore/dns/SplitDNS.cc | 5 +++++ iocore/net/SSLConfig.cc | 4 ++-- iocore/net/SSLSNIConfig.cc | 8 ++++---- iocore/net/SSLUtils.cc | 2 +- lib/records/P_RecCore.cc | 2 +- proxy/CacheControl.cc | 6 +++++- proxy/IPAllow.cc | 4 +++- proxy/ParentSelection.cc | 8 ++++++++ proxy/Plugin.cc | 8 +++++++- proxy/ReverseProxy.cc | 11 +++++++---- proxy/logging/LogConfig.cc | 6 +++--- tests/gold_tests/tls/tls_client_cert.test.py | 2 +- tests/gold_tests/tls/tls_tunnel.test.py | 6 +++--- 15 files changed, 67 insertions(+), 22 deletions(-) diff --git a/iocore/cache/CacheHosting.cc b/iocore/cache/CacheHosting.cc index 08573d7cdfc..a57d3bbaf10 100644 --- a/iocore/cache/CacheHosting.cc +++ b/iocore/cache/CacheHosting.cc @@ -244,6 +244,8 @@ int fstat_wrapper(int fd, struct stat *s); int CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_buf) { + Note("hosting.config loading ..."); + // Table build locals Tokenizer bufTok("\n"); tok_iter_state i_state; @@ -323,6 +325,7 @@ CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_bu if (gen_host_rec.Init(type)) { Warning("Problems encountered while initializing the Generic Volume"); } + Note("hosting.config finished loading"); return 0; } @@ -370,6 +373,8 @@ CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_bu last = current; current = current->next; ats_free(last); + + Note("hosting.config finished loading"); } if (!generic_rec_initd) { @@ -591,14 +596,20 @@ ConfigVolumes::read_config_file() config_path = RecConfigReadConfigPath("proxy.config.cache.volume_filename"); ink_release_assert(config_path); + Note("volume.config loading ..."); + file_buf = readIntoBuffer(config_path, "[CacheVolition]", nullptr); if (file_buf == nullptr) { + Error("volume.config failed to load"); Warning("Cannot read the config file: %s", (const char *)config_path); return; } BuildListFromString(config_path, file_buf); ats_free(file_buf); + + Note("volume.config finished loading"); + return; } diff --git a/iocore/cache/Store.cc b/iocore/cache/Store.cc index d3028160b6f..bcafbd4b270 100644 --- a/iocore/cache/Store.cc +++ b/iocore/cache/Store.cc @@ -327,9 +327,11 @@ Store::read_config() ats_scoped_fd fd; ats_scoped_str storage_path(RecConfigReadConfigPath("proxy.config.cache.storage_filename", "storage.config")); + Note("storage.config loading ..."); Debug("cache_init", "Store::read_config, fd = -1, \"%s\"", (const char *)storage_path); fd = ::open(storage_path, O_RDONLY); if (fd < 0) { + Error("storage.config failed to load"); return Result::failure("open %s: %s", (const char *)storage_path, strerror(errno)); } @@ -369,6 +371,7 @@ Store::read_config() if (ParseRules::is_digit(*e)) { if ((size = ink_atoi64(e)) <= 0) { delete sd; + Error("storage.config failed to load"); return Result::failure("failed to parse size '%s'", e); } } else if (0 == strncasecmp(HASH_BASE_STRING_KEY, e, sizeof(HASH_BASE_STRING_KEY) - 1)) { @@ -386,6 +389,7 @@ Store::read_config() } if (!*e || !ParseRules::is_digit(*e) || 0 >= (volume_num = ink_atoi(e))) { delete sd; + Error("storage.config failed to load"); return Result::failure("failed to parse volume number '%s'", e); } } @@ -437,6 +441,8 @@ Store::read_config() sd = nullptr; // these are all used. sort(); + Note("storage.config finished loading"); + return Result::ok(); } diff --git a/iocore/dns/SplitDNS.cc b/iocore/dns/SplitDNS.cc index 9834a8be6b9..998ef74de6d 100644 --- a/iocore/dns/SplitDNS.cc +++ b/iocore/dns/SplitDNS.cc @@ -132,12 +132,15 @@ SplitDNSConfig::reconfigure() return; } + Note("splitdns.config loading ..."); + SplitDNS *params = new SplitDNS; params->m_SplitDNSlEnable = gsplit_dns_enabled; params->m_DNSSrvrTable = new DNS_table("proxy.config.dns.splitdns.filename", modulePrefix, &sdns_dest_tags); if (nullptr == params->m_DNSSrvrTable || (0 == params->m_DNSSrvrTable->getEntryCount())) { + Error("splitdns.config failed to load"); Warning("No NAMEDs provided! Disabling SplitDNS"); gsplit_dns_enabled = 0; delete params; @@ -157,6 +160,8 @@ SplitDNSConfig::reconfigure() if (is_debug_tag_set("splitdns_config")) { SplitDNSConfig::print(); } + + Note("splitdns.config finished loading"); } /* -------------------------------------------------------------- diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 71d79fb00ce..2d67244e48c 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -534,9 +534,9 @@ SSLCertificateConfig::reconfigure() } if (retStatus) { - Note("ssl_multicert.config done reloading!"); + Note("ssl_multicert.config finished loading"); } else { - Note("failed to reload ssl_multicert.config"); + Error("ssl_multicert.config failed to load"); } return retStatus; diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 1925374e2c7..006f56a193b 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -124,11 +124,11 @@ SNIConfigParams::Initialize() { sni_filename = ats_stringdup(RecConfigReadConfigPath("proxy.config.ssl.servername.filename")); - Note("loading %s", sni_filename); + Note("ssl_server_name.yaml loading ..."); struct stat sbuf; if (stat(sni_filename, &sbuf) == -1 && errno == ENOENT) { - Note("failed to reload ssl_server_name.yaml"); + Note("ssl_server_name.yaml failed to load"); Warning("Loading SNI configuration - filename: %s doesn't exist", sni_filename); return 1; } @@ -137,12 +137,12 @@ SNIConfigParams::Initialize() if (!zret.isOK()) { std::stringstream errMsg; errMsg << zret; - Error("failed to load ssl_server_name.yaml: %s", errMsg.str().c_str()); + Error("ssl_server_name.yaml failed to load: %s", errMsg.str().c_str()); return 1; } loadSNIConfig(); - Note("ssl_server_name.yaml done reloading!"); + Note("ssl_server_name.yaml finished loading"); return 0; } diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index cf87bd41273..53255b1f002 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -2212,7 +2212,7 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l const matcher_tags sslCertTags = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, false}; - Note("loading SSL certificate configuration from %s", params->configFilePath); + Note("ssl_multicert.config loading ..."); if (params->configFilePath) { file_buf = readIntoBuffer(params->configFilePath, __func__, nullptr); diff --git a/lib/records/P_RecCore.cc b/lib/records/P_RecCore.cc index 1da0cc9a067..e89ccb9771e 100644 --- a/lib/records/P_RecCore.cc +++ b/lib/records/P_RecCore.cc @@ -641,7 +641,7 @@ RecReadConfigFile(bool inc_version) // lock our hash table ink_rwlock_wrlock(&g_records_rwlock); - // Parse the actual fileand hash the values. + // Parse the actual file and hash the values. RecConfigFileParse(g_rec_config_fpath, RecConsumeConfigEntry, inc_version); // release our hash table diff --git a/proxy/CacheControl.cc b/proxy/CacheControl.cc index 5d5da5efde3..6e78ed0ebbf 100644 --- a/proxy/CacheControl.cc +++ b/proxy/CacheControl.cc @@ -144,17 +144,21 @@ initCacheControl() // // Called when the cache.conf file changes. Since it called // infrequently, we do the load of new file as blocking I/O and -// lock aquire is also blocking +// lock acquire is also blocking // void reloadCacheControl() { + Note("cache.config loading ..."); + CC_table *newTable; Debug("cache_control", "cache.config updated, reloading"); eventProcessor.schedule_in(new CC_FreerContinuation(CacheControlTable), CACHE_CONTROL_TIMEOUT, ET_CACHE); newTable = new CC_table("proxy.config.cache.control.filename", modulePrefix, &http_dest_tags); ink_atomic_swap(&CacheControlTable, newTable); + + Note("cache.config finished loading"); } void diff --git a/proxy/IPAllow.cc b/proxy/IPAllow.cc index ed208d99b72..3419258280a 100644 --- a/proxy/IPAllow.cc +++ b/proxy/IPAllow.cc @@ -77,12 +77,14 @@ IpAllow::reconfigure() { self_type *new_table; - Note("ip_allow.config updated, reloading"); + Note("ip_allow.config loading ..."); new_table = new self_type("proxy.config.cache.ip_allow.filename"); new_table->BuildTable(); configid = configProcessor.set(configid, new_table); + + Note("ip_allow.config finished loading"); } IpAllow * diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index af463f993ab..6e7e963b579 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -279,6 +279,8 @@ ParentConfig::startup() void ParentConfig::reconfigure() { + Note("parent.config loading ..."); + ParentConfigParams *params = nullptr; // Allocate parent table @@ -292,6 +294,8 @@ ParentConfig::reconfigure() if (is_debug_tag_set("parent_config")) { ParentConfig::print(); } + + Note("parent.config finished loading"); } // void ParentConfig::print @@ -895,6 +899,8 @@ setup_socks_servers(ParentRecord *rec_arr, int len) void SocksServerConfig::reconfigure() { + Note("socks.config loading ..."); + char *default_val = nullptr; int retry_time = 30; int fail_threshold; @@ -936,6 +942,8 @@ SocksServerConfig::reconfigure() if (is_debug_tag_set("parent_config")) { SocksServerConfig::print(); } + + Note("socks.config finished loading"); } void diff --git a/proxy/Plugin.cc b/proxy/Plugin.cc index 174c6d07390..e6b43e58a46 100644 --- a/proxy/Plugin.cc +++ b/proxy/Plugin.cc @@ -234,10 +234,11 @@ plugin_init(bool validateOnly) INIT_ONCE = false; } + Note("plugin.config loading ..."); path = RecConfigReadConfigPath(nullptr, "plugin.config"); fd = open(path, O_RDONLY); if (fd < 0) { - Warning("unable to open plugin config file '%s': %d, %s", (const char *)path, errno, strerror(errno)); + Warning("plugin.config failed to load: %d, %s", errno, strerror(errno)); return false; } @@ -312,5 +313,10 @@ plugin_init(bool validateOnly) } close(fd); + if (retVal) { + Note("plugin.config finished loading"); + } else { + Error("plugin.config failed to load"); + } return retVal; } diff --git a/proxy/ReverseProxy.cc b/proxy/ReverseProxy.cc index 99c7677c4fd..8fe9da8f933 100644 --- a/proxy/ReverseProxy.cc +++ b/proxy/ReverseProxy.cc @@ -64,9 +64,11 @@ init_reverse_proxy() reconfig_mutex = new_ProxyMutex(); rewrite_table = new UrlRewrite(); + Note("remap.config loading ..."); if (!rewrite_table->is_valid()) { - Fatal("unable to load remap.config"); + Fatal("remap.config failed to load"); } + Note("remap.config finished loading"); REC_RegisterConfigUpdateFunc("proxy.config.url_remap.filename", url_rewrite_CB, (void *)FILE_CHANGED); REC_RegisterConfigUpdateFunc("proxy.config.proxy_name", url_rewrite_CB, (void *)TSNAME_CHANGED); @@ -134,10 +136,11 @@ reloadUrlRewrite() { UrlRewrite *newTable, *oldTable; + Note("remap.config loading ..."); Debug("url_rewrite", "remap.config updated, reloading..."); newTable = new UrlRewrite(); if (newTable->is_valid()) { - static const char *msg = "remap.config done reloading!"; + static const char *msg = "remap.config finished loading"; // Hold at least one lease, until we reload the configuration newTable->acquire(); @@ -154,11 +157,11 @@ reloadUrlRewrite() Note("%s", msg); return true; } else { - static const char *msg = "failed to reload remap.config, not replacing!"; + static const char *msg = "remap.config failed to load"; delete newTable; Debug("url_rewrite", "%s", msg); - Warning("%s", msg); + Error("%s", msg); return false; } } diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc index 757c0bb4ccf..4b09754f612 100644 --- a/proxy/logging/LogConfig.cc +++ b/proxy/logging/LogConfig.cc @@ -972,14 +972,14 @@ LogConfig::evaluate_config() return false; } - Note("loading logging.yaml"); + Note("logging.yaml loading ..."); YamlLogConfig y(this); bool zret = y.parse(path.get()); if (zret) { - Note("logging.yaml done reloading!"); + Note("logging.yaml finished loading"); } else { - Note("failed to reload logging.yaml"); + Note("logging.yaml failed to load"); } return zret; diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index 58bb03eafcc..6c6bcc8e3e7 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -178,7 +178,7 @@ # At that point the new ssl_server_name settings are ready to go def ssl_server_name_reload_done(tsenv): def done_reload(process, hasRunFor, **kw): - cmd = "grep 'ssl_server_name.yaml done reloading!' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) + cmd = "grep 'ssl_server_name.yaml finished loading' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) retval = subprocess.run(cmd, shell=True, env=tsenv) if retval.returncode == 0: cmd ="if [ -f {0}/test.out -a \"`cat {0}/test.out`\" = \"2\" ] ; then true; else false; fi".format(Test.RunDirectory) diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index 5f653db20cc..36dc0125c53 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -33,7 +33,7 @@ server_bar = Test.MakeOriginServer("server_bar", ssl=True) server2 = Test.MakeOriginServer("server2") -request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "foo ok"} response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "bar ok"} @@ -75,7 +75,7 @@ }) # foo.com should not terminate. Just tunnel to server_foo -# bar.com should terminate. +# bar.com should terminate. # empty SNI should tunnel to server_bar ts.Disk.ssl_server_name_yaml.AddLines([ '- fqdn: foo.com', @@ -160,7 +160,7 @@ # At that point the new ssl_server_name settings are ready to go def ssl_server_name_reload_done(tsenv): def done_reload(process, hasRunFor, **kw): - cmd = "grep 'ssl_server_name.yaml done reloading!' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) + cmd = "grep 'ssl_server_name.yaml finished loading' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) retval = subprocess.run(cmd, shell=True, env=tsenv) if retval.returncode == 0: cmd ="if [ -f {0}/test.out -a \"`cat {0}/test.out`\" = \"2\" ] ; then true; else false; fi".format(Test.RunDirectory) From 53fd833f6b0d3022def83cd3b6f079a2dc945699 Mon Sep 17 00:00:00 2001 From: dyrock Date: Mon, 4 Feb 2019 16:04:08 -0600 Subject: [PATCH 286/526] Added TS_SSL_CLIENT_HELLO_HOOK and docs Added new test for client hello hook --- .../hooks-and-transactions/ssl-hooks.en.rst | 21 +++- include/ts/apidefs.h.in | 2 + iocore/net/P_SSLNetVConnection.h | 16 ++- iocore/net/P_SSLUtils.h | 1 + iocore/net/SSLNetVConnection.cc | 82 ++++++++++++--- iocore/net/SSLUtils.cc | 12 ++- proxy/InkAPIInternal.h | 1 + proxy/http/HttpDebugNames.cc | 4 + src/traffic_server/InkAPITest.cc | 1 + .../tls_hooks/gold/client-hello-1.gold | 0 .../tls_hooks/gold/ts-cert-1-im-2.gold | 2 +- .../gold_tests/tls_hooks/gold/ts-cert-1.gold | 2 +- .../gold_tests/tls_hooks/gold/ts-cert-2.gold | 2 +- .../tls_hooks/gold/ts-cert-im-1.gold | 2 +- .../tls_hooks/gold/ts-client-hello-1.gold | 3 + .../tls_hooks/gold/ts-client-hello-2.gold | 5 + .../gold/ts-client-hello-delayed-1.gold | 4 + .../tls_hooks/gold/ts-out-delay-start-2.gold | 2 +- .../tls_hooks/gold/ts-preaccept-1.gold | 2 +- .../tls_hooks/gold/ts-preaccept-2.gold | 2 +- .../ts-preaccept-delayed-1-immdate-2.gold | 2 +- .../gold/ts-preaccept-delayed-1.gold | 2 +- .../gold/ts-preaccept1-sni1-cert1.gold | 2 +- tests/gold_tests/tls_hooks/gold/ts-sni-1.gold | 2 +- tests/gold_tests/tls_hooks/gold/ts-sni-2.gold | 2 +- .../gold_tests/tls_hooks/tls_hooks16.test.py | 79 +++++++++++++++ .../gold_tests/tls_hooks/tls_hooks17.test.py | 79 +++++++++++++++ .../gold_tests/tls_hooks/tls_hooks18.test.py | 83 ++++++++++++++++ tests/tools/plugins/ssl_hook_test.cc | 99 +++++++++++++++---- tests/tools/plugins/test_hooks.cc | 2 - 30 files changed, 463 insertions(+), 55 deletions(-) create mode 100644 tests/gold_tests/tls_hooks/gold/client-hello-1.gold create mode 100644 tests/gold_tests/tls_hooks/gold/ts-client-hello-1.gold create mode 100644 tests/gold_tests/tls_hooks/gold/ts-client-hello-2.gold create mode 100644 tests/gold_tests/tls_hooks/gold/ts-client-hello-delayed-1.gold create mode 100644 tests/gold_tests/tls_hooks/tls_hooks16.test.py create mode 100644 tests/gold_tests/tls_hooks/tls_hooks17.test.py create mode 100644 tests/gold_tests/tls_hooks/tls_hooks18.test.py diff --git a/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst b/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst index a21e6365e05..1b7d49c7c0c 100644 --- a/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst +++ b/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst @@ -60,6 +60,12 @@ TS_VCONN_CLOSE_HOOK This hook is invoked after the SSL handshake is done and when the IO is closing. The TSVConnArgs should be cleaned up here. A callback at this point must reenable. +TS_SSL_CLIENT_HELLO_HOOK +------------------------ +This hook is called when the client hello arrived for the TLS handshake. If called it will always be called after TS_VCONN_START_HOOK. The plugin callback can execute code to examine client hello information. + +TLS handshake processing will pause until the hook callback executes :c:func:`TSVConnReenable()`. + TS_SSL_SERVERNAME_HOOK ---------------------- @@ -87,7 +93,7 @@ a certificate. TS_SSL_VERIFY_CLIENT_HOOK ------------------------- -This hook is called when a client connects to Traffic Server and presents a +This hook is called when a client connects to Traffic Server and presents a client certificate in the case of a mutual TLS handshake. The callback can get the SSL object from the TSVConn argument and use that to access the client certificate and make any additional checks. @@ -110,7 +116,7 @@ for pausing processing during the certificate verify callback. TS_VCONN_OUTBOUND_START_HOOK ---------------------------- -This hook is invoked after ATS has connected to the upstream server and before the SSL handshake has started. This gives the plugin the option of +This hook is invoked after ATS has connected to the upstream server and before the SSL handshake has started. This gives the plugin the option of overriding the default SSL connection options on the SSL object. In theory this hook could apply and be useful for non-SSL connections as well, but at this point this hook is only called in the SSL sequence. @@ -137,14 +143,19 @@ TLS Inbound Hook State Diagram TS_VCONN_START_HOOK -> HANDSHAKE_HOOKS_PRE_INVOKE; HANDSHAKE_HOOKS_PRE_INVOKE -> TSVConnReenable; TSVConnReenable -> HANDSHAKE_HOOKS_PRE; + TS_SSL_CLIENT_HELLO_HOOK -> HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE; + HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE -> TSVConnReenable2; + TSVConnReenable2 -> HANDSHAKE_HOOKS_CLIENT_HELLO; + HANDSHAKE_HOOKS_CLIENT_HELLO -> TS_SSL_CLIENT_HELLO_HOOK; + HANDSHAKE_HOOKS_CLIENT_HELLO -> TS_SSL_SERVERNAME_HOOK; TS_SSL_SERVERNAME_HOOK -> HANDSHAKE_HOOKS_SNI; HANDSHAKE_HOOKS_SNI -> TS_SSL_SERVERNAME_HOOK; HANDSHAKE_HOOKS_SNI -> TS_SSL_CERT_HOOK; HANDSHAKE_HOOKS_SNI -> HANDSHAKE_HOOKS_DONE; HANDSHAKE_HOOKS_CERT -> TS_SSL_CERT_HOOK; TS_SSL_CERT_HOOK -> HANDSHAKE_HOOKS_CERT_INVOKE; - HANDSHAKE_HOOKS_CERT_INVOKE -> TSVConnReenable2; - TSVConnReenable2 -> HANDSHAKE_HOOKS_CERT; + HANDSHAKE_HOOKS_CERT_INVOKE -> TSVConnReenable3; + TSVConnReenable3 -> HANDSHAKE_HOOKS_CERT; HANDSHAKE_HOOKS_CERT -> TS_SSL_VERIFY_CLIENT_HOOK; HANDSHAKE_HOOKS_SNI -> TS_SSL_VERIFY_CLIENT_HOOK; @@ -159,6 +170,8 @@ TLS Inbound Hook State Diagram HANDSHAKE_HOOKS_PRE [shape=box]; HANDSHAKE_HOOKS_PRE_INVOKE [shape=box]; + HANDSHAKE_HOOKS_CLIENT_HELLO [shape=box]; + HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE [shape=box]; HANDSHAKE_HOOKS_SNI [shape=box]; HANDSHAKE_HOOKS_VERIFY [shape=box]; HANDSHAKE_HOOKS_CERT [shape=box]; diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 7adf619de29..2b1e5d2074f 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -286,6 +286,7 @@ typedef enum { TS_VCONN_START_HOOK = TS_SSL_FIRST_HOOK, TS_VCONN_PRE_ACCEPT_HOOK = TS_VCONN_START_HOOK, // Deprecated but compatible for now. TS_VCONN_CLOSE_HOOK, + TS_SSL_CLIENT_HELLO_HOOK, TS_SSL_SNI_HOOK, TS_SSL_CERT_HOOK = TS_SSL_SNI_HOOK, TS_SSL_SERVERNAME_HOOK, @@ -494,6 +495,7 @@ typedef enum { TS_EVENT_SSL_SERVERNAME = 60204, TS_EVENT_SSL_VERIFY_SERVER = 60205, TS_EVENT_SSL_VERIFY_CLIENT = 60206, + TS_EVENT_SSL_CLIENT_HELLO = 60207, TS_EVENT_MGMT_UPDATE = 60300 } TSEvent; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 4de0038f751..25e84b57611 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -241,9 +241,19 @@ class SSLNetVConnection : public UnixNetVConnection } } break; - case HANDSHAKE_HOOKS_SNI: + case HANDSHAKE_HOOKS_CLIENT_HELLO: + case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: if (eventId == TS_EVENT_VCONN_START) { retval = true; + } else if (eventId == TS_EVENT_SSL_CLIENT_HELLO) { + if (curHook) { + retval = true; + } + } + break; + case HANDSHAKE_HOOKS_SNI: + if (eventId == TS_EVENT_VCONN_START || eventId == TS_EVENT_SSL_CLIENT_HELLO) { + retval = true; } else if (eventId == TS_EVENT_SSL_SERVERNAME) { if (curHook) { retval = true; @@ -252,7 +262,7 @@ class SSLNetVConnection : public UnixNetVConnection break; case HANDSHAKE_HOOKS_CERT: case HANDSHAKE_HOOKS_CERT_INVOKE: - if (eventId == TS_EVENT_VCONN_START || eventId == TS_EVENT_SSL_SERVERNAME) { + if (eventId == TS_EVENT_VCONN_START || eventId == TS_EVENT_SSL_CLIENT_HELLO || eventId == TS_EVENT_SSL_SERVERNAME) { retval = true; } else if (eventId == TS_EVENT_SSL_CERT) { if (curHook) { @@ -394,6 +404,8 @@ class SSLNetVConnection : public UnixNetVConnection enum SSLHandshakeHookState { HANDSHAKE_HOOKS_PRE, HANDSHAKE_HOOKS_PRE_INVOKE, + HANDSHAKE_HOOKS_CLIENT_HELLO, + HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE, HANDSHAKE_HOOKS_SNI, HANDSHAKE_HOOKS_CERT, HANDSHAKE_HOOKS_CERT_INVOKE, diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index 6990db6e4d6..e70dbc551c4 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -85,6 +85,7 @@ enum SSL_Stats { /* error stats */ ssl_error_want_write, ssl_error_want_read, + ssl_error_want_client_hello_cb, ssl_error_want_x509_lookup, ssl_error_syscall, ssl_error_read_eos, diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 14712eee096..5d08d3ddb4c 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -265,6 +265,13 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) SSL_INCREMENT_DYN_STAT(ssl_error_want_read); Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read)"); break; +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: + event = SSL_READ_WOULD_BLOCK; + SSL_INCREMENT_DYN_STAT(ssl_error_want_client_hello_cb); + Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read/client hello cb)"); + break; +#endif case SSL_ERROR_WANT_X509_LOOKUP: event = SSL_READ_WOULD_BLOCK; SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); @@ -806,6 +813,9 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf Debug("ssl.error", "SSL_write-SSL_ERROR_WANT_READ"); break; case SSL_ERROR_WANT_WRITE: +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: +#endif case SSL_ERROR_WANT_X509_LOOKUP: { if (SSL_ERROR_WANT_WRITE == err) { SSL_INCREMENT_DYN_STAT(ssl_error_want_write); @@ -813,7 +823,11 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf } else if (SSL_ERROR_WANT_X509_LOOKUP == err) { SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); } - +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + else if (SSL_ERROR_WANT_CLIENT_HELLO_CB == err) { + SSL_INCREMENT_DYN_STAT(ssl_error_want_client_hello_cb); + } +#endif needs |= EVENTIO_WRITE; num_really_written = -EAGAIN; Debug("ssl.error", "SSL_write-SSL_ERROR_WANT_WRITE"); @@ -1098,7 +1112,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) { // Continue on if we are in the invoked state. The hook has not yet reenabled if (sslHandshakeHookState == HANDSHAKE_HOOKS_CERT_INVOKE || sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE || - sslHandshakeHookState == HANDSHAKE_HOOKS_PRE_INVOKE) { + sslHandshakeHookState == HANDSHAKE_HOOKS_PRE_INVOKE || sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE) { return SSL_WAIT_FOR_HOOK; } @@ -1110,9 +1124,10 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) } else { curHook = curHook->next(); } - // If no more hooks, move onto SNI + // If no more hooks, move onto CLIENT HELLO + if (nullptr == curHook) { - sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; + sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO; } else { sslHandshakeHookState = HANDSHAKE_HOOKS_PRE_INVOKE; ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_START, this); @@ -1294,7 +1309,10 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) case SSL_ERROR_WANT_READ: return SSL_HANDSHAKE_WANT_READ; - +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: + return EVENT_CONT; +#endif // This value is only defined in openssl has been patched to // enable the sni callback to break out of the SSL_accept processing #ifdef SSL_ERROR_WANT_SNI_RESOLVE @@ -1405,7 +1423,12 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) SSL_INCREMENT_DYN_STAT(ssl_error_want_read); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_READ"); return SSL_HANDSHAKE_WANT_READ; - +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: + SSL_INCREMENT_DYN_STAT(ssl_error_want_client_hello_cb); + Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_CLIENT_HELLO_CB"); + break; +#endif case SSL_ERROR_WANT_X509_LOOKUP: SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_X509_LOOKUP"); @@ -1516,6 +1539,9 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) case HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE: sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE; break; + case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: + sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO; + break; case HANDSHAKE_HOOKS_CERT_INVOKE: sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; break; @@ -1541,7 +1567,10 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) } if (curHook != nullptr) { // Invoke the hook and return, wait for next reenable - if (sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_CERT) { + if (sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_HELLO) { + sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE; + curHook->invoke(TS_EVENT_SSL_CLIENT_HELLO, this); + } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_CERT) { sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE; curHook->invoke(TS_EVENT_SSL_VERIFY_CLIENT, this); } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_CERT) { @@ -1573,6 +1602,10 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) switch (this->sslHandshakeHookState) { case HANDSHAKE_HOOKS_PRE: case HANDSHAKE_HOOKS_PRE_INVOKE: + sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO; + break; + case HANDSHAKE_HOOKS_CLIENT_HELLO: + case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; break; case HANDSHAKE_HOOKS_SNI: @@ -1620,15 +1653,18 @@ bool SSLNetVConnection::callHooks(TSEvent eventId) { // Only dealing with the SNI/CERT hook so far. - ink_assert(eventId == TS_EVENT_SSL_CERT || eventId == TS_EVENT_SSL_SERVERNAME || eventId == TS_EVENT_SSL_VERIFY_SERVER || - eventId == TS_EVENT_SSL_VERIFY_CLIENT || eventId == TS_EVENT_VCONN_CLOSE || eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE); + ink_assert(eventId == TS_EVENT_SSL_CLIENT_HELLO || eventId == TS_EVENT_SSL_CERT || eventId == TS_EVENT_SSL_SERVERNAME || + eventId == TS_EVENT_SSL_VERIFY_SERVER || eventId == TS_EVENT_SSL_VERIFY_CLIENT || eventId == TS_EVENT_VCONN_CLOSE || + eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE); Debug("ssl", "callHooks sslHandshakeHookState=%d eventID=%d", this->sslHandshakeHookState, eventId); // Move state if it is appropriate switch (this->sslHandshakeHookState) { case HANDSHAKE_HOOKS_PRE: case HANDSHAKE_HOOKS_OUTBOUND_PRE: - if (eventId == TS_EVENT_SSL_SERVERNAME) { + if (eventId == TS_EVENT_SSL_CLIENT_HELLO) { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO; + } else if (eventId == TS_EVENT_SSL_SERVERNAME) { this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; } else if (eventId == TS_EVENT_SSL_VERIFY_SERVER) { this->sslHandshakeHookState = HANDSHAKE_HOOKS_VERIFY_SERVER; @@ -1636,6 +1672,16 @@ SSLNetVConnection::callHooks(TSEvent eventId) this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; } break; + case HANDSHAKE_HOOKS_CLIENT_HELLO: + if (eventId == TS_EVENT_SSL_SERVERNAME) { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; + } else if (eventId == TS_EVENT_SSL_CERT) { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; + } else if (eventId == TS_EVENT_VCONN_CLOSE) { + // Jump to the end + this->sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; + } + break; case HANDSHAKE_HOOKS_SNI: if (eventId == TS_EVENT_SSL_CERT) { this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; @@ -1650,6 +1696,19 @@ SSLNetVConnection::callHooks(TSEvent eventId) // Look for hooks associated with the event switch (this->sslHandshakeHookState) { + case HANDSHAKE_HOOKS_CLIENT_HELLO: + case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: + if (!curHook) { + curHook = ssl_hooks->get(TS_SSL_CLIENT_HELLO_INTERNAL_HOOK); + } else { + curHook = curHook->next(); + } + if (curHook == nullptr) { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; + } else { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE; + } + break; case HANDSHAKE_HOOKS_VERIFY_SERVER: // The server verify event addresses ATS to origin handshake // All the other events are for client to ATS @@ -1731,7 +1790,8 @@ SSLNetVConnection::callHooks(TSEvent eventId) if (curHook != nullptr) { curHook->invoke(eventId, this); reenabled = - (this->sslHandshakeHookState != HANDSHAKE_HOOKS_CERT_INVOKE && this->sslHandshakeHookState != HANDSHAKE_HOOKS_PRE_INVOKE); + (this->sslHandshakeHookState != HANDSHAKE_HOOKS_CERT_INVOKE && this->sslHandshakeHookState != HANDSHAKE_HOOKS_PRE_INVOKE && + this->sslHandshakeHookState != HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE); Debug("ssl", "Called hook on state=%d reenabled=%d", sslHandshakeHookState, reenabled); } diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 53255b1f002..305317af312 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -423,7 +423,8 @@ PerformAction(Continuation *cont, const char *servername) static int ssl_client_hello_callback(SSL *s, int *al, void *arg) { - const char *servername = nullptr; + SSLNetVConnection *netvc = SSLNetVCAccess(s); + const char *servername = nullptr; const unsigned char *p; size_t remaining, len; // Parse the servrer name if the get extension call succeeds and there are more than 2 bytes to parse @@ -452,9 +453,6 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) } } } - - SSLNetVConnection *netvc = SSLNetVCAccess(s); - netvc->serverName = servername ? servername : ""; int ret = PerformAction(netvc, netvc->serverName); if (ret != SSL_TLSEXT_ERR_OK) { @@ -466,6 +464,12 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) if (netvc->protocol_mask_set) { setTLSValidProtocols(s, netvc->protocol_mask, TLSValidProtocols::max_mask); } + + bool reenabled = netvc->callHooks(TS_EVENT_SSL_CLIENT_HELLO); + + if (!reenabled) { + return SSL_CLIENT_HELLO_RETRY; + } return SSL_CLIENT_HELLO_SUCCESS; } #endif diff --git a/proxy/InkAPIInternal.h b/proxy/InkAPIInternal.h index cd75d64fc47..7955c532328 100644 --- a/proxy/InkAPIInternal.h +++ b/proxy/InkAPIInternal.h @@ -269,6 +269,7 @@ typedef enum { TS_SSL_INTERNAL_FIRST_HOOK, TS_VCONN_START_INTERNAL_HOOK = TS_SSL_INTERNAL_FIRST_HOOK, TS_VCONN_CLOSE_INTERNAL_HOOK, + TS_SSL_CLIENT_HELLO_INTERNAL_HOOK, TS_SSL_CERT_INTERNAL_HOOK, TS_SSL_SERVERNAME_INTERNAL_HOOK, TS_SSL_VERIFY_SERVER_INTERNAL_HOOK, diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index c76ef0e8973..f99e5b1c135 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -365,6 +365,8 @@ HttpDebugNames::get_event_name(int event) return "TS_EVENT_INTERNAL_60201"; case TS_EVENT_INTERNAL_60202: return "TS_EVENT_INTERNAL_60202"; + case TS_EVENT_SSL_CLIENT_HELLO: + return "TS_EVENT_SSL_CLIENT_HELLO"; case TS_EVENT_SSL_CERT: return "TS_EVENT_SSL_CERT"; case TS_EVENT_SSL_SERVERNAME: @@ -607,6 +609,8 @@ HttpDebugNames::get_api_hook_name(TSHttpHookID t) return "TS_VCONN_START_HOOK"; case TS_VCONN_CLOSE_HOOK: return "TS_VCONN_CLOSE_HOOK"; + case TS_SSL_CLIENT_HELLO_HOOK: + return "TS_SSL_CLIENT_HELLO_HOOK"; case TS_SSL_CERT_HOOK: return "TS_SSL_CERT_HOOK"; case TS_SSL_SERVERNAME_HOOK: diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index dcf7a84b6b7..d0008386c78 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -6620,6 +6620,7 @@ typedef enum { ORIG_TS_SSL_FIRST_HOOK, ORIG_TS_VCONN_START_HOOK = ORIG_TS_SSL_FIRST_HOOK, ORIG_TS_VCONN_CLOSE_HOOK, + ORIG_TS_SSL_CLIENT_HELLO_HOOK, ORIG_TS_SSL_SNI_HOOK, ORIG_TS_SSL_SERVERNAME_HOOK, ORIG_TS_SSL_VERIFY_SERVER_HOOK, diff --git a/tests/gold_tests/tls_hooks/gold/client-hello-1.gold b/tests/gold_tests/tls_hooks/gold/client-hello-1.gold new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/gold_tests/tls_hooks/gold/ts-cert-1-im-2.gold b/tests/gold_tests/tls_hooks/gold/ts-cert-1-im-2.gold index a2ec9b73667..98c8ce588f6 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-cert-1-im-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-cert-1-im-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=1 cert_imm=2 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=1 cert_imm=2 pa_delay=0 `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` `` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-cert-1.gold b/tests/gold_tests/tls_hooks/gold/ts-cert-1.gold index 91f7b38c3d3..4a77e239a99 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-cert-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-cert-1.gold @@ -1,3 +1,3 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=1 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=1 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` `` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-cert-2.gold b/tests/gold_tests/tls_hooks/gold/ts-cert-2.gold index 355c5aefbfa..b56bd6d581b 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-cert-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-cert-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=2 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=2 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` `` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` `` DIAG: (ssl_hook_test) Cert callback 1 ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-cert-im-1.gold b/tests/gold_tests/tls_hooks/gold/ts-cert-im-1.gold index c571bae3b87..11bb4e7fcd1 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-cert-im-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-cert-im-1.gold @@ -1,2 +1,2 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=0 cert_imm=1 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=1 pa_delay=0 `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-client-hello-1.gold b/tests/gold_tests/tls_hooks/gold/ts-client-hello-1.gold new file mode 100644 index 00000000000..8f444f3c95b --- /dev/null +++ b/tests/gold_tests/tls_hooks/gold/ts-client-hello-1.gold @@ -0,0 +1,3 @@ +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=1 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Client Hello callback 0 `` +`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-client-hello-2.gold b/tests/gold_tests/tls_hooks/gold/ts-client-hello-2.gold new file mode 100644 index 00000000000..4489a55652f --- /dev/null +++ b/tests/gold_tests/tls_hooks/gold/ts-client-hello-2.gold @@ -0,0 +1,5 @@ +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=2 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Client Hello callback 0 ssl_vc=`` +`` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` +`` DIAG: (ssl_hook_test) Client Hello callback 1 ssl_vc=`` +`` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-client-hello-delayed-1.gold b/tests/gold_tests/tls_hooks/gold/ts-client-hello-delayed-1.gold new file mode 100644 index 00000000000..4cea3ccd6f5 --- /dev/null +++ b/tests/gold_tests/tls_hooks/gold/ts-client-hello-delayed-1.gold @@ -0,0 +1,4 @@ +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=1 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Client Hello callback 0 `` +`` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` +`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-out-delay-start-2.gold b/tests/gold_tests/tls_hooks/gold/ts-out-delay-start-2.gold index 21c5d61073c..89ef1cc4573 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-out-delay-start-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-out-delay-start-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Outbound delay start callback 0 `` `` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` `` DIAG: (ssl_hook_test) Outbound delay start callback 1 `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept-1.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept-1.gold index c8278ea9866..a2abb9d50ca 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept-1.gold @@ -1,3 +1,3 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=1 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=1 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Pre accept callback 0 `` - event is good `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept-2.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept-2.gold index cfac682852b..f70e8724e8b 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=2 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=2 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Pre accept callback 0 `` - event is good `` DIAG: (ssl_hook_test) Pre accept callback 1 `` - event is good `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1-immdate-2.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1-immdate-2.gold index 16427c9b444..685ffb492cf 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1-immdate-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1-immdate-2.gold @@ -1,4 +1,4 @@ -``DIAG: (ssl_hook_test) Setup callbacks pa=2 sni=0 cert=0 cert_imm=0 pa_delay=1 +``DIAG: (ssl_hook_test) Setup callbacks pa=2 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=1 ``DIAG: (ssl_hook_test) Pre accept callback 0 `` - event is good ``DIAG: (ssl_hook_test) Pre accept callback 1 `` - event is good ``DIAG: (ssl_hook_test) Pre accept delay callback 0 `` - event is good diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1.gold index 0b85e1786c6..3ebced31bcb 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1.gold @@ -1,3 +1,3 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=0 cert_imm=0 pa_delay=1 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=1 `` DIAG: (ssl_hook_test) Pre accept delay callback 0 `` - event is good `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept1-sni1-cert1.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept1-sni1-cert1.gold index 07ee131bc85..b028f5eea93 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept1-sni1-cert1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept1-sni1-cert1.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=1 sni=1 cert=1 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=1 client_hello=0 client_hello_imm=0 sni=1 cert=1 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Pre accept callback 0 `` - event is good `` DIAG: (ssl_hook_test) SNI callback 0 `` `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-sni-1.gold b/tests/gold_tests/tls_hooks/gold/ts-sni-1.gold index 4b7d335f90c..bda783ae230 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-sni-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-sni-1.gold @@ -1,3 +1,3 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=1 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=1 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) SNI callback 0 `` `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-sni-2.gold b/tests/gold_tests/tls_hooks/gold/ts-sni-2.gold index ecb2cb6cbef..7ca56f72920 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-sni-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-sni-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=2 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=2 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) SNI callback 0 `` `` DIAG: (ssl_hook_test) SNI callback 1 `` `` diff --git a/tests/gold_tests/tls_hooks/tls_hooks16.test.py b/tests/gold_tests/tls_hooks/tls_hooks16.test.py new file mode 100644 index 00000000000..877c32018d5 --- /dev/null +++ b/tests/gold_tests/tls_hooks/tls_hooks16.test.py @@ -0,0 +1,79 @@ +''' +Test single immediate client hello hook +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test different combinations of TLS handshake hooks to ensure they are applied consistently. +''' + +Test.SkipUnless( + Condition.HasProgram("grep", "grep needs to be installed on system for this test to work"), + Condition.HasOpenSSLVersion("1.1.1")) + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_hook_test', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello_imm=1') + +tr = Test.AddTestRun("Test one immediate client hello hook") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/client-hello-1.gold" + +ts.Streams.stderr = "gold/ts-client-hello-1.gold" + +snistring = "Client Hello callback 0" +ts.Streams.All = Testers.ContainsExpression( + "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(snistring), "Client Hello message appears only once", reflags=re.S | re.M) + +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 diff --git a/tests/gold_tests/tls_hooks/tls_hooks17.test.py b/tests/gold_tests/tls_hooks/tls_hooks17.test.py new file mode 100644 index 00000000000..327c36d7204 --- /dev/null +++ b/tests/gold_tests/tls_hooks/tls_hooks17.test.py @@ -0,0 +1,79 @@ +''' +Test single delayed client hello hook +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test different combinations of TLS handshake hooks to ensure they are applied consistently. +''' + +Test.SkipUnless( + Condition.HasProgram("grep", "grep needs to be installed on system for this test to work"), + Condition.HasOpenSSLVersion("1.1.1")) + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_hook_test', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello=1') + +tr = Test.AddTestRun("Test one delayed client hello hook") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/client-hello-1.gold" + +ts.Streams.stderr = "gold/ts-client-hello-delayed-1.gold" + +snistring = "Client Hello callback 0" +ts.Streams.All = Testers.ContainsExpression( + "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(snistring), "Client Hello message appears only once", reflags=re.S | re.M) + +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 diff --git a/tests/gold_tests/tls_hooks/tls_hooks18.test.py b/tests/gold_tests/tls_hooks/tls_hooks18.test.py new file mode 100644 index 00000000000..4ce45d12dba --- /dev/null +++ b/tests/gold_tests/tls_hooks/tls_hooks18.test.py @@ -0,0 +1,83 @@ +''' +Test two delayed client hello callbacks +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test different combinations of TLS handshake hooks to ensure they are applied consistently. +''' + +Test.SkipUnless( + Condition.HasProgram("grep", "grep needs to be installed on system for this test to work"), + Condition.HasOpenSSLVersion("1.1.1") + ) + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_hook_test', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello=2') + +tr = Test.AddTestRun("Test two client hello hooks") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/preaccept-1.gold" + +ts.Streams.stderr = "gold/ts-client-hello-2.gold" + +certstring0 = "Client Hello callback 0" +certstring1 = "Client Hello callback 1" +ts.Streams.All = Testers.ContainsExpression( + "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring0), "Cert message appears only once", reflags=re.S | re.M) +ts.Streams.All = Testers.ContainsExpression( + "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring1), "Cert message appears only once", reflags=re.S | re.M) + +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 diff --git a/tests/tools/plugins/ssl_hook_test.cc b/tests/tools/plugins/ssl_hook_test.cc index a863c8ce0d9..2c9f02f57ea 100644 --- a/tests/tools/plugins/ssl_hook_test.cc +++ b/tests/tools/plugins/ssl_hook_test.cc @@ -141,6 +141,38 @@ CB_out_close(TSCont cont, TSEvent event, void *edata) TSVConnReenable(ssl_vc); return TS_SUCCESS; } +int +CB_Client_Hello_Immediate(TSCont cont, TSEvent event, void *edata) +{ + TSVConn ssl_vc = reinterpret_cast(edata); + + int count = reinterpret_cast(TSContDataGet(cont)); + + TSDebug(PN, "Client Hello callback %d ssl_vc=%p", count, ssl_vc); + + // All done, reactivate things + TSVConnReenable(ssl_vc); + return TS_SUCCESS; +} + +int +CB_Client_Hello(TSCont cont, TSEvent event, void *edata) +{ + TSVConn ssl_vc = reinterpret_cast(edata); + + int count = reinterpret_cast(TSContDataGet(cont)); + + TSDebug(PN, "Client Hello callback %d ssl_vc=%p", count, ssl_vc); + + TSCont cb = TSContCreate(&ReenableSSL, TSMutexCreate()); + + TSContDataSet(cb, ssl_vc); + + // Schedule to reenable in a bit + TSContScheduleOnPool(cb, 2000, TS_THREAD_POOL_NET); + + return TS_SUCCESS; +} int CB_SNI(TSCont cont, TSEvent event, void *edata) @@ -189,9 +221,9 @@ CB_Cert(TSCont cont, TSEvent event, void *edata) } void -parse_callbacks(int argc, const char *argv[], int &preaccept_count, int &sni_count, int &cert_count, int &cert_count_immediate, - int &preaccept_count_delay, int &close_count, int &out_start_count, int &out_start_delay_count, - int &out_close_count) +parse_callbacks(int argc, const char *argv[], int &preaccept_count, int &client_hello_count, int &client_hello_count_immediate, + int &sni_count, int &cert_count, int &cert_count_immediate, int &preaccept_count_delay, int &close_count, + int &out_start_count, int &out_start_delay_count, int &out_close_count) { int i = 0; const char *ptr; @@ -215,6 +247,10 @@ parse_callbacks(int argc, const char *argv[], int &preaccept_count, int &sni_cou if (ptr) { if (strncmp(argv[i] + 1, "close", strlen("close")) == 0) { close_count = atoi(ptr + i); + } else if (strncmp(argv[i] + 1, "client_hello_imm", strlen("client_hello_imm")) == 0) { + client_hello_count_immediate = atoi(ptr + i); + } else if (strncmp(argv[i] + 1, "client_hello", strlen("client_hello")) == 0) { + client_hello_count = atoi(ptr + i); } else { cert_count = atoi(ptr + 1); } @@ -249,14 +285,15 @@ parse_callbacks(int argc, const char *argv[], int &preaccept_count, int &sni_cou } void -setup_callbacks(TSHttpTxn txn, int preaccept_count, int sni_count, int cert_count, int cert_count_immediate, - int preaccept_count_delay, int close_count, int out_start_count, int out_start_delay_count, int out_close_count) +setup_callbacks(TSHttpTxn txn, int preaccept_count, int client_hello_count, int client_hello_count_immediate, int sni_count, + int cert_count, int cert_count_immediate, int preaccept_count_delay, int close_count, int out_start_count, + int out_start_delay_count, int out_close_count) { TSCont cb = nullptr; // pre-accept callback continuation int i; - TSDebug(PN, "Setup callbacks pa=%d sni=%d cert=%d cert_imm=%d pa_delay=%d", preaccept_count, sni_count, cert_count, - cert_count_immediate, preaccept_count_delay); + TSDebug(PN, "Setup callbacks pa=%d client_hello=%d client_hello_imm=%d sni=%d cert=%d cert_imm=%d pa_delay=%d", preaccept_count, + client_hello_count, client_hello_count_immediate, sni_count, cert_count, cert_count_immediate, preaccept_count_delay); for (i = 0; i < preaccept_count; i++) { cb = TSContCreate(&CB_Pre_Accept, TSMutexCreate()); TSContDataSet(cb, (void *)(intptr_t)i); @@ -275,6 +312,24 @@ setup_callbacks(TSHttpTxn txn, int preaccept_count, int sni_count, int cert_coun TSHttpHookAdd(TS_VCONN_START_HOOK, cb); } } + for (i = 0; i < client_hello_count; i++) { + cb = TSContCreate(&CB_Client_Hello, TSMutexCreate()); + TSContDataSet(cb, (void *)(intptr_t)i); + if (txn) { + TSHttpTxnHookAdd(txn, TS_SSL_CLIENT_HELLO_HOOK, cb); + } else { + TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, cb); + } + } + for (i = 0; i < client_hello_count_immediate; i++) { + cb = TSContCreate(&CB_Client_Hello_Immediate, TSMutexCreate()); + TSContDataSet(cb, (void *)(intptr_t)i); + if (txn) { + TSHttpTxnHookAdd(txn, TS_SSL_CLIENT_HELLO_HOOK, cb); + } else { + TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, cb); + } + } for (i = 0; i < sni_count; i++) { cb = TSContCreate(&CB_SNI, TSMutexCreate()); TSContDataSet(cb, (void *)(intptr_t)i); @@ -355,18 +410,22 @@ TSPluginInit(int argc, const char *argv[]) TSError("[%s] Plugin registration failed", PN); } - int preaccept_count = 0; - int sni_count = 0; - int cert_count = 0; - int cert_count_immediate = 0; - int preaccept_count_delay = 0; - int close_count = 0; - int out_start_count = 0; - int out_start_delay_count = 0; - int out_close_count = 0; - parse_callbacks(argc, argv, preaccept_count, sni_count, cert_count, cert_count_immediate, preaccept_count_delay, close_count, - out_start_count, out_start_delay_count, out_close_count); - setup_callbacks(nullptr, preaccept_count, sni_count, cert_count, cert_count_immediate, preaccept_count_delay, close_count, - out_start_count, out_start_delay_count, out_close_count); + int preaccept_count = 0; + int client_hello_count = 0; + int client_hello_count_immediate = 0; + int sni_count = 0; + int cert_count = 0; + int cert_count_immediate = 0; + int preaccept_count_delay = 0; + int close_count = 0; + int out_start_count = 0; + int out_start_delay_count = 0; + int out_close_count = 0; + parse_callbacks(argc, argv, preaccept_count, client_hello_count, client_hello_count_immediate, sni_count, cert_count, + cert_count_immediate, preaccept_count_delay, close_count, out_start_count, out_start_delay_count, + out_close_count); + setup_callbacks(nullptr, preaccept_count, client_hello_count, client_hello_count_immediate, sni_count, cert_count, + cert_count_immediate, preaccept_count_delay, close_count, out_start_count, out_start_delay_count, + out_close_count); return; } diff --git a/tests/tools/plugins/test_hooks.cc b/tests/tools/plugins/test_hooks.cc index f5c3897bb20..67ff9bbda4e 100644 --- a/tests/tools/plugins/test_hooks.cc +++ b/tests/tools/plugins/test_hooks.cc @@ -180,7 +180,6 @@ globalContFunc(TSCont, TSEvent event, void *eventData) TSVConnReenable(vConn); } break; - case TS_EVENT_SSL_CERT: case TS_EVENT_SSL_SERVERNAME: { auto vConn = static_cast(eventData); @@ -327,7 +326,6 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, gCont); TSHttpHookAdd(TS_SSL_CERT_HOOK, gCont); TSHttpHookAdd(TS_SSL_SERVERNAME_HOOK, gCont); - // NOTE: as of January 2019 these two hooks are only triggered for TLS connections. It seems that, at trafficserver // startup, spurious data on the TLS TCP port may cause trafficserver to attempt (and fail) to create a TLS // connection. If this happens, it will result in TS_VCONN_START_HOOK being triggered, and then TS_VCONN_CLOSE_HOOK From d57057d3590d73b58bc2b643501f1a584b94f54e Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 25 Oct 2018 17:02:25 -0500 Subject: [PATCH 287/526] Refresh mgmt signal logic to remove possible memory issues. --- CMakeLists.txt | 1 + iocore/net/test_I_UDPNet.cc | 2 +- lib/records/I_RecCore.h | 8 +- lib/records/P_RecMessage.h | 3 +- lib/records/RecLocal.cc | 6 +- lib/records/RecMessage.cc | 8 +- lib/records/RecProcess.cc | 6 +- mgmt/BaseManager.cc | 119 +++++++++++---------------- mgmt/BaseManager.h | 72 +++++++++++----- mgmt/LocalManager.cc | 25 +++--- mgmt/MgmtDefs.h | 16 ++-- mgmt/ProcessManager.cc | 32 +++---- proxy/logging/LogConfig.cc | 7 +- proxy/logging/LogConfig.h | 3 +- src/traffic_server/HostStatus.cc | 22 ++--- src/traffic_server/traffic_server.cc | 51 ++++++------ 16 files changed, 197 insertions(+), 184 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03d296b9082..bfc012dc6b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,7 @@ CPP_ADD_SOURCES(mgmt mgmt/api) CPP_ADD_SOURCES(mgmt mgmt/utils) CPP_LIB(records lib/records lib/records) +CPP_LIB(logging proxy/logging proxy/logging) CPP_LIB(tsconfig lib/tsconfig lib/tsconfig) CPP_LIB(wccp src/wccp include/wccp) diff --git a/iocore/net/test_I_UDPNet.cc b/iocore/net/test_I_UDPNet.cc index 5ee79f0c4cc..16810cbd22e 100644 --- a/iocore/net/test_I_UDPNet.cc +++ b/iocore/net/test_I_UDPNet.cc @@ -362,7 +362,7 @@ StatPagesManager statPagesManager; inkcoreapi ProcessManager *pmgmt = nullptr; int -BaseManager::registerMgmtCallback(int, MgmtCallback, void *) +BaseManager::registerMgmtCallback(int, MgmtCallback const &) { ink_assert(false); return 0; diff --git a/lib/records/I_RecCore.h b/lib/records/I_RecCore.h index b9157679367..3bea5c5891e 100644 --- a/lib/records/I_RecCore.h +++ b/lib/records/I_RecCore.h @@ -23,13 +23,15 @@ #pragma once +#include + #include "tscore/Diags.h" #include "I_RecDefs.h" #include "I_RecAlarms.h" #include "I_RecSignals.h" #include "I_RecEvents.h" -#include +#include "tscpp/util/MemSpan.h" struct RecRecord; @@ -305,5 +307,5 @@ RecErrT RecSetSyncRequired(char *name, bool lock = true); //------------------------------------------------------------------------ // Manager Callback //------------------------------------------------------------------------ -typedef void *(*RecManagerCb)(void *opaque_cb_data, char *data_raw, int data_len); -int RecRegisterManagerCb(int _signal, RecManagerCb _fn, void *_data = nullptr); +using RecManagerCb = std::function; +int RecRegisterManagerCb(int _signal, RecManagerCb const &_fn); diff --git a/lib/records/P_RecMessage.h b/lib/records/P_RecMessage.h index 24db24cb4c8..d41f74c5ec4 100644 --- a/lib/records/P_RecMessage.h +++ b/lib/records/P_RecMessage.h @@ -24,6 +24,7 @@ #pragma once #include "P_RecDefs.h" +#include "tscpp/util/MemSpan.h" //------------------------------------------------------------------------- // Initialization @@ -44,7 +45,7 @@ int RecMessageUnmarshalNext(RecMessage *msg, RecMessageItr *itr, RecRecord **rec int RecMessageSend(RecMessage *msg); int RecMessageRegisterRecvCb(RecMessageRecvCb recv_cb, void *cookie); -void *RecMessageRecvThis(void *cookie, char *data_raw, int data_len); +void RecMessageRecvThis(ts::MemSpan); RecMessage *RecMessageReadFromDisk(const char *fpath); int RecMessageWriteToDisk(RecMessage *msg, const char *fpath); diff --git a/lib/records/RecLocal.cc b/lib/records/RecLocal.cc index ad927e526fb..34c721512d4 100644 --- a/lib/records/RecLocal.cc +++ b/lib/records/RecLocal.cc @@ -145,7 +145,7 @@ void RecMessageInit() { ink_assert(g_mode_type != RECM_NULL); - lmgmt->registerMgmtCallback(MGMT_SIGNAL_LIBRECORDS, RecMessageRecvThis, nullptr); + lmgmt->registerMgmtCallback(MGMT_SIGNAL_LIBRECORDS, &RecMessageRecvThis); message_initialized_p = true; } @@ -207,9 +207,9 @@ RecLocalStart(FileManager *configFiles) } int -RecRegisterManagerCb(int id, RecManagerCb _fn, void *_data) +RecRegisterManagerCb(int id, RecManagerCb const &_fn) { - return lmgmt->registerMgmtCallback(id, _fn, _data); + return lmgmt->registerMgmtCallback(id, _fn); } void diff --git a/lib/records/RecMessage.cc b/lib/records/RecMessage.cc index 743649066c4..c18d50253dc 100644 --- a/lib/records/RecMessage.cc +++ b/lib/records/RecMessage.cc @@ -31,6 +31,7 @@ #include "P_RecUtils.h" #include "P_RecCore.h" #include "tscore/I_Layout.h" +#include "tscpp/util/MemSpan.h" static RecMessageRecvCb g_recv_cb = nullptr; static void *g_recv_cookie = nullptr; @@ -243,12 +244,11 @@ RecMessageRegisterRecvCb(RecMessageRecvCb recv_cb, void *cookie) // RecMessageRecvThis //------------------------------------------------------------------------- -void * -RecMessageRecvThis(void * /* cookie */, char *data_raw, int /* data_len */) +void +RecMessageRecvThis(ts::MemSpan span) { - RecMessage *msg = (RecMessage *)data_raw; + RecMessage *msg = static_cast(span.data()); g_recv_cb(msg, msg->msg_type, g_recv_cookie); - return nullptr; } //------------------------------------------------------------------------- diff --git a/lib/records/RecProcess.cc b/lib/records/RecProcess.cc index 08848f46cc3..29fcb07db6c 100644 --- a/lib/records/RecProcess.cc +++ b/lib/records/RecProcess.cc @@ -231,7 +231,7 @@ void RecMessageInit() { ink_assert(g_mode_type != RECM_NULL); - pmgmt->registerMgmtCallback(MGMT_EVENT_LIBRECORDS, RecMessageRecvThis, nullptr); + pmgmt->registerMgmtCallback(MGMT_EVENT_LIBRECORDS, &RecMessageRecvThis); message_initialized_p = true; } @@ -300,9 +300,9 @@ RecSignalManager(int id, const char *msg, size_t msgsize) } int -RecRegisterManagerCb(int _signal, RecManagerCb _fn, void *_data) +RecRegisterManagerCb(int _signal, RecManagerCb const &_fn) { - return pmgmt->registerMgmtCallback(_signal, _fn, _data); + return pmgmt->registerMgmtCallback(_signal, _fn); } //------------------------------------------------------------------------- diff --git a/mgmt/BaseManager.cc b/mgmt/BaseManager.cc index abd99ce2ede..86ef10034a1 100644 --- a/mgmt/BaseManager.cc +++ b/mgmt/BaseManager.cc @@ -4,103 +4,80 @@ @section license License - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. */ #include "tscore/ink_memory.h" +#include "tscore/ink_mutex.h" #include "BaseManager.h" BaseManager::BaseManager() { - /* Setup the event queue and callback tables */ - mgmt_event_queue = create_queue(); - -} /* End BaseManager::BaseManager */ + ink_sem_init(&q_sem, 0); +} BaseManager::~BaseManager() { - while (!queue_is_empty(mgmt_event_queue)) { - MgmtMessageHdr *mh = (MgmtMessageHdr *)dequeue(mgmt_event_queue); - ats_free(mh); + while (!queue.empty()) { + ats_free(queue.front()); + queue.pop(); } - ats_free(mgmt_event_queue); - - for (auto &&it : mgmt_callback_table) { - MgmtCallbackList *tmp, *cb_list = it.second; +} - for (tmp = cb_list->next; tmp; tmp = cb_list->next) { - ats_free(cb_list); - cb_list = tmp; - } - ats_free(cb_list); - } +void +BaseManager::enqueue(MgmtMessageHdr *mh) +{ + std::lock_guard lock(q_mutex); + queue.emplace(mh); + ink_sem_post(&q_sem); +} - return; -} /* End BaseManager::~BaseManager */ +bool +BaseManager::queue_empty() +{ + std::lock_guard lock(q_mutex); + return queue.empty(); +} -/* - * registerMgmtCallback(...) - * Function to register callback's for various management events, such - * as shutdown, re-init, etc. The following callbacks should be - * registered: - * MGMT_EVENT_SHUTDOWN (graceful shutdown) - * MGMT_EVENT_RESTART (graceful reboot) - * ... - * - * Returns: -1 on error(invalid event id passed in) - * or value - */ -int -BaseManager::registerMgmtCallback(int msg_id, MgmtCallback cb, void *opaque_cb_data) +MgmtMessageHdr * +BaseManager::dequeue() { - MgmtCallbackList *cb_list; + MgmtMessageHdr *msg{nullptr}; - if (auto it = mgmt_callback_table.find(msg_id); it != mgmt_callback_table.end()) { - cb_list = it->second; - } else { - cb_list = nullptr; + ink_sem_wait(&q_sem); + { + std::lock_guard lock(q_mutex); + msg = queue.front(); + queue.pop(); } + return msg; +} - if (cb_list) { - MgmtCallbackList *tmp; - - for (tmp = cb_list; tmp->next; tmp = tmp->next) { - ; - } - tmp->next = (MgmtCallbackList *)ats_malloc(sizeof(MgmtCallbackList)); - tmp->next->func = cb; - tmp->next->opaque_data = opaque_cb_data; - tmp->next->next = nullptr; - } else { - cb_list = (MgmtCallbackList *)ats_malloc(sizeof(MgmtCallbackList)); - cb_list->func = cb; - cb_list->opaque_data = opaque_cb_data; - cb_list->next = nullptr; - mgmt_callback_table.emplace(msg_id, cb_list); - } +int +BaseManager::registerMgmtCallback(int msg_id, MgmtCallback const &cb) +{ + auto &cb_list{mgmt_callback_table[msg_id]}; + cb_list.emplace_back(cb); return msg_id; -} /* End BaseManager::registerMgmtCallback */ +} void -BaseManager::executeMgmtCallback(int msg_id, char *data_raw, int data_len) +BaseManager::executeMgmtCallback(int msg_id, ts::MemSpan span) { if (auto it = mgmt_callback_table.find(msg_id); it != mgmt_callback_table.end()) { - for (MgmtCallbackList *cb_list = it->second; cb_list; cb_list = cb_list->next) { - (*((MgmtCallback)(cb_list->func)))(cb_list->opaque_data, data_raw, data_len); + for (auto &&cb : it->second) { + cb(span); } } } diff --git a/mgmt/BaseManager.h b/mgmt/BaseManager.h index a254a722f6c..9f72212f833 100644 --- a/mgmt/BaseManager.h +++ b/mgmt/BaseManager.h @@ -23,15 +23,18 @@ #pragma once +#include +#include +#include +#include + #include "tscore/ink_thread.h" #include "tscore/ink_mutex.h" -#include "tscore/ink_llqueue.h" +#include "tscpp/util/MemSpan.h" #include "MgmtDefs.h" #include "MgmtMarshall.h" -#include - /* * MgmtEvent defines. */ @@ -88,30 +91,61 @@ #define MGMT_SIGNAL_SAC_SERVER_DOWN 400 -typedef struct _mgmt_message_hdr_type { +struct MgmtMessageHdr { int msg_id; int data_len; -} MgmtMessageHdr; - -typedef struct _mgmt_event_callback_list { - MgmtCallback func; - void *opaque_data; - struct _mgmt_event_callback_list *next; -} MgmtCallbackList; + ts::MemSpan + payload() + { + return {reinterpret_cast(this) + sizeof(*this), data_len}; + } +}; class BaseManager { + using MgmtCallbackList = std::list; + public: BaseManager(); - ~BaseManager(); - int registerMgmtCallback(int msg_id, MgmtCallback func, void *opaque_callback_data = nullptr); + ~BaseManager(); - LLQ *mgmt_event_queue; - std::unordered_map mgmt_callback_table; + /** Associate a callback function @a func with message identifier @a msg_id. + * + * @param msg_id Message identifier for the callback. + * @param func The callback function. + * @return @a msg_id on success, -1 on failure. + * + * @a msg_id should be one of the @c MGMT_EVENT_... values. + * + * If a management message with @a msg is received, the callbacks for that message id + * are invoked and passed the message payload (not including the header). + */ + int registerMgmtCallback(int msg_id, MgmtCallback const &func); + + /// Add a @a msg to the queue. + /// This must be the entire message as read off the wire including the header. + void enqueue(MgmtMessageHdr *msg); + + /// Current size of the queue. + /// @note This does not block on the semaphore. + bool queue_empty(); + + /// Dequeue a msg. + /// This waits on the semaphore for a message to arrive. + MgmtMessageHdr *dequeue(); protected: - void executeMgmtCallback(int msg_id, char *data_raw, int data_len); - -private: -}; /* End class BaseManager */ + void executeMgmtCallback(int msg_id, ts::MemSpan span); + + /// The mapping from an event type to a list of callbacks to invoke. + std::unordered_map mgmt_callback_table; + + /// Message queue. + // These holds the entire message object, including the header. + std::queue queue; + /// Locked access to the queue. + std::mutex q_mutex; + /// Semaphore to signal queue state. + ink_semaphore q_sem; +}; diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index 0d355242fc2..a9205bed65d 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -579,9 +579,9 @@ LocalManager::handleMgmtMsgFromProcesses(MgmtMessageHdr *mh) } case MGMT_SIGNAL_LIBRECORDS: if (mh->data_len > 0) { - executeMgmtCallback(MGMT_SIGNAL_LIBRECORDS, data_raw, mh->data_len); + executeMgmtCallback(MGMT_SIGNAL_LIBRECORDS, {data_raw, mh->data_len}); } else { - executeMgmtCallback(MGMT_SIGNAL_LIBRECORDS, nullptr, 0); + executeMgmtCallback(MGMT_SIGNAL_LIBRECORDS, {}); } break; case MGMT_SIGNAL_CONFIG_FILE_CHILD: { @@ -755,12 +755,15 @@ void LocalManager::signalEvent(int msg_id, const char *data_raw, int data_len) { MgmtMessageHdr *mh; + size_t n = sizeof(MgmtMessageHdr) + data_len; - mh = (MgmtMessageHdr *)ats_malloc(sizeof(MgmtMessageHdr) + data_len); + mh = static_cast(ats_malloc(n)); mh->msg_id = msg_id; mh->data_len = data_len; - memcpy((char *)mh + sizeof(MgmtMessageHdr), data_raw, data_len); - ink_assert(enqueue(mgmt_event_queue, mh)); + auto payload = mh->payload(); + memcpy(payload.data(), data_raw, data_len); + this->enqueue(mh); + // ink_assert(enqueue(mgmt_event_queue, mh)); #if HAVE_EVENTFD // we don't care about the actual value of wakeup_fd, so just keep adding 1. just need to @@ -788,16 +791,16 @@ LocalManager::processEventQueue() { bool handled_by_mgmt; - while (!queue_is_empty(mgmt_event_queue)) { + while (!this->queue_empty()) { handled_by_mgmt = false; - MgmtMessageHdr *mh = (MgmtMessageHdr *)dequeue(mgmt_event_queue); - char *data_raw = (char *)mh + sizeof(MgmtMessageHdr); + MgmtMessageHdr *mh = this->dequeue(); + auto payload = mh->payload(); // check if we have a local file update if (mh->msg_id == MGMT_EVENT_CONFIG_FILE_UPDATE || mh->msg_id == MGMT_EVENT_CONFIG_FILE_UPDATE_NO_INC_VERSION) { // records.config - if (!(strcmp(data_raw, REC_CONFIG_FILE))) { + if (!(strcmp(payload.begin(), REC_CONFIG_FILE))) { bool incVersion = mh->msg_id == MGMT_EVENT_CONFIG_FILE_UPDATE; if (RecReadConfigFile(incVersion) != REC_ERR_OKAY) { mgmt_elog(errno, "[fileUpdated] Config update failed for records.config\n"); @@ -813,10 +816,10 @@ LocalManager::processEventQueue() // Fix INKqa04984 // If traffic server hasn't completely come up yet, // we will hold off until next round. - ink_assert(enqueue(mgmt_event_queue, mh)); + this->enqueue(mh); return; } - Debug("lm", "[TrafficManager] ==> Sending signal event '%d' %s payload=%d", mh->msg_id, data_raw, mh->data_len); + Debug("lm", "[TrafficManager] ==> Sending signal event '%d' %s payload=%d", mh->msg_id, payload.begin(), int(payload.size())); lmgmt->sendMgmtMsgToProcesses(mh); } ats_free(mh); diff --git a/mgmt/MgmtDefs.h b/mgmt/MgmtDefs.h index ed8f681ccc7..69328682fe0 100644 --- a/mgmt/MgmtDefs.h +++ b/mgmt/MgmtDefs.h @@ -30,6 +30,7 @@ #include #include "tscore/ink_defs.h" +#include "tscpp/util/MemSpan.h" typedef int64_t MgmtIntCounter; typedef int64_t MgmtInt; @@ -37,20 +38,19 @@ typedef int8_t MgmtByte; typedef float MgmtFloat; typedef char *MgmtString; -typedef enum { +enum MgmtType { MGMT_INVALID = -1, MGMT_INT = 0, MGMT_FLOAT = 1, MGMT_STRING = 2, MGMT_COUNTER = 3, MGMT_TYPE_MAX = 4, -} MgmtType; +}; -/* - * MgmtCallback - * Management Callback functions. - */ -using MgmtCallback = void *(*)(void *opaque_cb_data, char *data_raw, int data_len); +/// Management callback signature. +/// The memory span is the message payload for the callback. +/// This can be a lambda, which should be used if additional context information is needed. +using MgmtCallback = std::function; //------------------------------------------------------------------------- // API conversion functions. @@ -142,4 +142,4 @@ inline MgmtConverter::MgmtConverter(MgmtInt (*_load_int)(void *), void (*_store_ { } -#define LM_CONNECTION_SERVER "processerver.sock" +constexpr std::string_view LM_CONNECTION_SERVER{"processerver.sock"}; diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc index 4f52c9e46a5..ad0c3f7dee0 100644 --- a/mgmt/ProcessManager.cc +++ b/mgmt/ProcessManager.cc @@ -122,7 +122,7 @@ ProcessManager::stop() poll_thread = ink_thread_null(); while (!queue_is_empty(mgmt_signal_queue)) { - char *sig = (char *)dequeue(mgmt_signal_queue); + char *sig = (char *)::dequeue(mgmt_signal_queue); ats_free(sig); } @@ -244,7 +244,7 @@ ProcessManager::signalManager(int msg_id, const char *data_raw, int data_len) mh->data_len = data_len; memcpy((char *)mh + sizeof(MgmtMessageHdr), data_raw, data_len); - ink_release_assert(enqueue(mgmt_signal_queue, mh)); + ink_release_assert(::enqueue(mgmt_signal_queue, mh)); #if HAVE_EVENTFD // we don't care about the actual value of wakeup_fd, so just keep adding 1. just need to @@ -265,7 +265,7 @@ int ProcessManager::processSignalQueue() { while (!queue_is_empty(mgmt_signal_queue)) { - MgmtMessageHdr *mh = (MgmtMessageHdr *)dequeue(mgmt_signal_queue); + MgmtMessageHdr *mh = (MgmtMessageHdr *)::dequeue(mgmt_signal_queue); Debug("pmgmt", "signaling local manager with message ID %d", mh->msg_id); @@ -414,35 +414,35 @@ ProcessManager::handleMgmtMsgFromLM(MgmtMessageHdr *mh) { ink_assert(mh != nullptr); - char *data_raw = (char *)mh + sizeof(MgmtMessageHdr); + auto payload = mh->payload(); Debug("pmgmt", "processing event id '%d' payload=%d", mh->msg_id, mh->data_len); switch (mh->msg_id) { case MGMT_EVENT_SHUTDOWN: - executeMgmtCallback(MGMT_EVENT_SHUTDOWN, nullptr, 0); + executeMgmtCallback(MGMT_EVENT_SHUTDOWN, {}); Alert("exiting on shutdown message"); break; case MGMT_EVENT_RESTART: - executeMgmtCallback(MGMT_EVENT_RESTART, nullptr, 0); + executeMgmtCallback(MGMT_EVENT_RESTART, {}); break; case MGMT_EVENT_DRAIN: - executeMgmtCallback(MGMT_EVENT_DRAIN, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_DRAIN, payload); break; case MGMT_EVENT_CLEAR_STATS: - executeMgmtCallback(MGMT_EVENT_CLEAR_STATS, nullptr, 0); + executeMgmtCallback(MGMT_EVENT_CLEAR_STATS, {}); break; case MGMT_EVENT_HOST_STATUS_UP: - executeMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, payload); break; case MGMT_EVENT_HOST_STATUS_DOWN: - executeMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, payload); break; case MGMT_EVENT_ROLL_LOG_FILES: - executeMgmtCallback(MGMT_EVENT_ROLL_LOG_FILES, nullptr, 0); + executeMgmtCallback(MGMT_EVENT_ROLL_LOG_FILES, {}); break; case MGMT_EVENT_PLUGIN_CONFIG_UPDATE: - if (data_raw != nullptr && data_raw[0] != '\0' && this->cbtable) { - this->cbtable->invoke(data_raw); + if (!payload.empty() && payload.at(0) != '\0' && this->cbtable) { + this->cbtable->invoke(static_cast(payload.data())); } break; case MGMT_EVENT_CONFIG_FILE_UPDATE: @@ -463,13 +463,13 @@ ProcessManager::handleMgmtMsgFromLM(MgmtMessageHdr *mh) */ break; case MGMT_EVENT_LIBRECORDS: - executeMgmtCallback(MGMT_EVENT_LIBRECORDS, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_LIBRECORDS, payload); break; case MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE: - executeMgmtCallback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, payload); break; case MGMT_EVENT_LIFECYCLE_MESSAGE: - executeMgmtCallback(MGMT_EVENT_LIFECYCLE_MESSAGE, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_LIFECYCLE_MESSAGE, payload); break; default: Warning("received unknown message ID %d\n", mh->msg_id); diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc index 4b09754f612..4437266f120 100644 --- a/proxy/logging/LogConfig.cc +++ b/proxy/logging/LogConfig.cc @@ -109,13 +109,10 @@ LogConfig::setup_default_values() max_line_size = 9216; // size of pipe buffer for SunOS 5.6 } -void * -LogConfig::reconfigure_mgmt_variables(void * /* token ATS_UNUSED */, char * /* data_raw ATS_UNUSED */, - int /* data_len ATS_UNUSED */) +void LogConfig::reconfigure_mgmt_variables(ts::MemSpan) { Note("received log reconfiguration event, rolling now"); Log::config->roll_log_files_now = true; - return nullptr; } void @@ -668,7 +665,7 @@ LogConfig::register_stat_callbacks() void LogConfig::register_mgmt_callbacks() { - RecRegisterManagerCb(REC_EVENT_ROLL_LOG_FILES, &LogConfig::reconfigure_mgmt_variables, nullptr); + RecRegisterManagerCb(REC_EVENT_ROLL_LOG_FILES, &LogConfig::reconfigure_mgmt_variables); } /*------------------------------------------------------------------------- diff --git a/proxy/logging/LogConfig.h b/proxy/logging/LogConfig.h index d8efcf3559b..fc413902783 100644 --- a/proxy/logging/LogConfig.h +++ b/proxy/logging/LogConfig.h @@ -31,6 +31,7 @@ #include "records/P_RecProcess.h" #include "ProxyConfig.h" #include "LogObject.h" +#include "tscpp/util/MemSpan.h" /* Instead of enumerating the stats in DynamicStats.h, each module needs to enumerate its stats separately and register them with librecords @@ -210,7 +211,7 @@ class LogConfig : public ConfigInfo void read_configuration_variables(); // CVR This is the mgmt callback function, hence all the strange arguments - static void *reconfigure_mgmt_variables(void *token, char *data_raw, int data_len); + static void reconfigure_mgmt_variables(ts::MemSpan); int get_max_space_mb() const diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index 365e2e0f7b7..ae6302d620a 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -37,16 +37,18 @@ getStatName(std::string &stat_name, const char *name, const char *reason) } } -static void * -mgmt_host_status_up_callback(void *x, char *data, int len) +static void +mgmt_host_status_up_callback(ts::MemSpan span) { MgmtInt op; MgmtMarshallString name; MgmtMarshallInt down_time; MgmtMarshallString reason; std::string reason_stat; + char *data = static_cast(span.data()); + auto len = span.size(); static const MgmtMarshallType fields[] = {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_STRING, MGMT_MARSHALL_INT}; - Debug("host_statuses", "%s:%s:%d - data: %s, len: %d\n", __FILE__, __func__, __LINE__, data, len); + Debug("host_statuses", "%s:%s:%d - data: %s, len: %ld\n", __FILE__, __func__, __LINE__, data, len); if (mgmt_message_parse(data, len, fields, countof(fields), &op, &name, &reason, &down_time) == -1) { Error("Plugin message - RPC parsing error - message discarded."); @@ -61,19 +63,20 @@ mgmt_host_status_up_callback(void *x, char *data, int len) } hs.setHostStatus(name, HostStatus_t::HOST_STATUS_UP, down_time, reason); } - return nullptr; } -static void * -mgmt_host_status_down_callback(void *x, char *data, int len) +static void +mgmt_host_status_down_callback(ts::MemSpan span) { MgmtInt op; MgmtMarshallString name; MgmtMarshallInt down_time; MgmtMarshallString reason; std::string reason_stat; + char *data = static_cast(span.data()); + auto len = span.size(); static const MgmtMarshallType fields[] = {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_STRING, MGMT_MARSHALL_INT}; - Debug("host_statuses", "%s:%s:%d - data: %s, len: %d\n", __FILE__, __func__, __LINE__, data, len); + Debug("host_statuses", "%s:%s:%d - data: %s, len: %ld\n", __FILE__, __func__, __LINE__, data, len); if (mgmt_message_parse(data, len, fields, countof(fields), &op, &name, &reason, &down_time) == -1) { Error("Plugin message - RPC parsing error - message discarded."); @@ -89,15 +92,14 @@ mgmt_host_status_down_callback(void *x, char *data, int len) } hs.setHostStatus(name, HostStatus_t::HOST_STATUS_DOWN, down_time, reason); } - return nullptr; } HostStatus::HostStatus() { ink_rwlock_init(&host_status_rwlock); ink_rwlock_init(&host_statids_rwlock); - pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, mgmt_host_status_up_callback, nullptr); - pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, mgmt_host_status_down_callback, nullptr); + pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, &mgmt_host_status_up_callback); + pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, &mgmt_host_status_down_callback); host_status_rsb = RecAllocateRawStatBlock((int)TS_MAX_API_STATS); } diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 2b9eaaac474..eea54b40094 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -114,10 +114,10 @@ extern "C" int plock(int); static const long MAX_LOGIN = ink_login_name_max(); -static void *mgmt_restart_shutdown_callback(void *, char *, int data_len); -static void *mgmt_drain_callback(void *, char *, int data_len); -static void *mgmt_storage_device_cmd_callback(void *x, char *data, int len); -static void *mgmt_lifecycle_msg_callback(void *x, char *data, int len); +static void mgmt_restart_shutdown_callback(ts::MemSpan); +static void mgmt_drain_callback(ts::MemSpan); +static void mgmt_storage_device_cmd_callback(int cmd, std::string_view const &arg); +static void mgmt_lifecycle_msg_callback(ts::MemSpan); static void init_ssl_ctx_callback(void *ctx, bool server); static void load_ssl_file_callback(const char *ssl_file, unsigned int options); static void load_remap_file_callback(const char *remap_file); @@ -1982,16 +1982,17 @@ main(int /* argc ATS_UNUSED */, const char **argv) start_SocksProxy(netProcessor.socks_conf_stuff->accept_port); } - pmgmt->registerMgmtCallback(MGMT_EVENT_SHUTDOWN, mgmt_restart_shutdown_callback, nullptr); - pmgmt->registerMgmtCallback(MGMT_EVENT_RESTART, mgmt_restart_shutdown_callback, nullptr); - pmgmt->registerMgmtCallback(MGMT_EVENT_DRAIN, mgmt_drain_callback, nullptr); + pmgmt->registerMgmtCallback(MGMT_EVENT_SHUTDOWN, &mgmt_restart_shutdown_callback); + pmgmt->registerMgmtCallback(MGMT_EVENT_RESTART, &mgmt_restart_shutdown_callback); + pmgmt->registerMgmtCallback(MGMT_EVENT_DRAIN, &mgmt_drain_callback); // Callback for various storage commands. These all go to the same function so we // pass the event code along so it can do the right thing. We cast that to first // just to be safe because the value is a #define, not a typed value. - pmgmt->registerMgmtCallback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, mgmt_storage_device_cmd_callback, - reinterpret_cast(static_cast(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE))); - pmgmt->registerMgmtCallback(MGMT_EVENT_LIFECYCLE_MESSAGE, mgmt_lifecycle_msg_callback, nullptr); + pmgmt->registerMgmtCallback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, [](ts::MemSpan span) -> void { + mgmt_storage_device_cmd_callback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, std::string_view{span}); + }); + pmgmt->registerMgmtCallback(MGMT_EVENT_LIFECYCLE_MESSAGE, &mgmt_lifecycle_msg_callback); ink_set_thread_name("[TS_MAIN]"); @@ -2036,42 +2037,37 @@ REGRESSION_TEST(Hdrs)(RegressionTest *t, int atype, int *pstatus) } #endif -static void * -mgmt_restart_shutdown_callback(void *, char *, int /* data_len ATS_UNUSED */) +static void mgmt_restart_shutdown_callback(ts::MemSpan) { sync_cache_dir_on_shutdown(); - return nullptr; } -static void * -mgmt_drain_callback(void *, char *arg, int len) +static void +mgmt_drain_callback(ts::MemSpan span) { - ts_is_draining = (len == 2 && arg[0] == '1'); + char *arg = static_cast(span.data()); + ts_is_draining = (span.size() == 2 && arg[0] == '1'); RecSetRecordInt("proxy.node.config.draining", ts_is_draining ? 1 : 0, REC_SOURCE_DEFAULT); - return nullptr; } -static void * -mgmt_storage_device_cmd_callback(void *data, char *arg, int len) +static void +mgmt_storage_device_cmd_callback(int cmd, std::string_view const &arg) { // data is the device name to control - CacheDisk *d = cacheProcessor.find_by_path(arg, len); - // Actual command is in @a data. - intptr_t cmd = reinterpret_cast(data); + CacheDisk *d = cacheProcessor.find_by_path(arg.data(), int(arg.size())); if (d) { switch (cmd) { case MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE: - Debug("server", "Marking %.*s offline", len, arg); + Debug("server", "Marking %.*s offline", int(arg.size()), arg.data()); cacheProcessor.mark_storage_offline(d, /* admin */ true); break; } } - return nullptr; } -static void * -mgmt_lifecycle_msg_callback(void *, char *data, int len) +static void +mgmt_lifecycle_msg_callback(ts::MemSpan span) { APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_MSG_HOOK); TSPluginMsg msg; @@ -2080,7 +2076,7 @@ mgmt_lifecycle_msg_callback(void *, char *data, int len) MgmtMarshallData payload; static const MgmtMarshallType fields[] = {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_DATA}; - if (mgmt_message_parse(data, len, fields, countof(fields), &op, &tag, &payload) == -1) { + if (mgmt_message_parse(span.data(), span.size(), fields, countof(fields), &op, &tag, &payload) == -1) { Error("Plugin message - RPC parsing error - message discarded."); } else { msg.tag = tag; @@ -2092,7 +2088,6 @@ mgmt_lifecycle_msg_callback(void *, char *data, int len) hook = hook->next(); } } - return nullptr; } static void From 5b7ce5fdb22d56af9d8c6eb7fb4ac120f6506f81 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 14 Feb 2019 17:53:30 -0600 Subject: [PATCH 288/526] Diags: Make some methods virtual so they can be overridden for unit tests. --- include/tscore/Diags.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/tscore/Diags.h b/include/tscore/Diags.h index f68cce87ef4..f9e8a2d5881 100644 --- a/include/tscore/Diags.h +++ b/include/tscore/Diags.h @@ -113,7 +113,7 @@ class Diags public: Diags(const char *prefix_string, const char *base_debug_tags, const char *base_action_tags, BaseLogFile *_diags_log, int diags_log_perm = -1, int output_log_perm = -1); - ~Diags(); + virtual ~Diags(); BaseLogFile *diags_log; BaseLogFile *stdout_log; @@ -208,7 +208,7 @@ class Diags va_end(ap); } - void error_va(DiagsLevel level, const SourceLocation *loc, const char *fmt, va_list ap) const; + virtual void error_va(DiagsLevel level, const SourceLocation *loc, const char *fmt, va_list ap) const; void dump(FILE *fp = stdout) const; From c83061d68dfbe9e25210a953ea640063a0525ab7 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 14 Feb 2019 16:35:16 -0600 Subject: [PATCH 289/526] LibRecord: Add unit tests. --- CMakeLists.txt | 1 + lib/records/Makefile.am | 21 +++++ lib/records/unit_tests/test_Diags.h | 38 +++++++++ lib/records/unit_tests/test_RecHttp.cc | 99 ++++++++++++++++++++++++ lib/records/unit_tests/unit_test_main.cc | 46 +++++++++++ 5 files changed, 205 insertions(+) create mode 100644 lib/records/unit_tests/test_Diags.h create mode 100644 lib/records/unit_tests/test_RecHttp.cc create mode 100644 lib/records/unit_tests/unit_test_main.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index bfc012dc6b3..68504fba114 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,7 @@ CPP_LIB(tscppapi src/tscpp/api include/tscpp/api) CC_EXEC(test_tscore src/tscore/unit_tests) CC_EXEC(test_tsutil src/tscpp/util/unit_tests) +CC_EXEC(test_librecords lib/records/unit_tests) CPP_LIB(proxy proxy proxy) CPP_ADD_SOURCES(proxy proxy/http) diff --git a/lib/records/Makefile.am b/lib/records/Makefile.am index 195d9089a30..e403a81313d 100644 --- a/lib/records/Makefile.am +++ b/lib/records/Makefile.am @@ -18,6 +18,8 @@ include $(top_srcdir)/build/tidy.mk +check_PROGRAMS = test_librecords + AM_CPPFLAGS += \ -I$(abs_top_srcdir)/iocore/eventsystem \ -I$(abs_top_srcdir)/iocore/utils \ @@ -67,5 +69,24 @@ librecords_p_a_SOURCES = \ P_RecProcess.h \ RecProcess.cc +TESTS = $(check_PROGRAMS) + +test_librecords_CPPFLAGS = $(AM_CPPFLAGS)\ + -I$(abs_top_srcdir)/tests/include + +test_librecords_SOURCES = \ + unit_tests/unit_test_main.cc \ + unit_tests/test_RecHttp.cc + +test_librecords_LDADD = \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/proxy/shared/libUglyLogStubs.a \ + @HWLOC_LIBS@ @LIBCAP@ + + clang-tidy-local: $(sort $(DIST_SOURCES)) $(CXX_Clang_Tidy) diff --git a/lib/records/unit_tests/test_Diags.h b/lib/records/unit_tests/test_Diags.h new file mode 100644 index 00000000000..2819c3a32de --- /dev/null +++ b/lib/records/unit_tests/test_Diags.h @@ -0,0 +1,38 @@ +/** @file + + This file used for catch based tests. It is the main() stub. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include "tscore/Diags.h" + +class CatchDiags : public Diags +{ +public: + mutable std::vector messages; + + CatchDiags() : Diags("catch", "", "", nullptr) {} + + void + error_va(DiagsLevel diags_level, const SourceLocation *loc, const char *fmt, va_list ap) const override + { + char buff[32768]; + vsnprintf(buff, sizeof(buff), fmt, ap); + messages.push_back(std::string{buff}); + va_end(ap); + } +}; diff --git a/lib/records/unit_tests/test_RecHttp.cc b/lib/records/unit_tests/test_RecHttp.cc new file mode 100644 index 00000000000..cba93ebd90f --- /dev/null +++ b/lib/records/unit_tests/test_RecHttp.cc @@ -0,0 +1,99 @@ +/** @file + + Catch-based tests for HdrsUtils.cc + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include + +#include "catch.hpp" + +#include "tscore/BufferWriter.h" +#include "records/I_RecHttp.h" +#include "test_Diags.h" + +using ts::TextView; + +TEST_CASE("RecHttp", "[librecords][RecHttp]") +{ + std::vector ports; + CatchDiags *cdiag = static_cast(diags); + cdiag->messages.clear(); + + SECTION("base") + { + HttpProxyPort::loadValue(ports, "8080"); + REQUIRE(ports.size() == 1); + REQUIRE(ports[0].m_port == 8080); + } + + SECTION("two") + { + HttpProxyPort::loadValue(ports, "8080 8090"); + REQUIRE(ports.size() == 2); + REQUIRE(ports[0].m_port == 8080); + REQUIRE(ports[1].m_port == 8090); + } + + SECTION("family") + { + HttpProxyPort::loadValue(ports, "7070:ipv4:ip-in=192.168.56.1"); + REQUIRE(ports.size() == 1); + REQUIRE(ports[0].m_port == 7070); + REQUIRE(ports[0].m_family == AF_INET); + REQUIRE(ports[0].isSSL() == false); + } + + SECTION("crossed-family") + { + HttpProxyPort::loadValue(ports, "7070:ipv6:ip-in=192.168.56.1"); + REQUIRE(ports.size() == 0); + REQUIRE(cdiag->messages.size() == 2); + REQUIRE(cdiag->messages[0].find("[ipv6]") != std::string::npos); + REQUIRE(cdiag->messages[0].find("[ipv4]") != std::string::npos); + } + + SECTION("ipv6-a") + { + TextView descriptor{"4443:ssl:ip-in=[ffee::24c3:3349:3cee:0143]"}; + HttpProxyPort::loadValue(ports, descriptor.data()); + REQUIRE(ports.size() == 1); + REQUIRE(ports[0].m_port == 4443); + REQUIRE(ports[0].m_family == AF_INET6); + REQUIRE(ports[0].isSSL() == true); + } + + SECTION("dual-addr") + { + TextView descriptor{"4443:ssl:ipv6:ip-out=[ffee::24c3:3349:3cee:0143]:ip-out=10.1.2.3"}; + HttpProxyPort::loadValue(ports, descriptor.data()); + char buff[256]; + ports[0].print(buff, sizeof(buff)); + std::string_view view{buff}; + REQUIRE(ports.size() == 1); + REQUIRE(ports[0].m_port == 4443); + REQUIRE(ports[0].m_family == AF_INET6); + REQUIRE(ports[0].isSSL() == true); + REQUIRE(ports[0].m_outbound_ip6.isValid() == true); + REQUIRE(ports[0].m_outbound_ip4.isValid() == true); + REQUIRE(ports[0].m_inbound_ip.isValid() == false); + REQUIRE(view.find(":ssl") != TextView::npos); + REQUIRE(view.find(":proto") == TextView::npos); // it's default, should not have this. + } +} diff --git a/lib/records/unit_tests/unit_test_main.cc b/lib/records/unit_tests/unit_test_main.cc new file mode 100644 index 00000000000..bfa4a61ba27 --- /dev/null +++ b/lib/records/unit_tests/unit_test_main.cc @@ -0,0 +1,46 @@ +/** @file + + This file used for catch based tests. It is the main() stub. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include "tscore/BufferWriter.h" +#include "tscore/ink_resolver.h" +#include "test_Diags.h" + +#define CATCH_CONFIG_RUNNER +#include "catch.hpp" + +Diags *diags = new CatchDiags; +extern void ts_session_protocol_well_known_name_indices_init(); + +int +main(int argc, char *argv[]) +{ + // Global data initialization needed for the unit tests. + ts_session_protocol_well_known_name_indices_init(); + // Cheat for ts_host_res_global_init as there's no records.config to check for non-default. + memcpy(host_res_default_preference_order, HOST_RES_DEFAULT_PREFERENCE_ORDER, sizeof(host_res_default_preference_order)); + + int result = Catch::Session().run(argc, argv); + + // global clean-up... + + return result; +} From 03cfef14f3d3bcb165879464287dca4a6360a063 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 15 Feb 2019 09:43:08 -0600 Subject: [PATCH 290/526] ink_inet: Fix family string printing, add IpAddr::isAnyAddr. --- include/tscore/ink_inet.h | 9 +++++++++ src/tscore/ink_inet.cc | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index b72aefa37b6..364d6be4278 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -1282,6 +1282,9 @@ struct IpAddr { /// Test for loopback bool isLoopback() const; + /// Test for any addr + bool isAnyAddr() const; + uint16_t _family; ///< Protocol family. /// Address data. union { @@ -1341,6 +1344,12 @@ IpAddr::isLoopback() const return (AF_INET == _family && 0x7F == _addr._byte[0]) || (AF_INET6 == _family && IN6_IS_ADDR_LOOPBACK(&_addr._ip6)); } +inline bool +IpAddr::isAnyAddr() const +{ + return (AF_INET == _family && INADDR_ANY == _addr._ip4) || (AF_INET6 == _family && IN6_IS_ADDR_UNSPECIFIED(&_addr._ip6)); +} + /// Assign sockaddr storage. inline IpAddr & IpAddr::assign(sockaddr const *addr) diff --git a/src/tscore/ink_inet.cc b/src/tscore/ink_inet.cc index c52c316c180..3f276c4e89f 100644 --- a/src/tscore/ink_inet.cc +++ b/src/tscore/ink_inet.cc @@ -841,7 +841,7 @@ bwformat(BufferWriter &w, BWFSpec const &spec, IpAddr const &addr) if (spec.has_numeric_type()) { bwformat(w, local_spec, static_cast(addr.family())); } else { - bwformat(w, local_spec, addr.family()); + bwformat(w, local_spec, ats_ip_family_name(addr.family())); } } return w; From 2906f166c0f02d5efd731819a121092d5c8fae49 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Sat, 2 Feb 2019 23:17:55 +0800 Subject: [PATCH 291/526] Optimize: rewrite getbyname_imm and getSRVbyname_imm as wrappers for getby --- iocore/hostdb/HostDB.cc | 291 ++++++++++++------------------ iocore/hostdb/I_HostDBProcessor.h | 17 +- proxy/http/HttpSM.cc | 8 +- 3 files changed, 124 insertions(+), 192 deletions(-) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index fbf410c2438..5b81c858277 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -605,57 +605,77 @@ HostDBContinuation::insert(unsigned int attl) // Get an entry by either name or IP // Action * -HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockaddr const *ip, bool aforce_dns, - HostResStyle host_res_style, int dns_lookup_timeout) +HostDBProcessor::getby(Continuation *cont, cb_process_result_pfn cb_process_result, HostDBHash &hash, Options const &opt) { - HostDBHash hash; + bool trylock = (cb_process_result == nullptr); + bool force_dns = false; EThread *thread = this_ethread(); Ptr mutex = thread->mutex; ip_text_buffer ipb; + if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { + force_dns = true; + } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { + force_dns = hostdb_re_dns_on_reload; + if (force_dns) { + HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); + } + } + HOSTDB_INCREMENT_DYN_STAT(hostdb_total_lookups_stat); - if ((!hostdb_enable || (hostname && !*hostname)) || (hostdb_disable_reverse_lookup && ip)) { - MUTEX_TRY_LOCK(lock, cont->mutex, thread); - if (!lock.is_locked()) { - goto Lretry; + if (!hostdb_enable || // if the HostDB is disabled, + (hash.host_name && !*hash.host_name) || // or host_name is empty string + (hostdb_disable_reverse_lookup && hash.ip.isValid())) { // or try to lookup by ip address when the reverse lookup disabled + if (cb_process_result) { + (cont->*cb_process_result)(nullptr); + } else { + MUTEX_TRY_LOCK(lock, cont->mutex, thread); + if (!lock.is_locked()) { + goto Lretry; + } + cont->handleEvent(EVENT_HOST_DB_LOOKUP, nullptr); } - cont->handleEvent(EVENT_HOST_DB_LOOKUP, nullptr); return ACTION_RESULT_DONE; } - // Load the hash data. - hash.set_host(hostname, hostname ? (len ? len : strlen(hostname)) : 0); - hash.ip.assign(ip); - hash.port = ip ? ats_ip_port_host_order(ip) : 0; - hash.db_mark = db_mark_for(host_res_style); - hash.refresh(); - // Attempt to find the result in-line, for level 1 hits - // - if (!aforce_dns) { + if (!force_dns) { MUTEX_TRY_LOCK(lock, cont->mutex, thread); bool loop = lock.is_locked(); while (loop) { loop = false; // Only loop on explicit set for retry. // find the partition lock - // Ptr bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); MUTEX_TRY_LOCK(lock2, bucket_mutex, thread); - + if (!lock2.is_locked() && !trylock) { + // Refer to: [TS-374](http://issues.apache.org/jira/browse/TS-374) + lock2.acquire(thread); + } if (lock2.is_locked()) { // If we can get the lock and a level 1 probe succeeds, return - Ptr r = probe(bucket_mutex, hash, aforce_dns); + Ptr r = probe(bucket_mutex, hash, false); if (r) { - if (r->is_failed() && hostname) { - loop = check_for_retry(hash.db_mark, host_res_style); + // fail, see if we should retry with alternate + if (hash.db_mark != HOSTDB_MARK_SRV && r->is_failed() && hash.host_name) { + loop = check_for_retry(hash.db_mark, opt.host_res_style); } if (!loop) { // No retry -> final result. Return it. - Debug("hostdb", "immediate answer for %s", - hostname ? hostname : ats_is_ip(ip) ? ats_ip_ntop(ip, ipb, sizeof ipb) : ""); + if (hash.db_mark == HOSTDB_MARK_SRV) { + Debug("hostdb", "immediate SRV answer for %.*s from hostdb", hash.host_len, hash.host_name); + Debug("dns_srv", "immediate SRV answer for %.*s from hostdb", hash.host_len, hash.host_name); + } else if (hash.host_name) { + Debug("hostdb", "immediate answer for %.*s", hash.host_len, hash.host_name); + } else { + Debug("hostdb", "immediate answer for %s", hash.ip.isValid() ? hash.ip.toString(ipb, sizeof ipb) : ""); + } HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat); - reply_to_cont(cont, r.get()); + if (cb_process_result) { + (cont->*cb_process_result)(r.get()); + } else { + reply_to_cont(cont, r.get()); + } return ACTION_RESULT_DONE; } hash.refresh(); // only on reloop, because we've changed the family. @@ -663,19 +683,28 @@ HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockad } } } - Debug("hostdb", "delaying force %d answer for %s", aforce_dns, - hostname ? hostname : ats_is_ip(ip) ? ats_ip_ntop(ip, ipb, sizeof ipb) : ""); + if (hash.db_mark == HOSTDB_MARK_SRV) { + Debug("hostdb", "delaying (force=%d) SRV answer for %.*s [timeout = %d]", force_dns, hash.host_len, hash.host_name, + opt.timeout); + Debug("dns_srv", "delaying (force=%d) SRV answer for %.*s [timeout = %d]", force_dns, hash.host_len, hash.host_name, + opt.timeout); + } else if (hash.host_name) { + Debug("hostdb", "delaying (force=%d) answer for %.*s [timeout %d]", force_dns, hash.host_len, hash.host_name, opt.timeout); + } else { + Debug("hostdb", "delaying (force=%d) answer for %s [timeout %d]", force_dns, + hash.ip.isValid() ? hash.ip.toString(ipb, sizeof ipb) : "", opt.timeout); + } Lretry: // Otherwise, create a continuation to do a deeper probe in the background // HostDBContinuation *c = hostDBContAllocator.alloc(); - HostDBContinuation::Options opt; - opt.timeout = dns_lookup_timeout; - opt.force_dns = aforce_dns; - opt.cont = cont; - opt.host_res_style = host_res_style; - c->init(hash, opt); + HostDBContinuation::Options copt; + copt.timeout = opt.timeout; + copt.force_dns = force_dns; + copt.cont = cont; + copt.host_res_style = (hash.db_mark == HOSTDB_MARK_SRV) ? HOST_RES_NONE : opt.host_res_style; + c->init(hash, copt); SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent); thread->schedule_in(c, MUTEX_RETRY_DELAY); @@ -688,188 +717,96 @@ HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockad Action * HostDBProcessor::getbyname_re(Continuation *cont, const char *ahostname, int len, Options const &opt) { - bool force_dns = false; - EThread *thread = this_ethread(); - ProxyMutex *mutex = thread->mutex.get(); + HostDBHash hash; - if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { - force_dns = true; - } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { - force_dns = (hostdb_re_dns_on_reload ? true : false); - if (force_dns) { - HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); - } - } - return getby(cont, ahostname, len, nullptr, force_dns, opt.host_res_style, opt.timeout); + ink_assert(nullptr != ahostname); + + // Load the hash data. + hash.set_host(ahostname, ahostname ? (len ? len : strlen(ahostname)) : 0); + // Leave hash.ip invalid + hash.port = 0; + hash.db_mark = db_mark_for(opt.host_res_style); + hash.refresh(); + + return getby(cont, nullptr, hash, opt); } Action * HostDBProcessor::getbynameport_re(Continuation *cont, const char *ahostname, int len, Options const &opt) { - bool force_dns = false; - EThread *thread = this_ethread(); - ProxyMutex *mutex = thread->mutex.get(); + HostDBHash hash; - if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { - force_dns = true; - } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { - force_dns = (hostdb_re_dns_on_reload ? true : false); - if (force_dns) { - HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); - } - } - sockaddr sa; - ats_ip4_set(&sa, INADDR_ANY, htons(opt.port)); - return getby(cont, ahostname, len, &sa, force_dns, opt.host_res_style, opt.timeout); + ink_assert(nullptr != ahostname); + + // Load the hash data. + hash.set_host(ahostname, ahostname ? (len ? len : strlen(ahostname)) : 0); + // Leave hash.ip invalid + hash.port = opt.port; + hash.db_mark = db_mark_for(opt.host_res_style); + hash.refresh(); + + return getby(cont, nullptr, hash, opt); } -/* Support SRV records */ +// Lookup Hostinfo by addr Action * -HostDBProcessor::getSRVbyname_imm(Continuation *cont, process_srv_info_pfn process_srv_info, const char *hostname, int len, - Options const &opt) +HostDBProcessor::getbyaddr_re(Continuation *cont, sockaddr const *aip) { - ink_assert(cont->mutex->thread_holding == this_ethread()); - bool force_dns = false; - EThread *thread = cont->mutex->thread_holding; - ProxyMutex *mutex = thread->mutex.get(); - - ink_assert(nullptr != hostname); - - if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { - force_dns = true; - } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { - force_dns = (hostdb_re_dns_on_reload ? true : false); - if (force_dns) { - HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); - } - } - HostDBHash hash; - HOSTDB_INCREMENT_DYN_STAT(hostdb_total_lookups_stat); + ink_assert(nullptr != aip); - if (!hostdb_enable || !*hostname) { - (cont->*process_srv_info)(nullptr); - return ACTION_RESULT_DONE; - } + HostDBProcessor::Options opt; + opt.host_res_style = HOST_RES_NONE; - hash.host_name = hostname; - hash.host_len = len ? len : strlen(hostname); - hash.port = 0; - hash.db_mark = HOSTDB_MARK_SRV; + // Leave hash.host_name as nullptr + hash.ip.assign(aip); + hash.port = ats_ip_port_host_order(aip); + hash.db_mark = db_mark_for(opt.host_res_style); hash.refresh(); - // Attempt to find the result in-line, for level 1 hits - if (!force_dns) { - // find the partition lock - Ptr bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); - MUTEX_TRY_LOCK(lock, bucket_mutex, thread); - - // If we can get the lock and a level 1 probe succeeds, return - if (lock.is_locked()) { - Ptr r = probe(bucket_mutex, hash, false); - if (r) { - Debug("hostdb", "immediate SRV answer for %s from hostdb", hostname); - Debug("dns_srv", "immediate SRV answer for %s from hostdb", hostname); - HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat); - (cont->*process_srv_info)(r.get()); - return ACTION_RESULT_DONE; - } - } - } + return getby(cont, nullptr, hash, opt); +} - Debug("dns_srv", "delaying (force=%d) SRV answer for %.*s [timeout = %d]", force_dns, hash.host_len, hash.host_name, opt.timeout); +/* Support SRV records */ +Action * +HostDBProcessor::getSRVbyname_imm(Continuation *cont, cb_process_result_pfn process_srv_info, const char *hostname, int len, + Options const &opt) +{ + ink_assert(cont->mutex->thread_holding == this_ethread()); + HostDBHash hash; - // Otherwise, create a continuation to do a deeper probe in the background - HostDBContinuation *c = hostDBContAllocator.alloc(); - HostDBContinuation::Options copt; - copt.timeout = opt.timeout; - copt.cont = cont; - copt.force_dns = force_dns; - c->init(hash, copt); - SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent); + ink_assert(nullptr != hostname); - thread->schedule_in(c, MUTEX_RETRY_DELAY); + hash.set_host(hostname, len ? len : strlen(hostname)); + // Leave hash.ip invalid + hash.port = 0; + hash.db_mark = HOSTDB_MARK_SRV; + hash.refresh(); - return &c->action; + return getby(cont, process_srv_info, hash, opt); } // Wrapper from getbyname to getby // Action * -HostDBProcessor::getbyname_imm(Continuation *cont, process_hostdb_info_pfn process_hostdb_info, const char *hostname, int len, +HostDBProcessor::getbyname_imm(Continuation *cont, cb_process_result_pfn process_hostdb_info, const char *hostname, int len, Options const &opt) { ink_assert(cont->mutex->thread_holding == this_ethread()); - bool force_dns = false; - EThread *thread = cont->mutex->thread_holding; - ProxyMutex *mutex = thread->mutex.get(); HostDBHash hash; ink_assert(nullptr != hostname); - if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { - force_dns = true; - } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { - force_dns = (hostdb_re_dns_on_reload ? true : false); - if (force_dns) { - HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); - } - } - - HOSTDB_INCREMENT_DYN_STAT(hostdb_total_lookups_stat); - - if (!hostdb_enable || !*hostname) { - (cont->*process_hostdb_info)(nullptr); - return ACTION_RESULT_DONE; - } - hash.set_host(hostname, len ? len : strlen(hostname)); + // Leave hash.ip invalid + // TODO: May I rename the wrapper name to getbynameport_imm ? - oknet + // By comparing getbyname_re and getbynameport_re, the hash.port should be 0 if only get hostinfo by name. hash.port = opt.port; hash.db_mark = db_mark_for(opt.host_res_style); hash.refresh(); - // Attempt to find the result in-line, for level 1 hits - if (!force_dns) { - bool loop; - do { - loop = false; // loop only on explicit set for retry - // find the partition lock - Ptr bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); - SCOPED_MUTEX_LOCK(lock, bucket_mutex, thread); - // do a level 1 probe for immediate result. - Ptr r = probe(bucket_mutex, hash, false); - if (r) { - if (r->is_failed()) { // fail, see if we should retry with alternate - loop = check_for_retry(hash.db_mark, opt.host_res_style); - } - if (!loop) { - // No retry -> final result. Return it. - Debug("hostdb", "immediate answer for %.*s", hash.host_len, hash.host_name); - HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat); - (cont->*process_hostdb_info)(r.get()); - return ACTION_RESULT_DONE; - } - hash.refresh(); // Update for retry. - } - } while (loop); - } - - Debug("hostdb", "delaying force %d answer for %.*s [timeout %d]", force_dns, hash.host_len, hash.host_name, opt.timeout); - - // Otherwise, create a continuation to do a deeper probe in the background - HostDBContinuation *c = hostDBContAllocator.alloc(); - HostDBContinuation::Options copt; - copt.cont = cont; - copt.force_dns = force_dns; - copt.timeout = opt.timeout; - copt.host_res_style = opt.host_res_style; - c->init(hash, copt); - SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent); - - thread->schedule_in(c, HOST_DB_RETRY_PERIOD); - - return &c->action; + return getby(cont, process_hostdb_info, hash, opt); } Action * diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h index a9f5d0cf096..347f1a94832 100644 --- a/iocore/hostdb/I_HostDBProcessor.h +++ b/iocore/hostdb/I_HostDBProcessor.h @@ -393,11 +393,11 @@ struct HostDBRoundRobin { }; struct HostDBCache; +struct HostDBHash; // Prototype for inline completion functionf or // getbyname_imm() -typedef void (Continuation::*process_hostdb_info_pfn)(HostDBInfo *r); -typedef void (Continuation::*process_srv_info_pfn)(HostDBInfo *r); +typedef void (Continuation::*cb_process_result_pfn)(HostDBInfo *r); Action *iterate(Continuation *cont); @@ -448,20 +448,16 @@ struct HostDBProcessor : public Processor { Action *getbynameport_re(Continuation *cont, const char *hostname, int len, Options const &opt = DEFAULT_OPTIONS); - Action *getSRVbyname_imm(Continuation *cont, process_srv_info_pfn process_srv_info, const char *hostname, int len, + Action *getSRVbyname_imm(Continuation *cont, cb_process_result_pfn process_srv_info, const char *hostname, int len, Options const &opt = DEFAULT_OPTIONS); - Action *getbyname_imm(Continuation *cont, process_hostdb_info_pfn process_hostdb_info, const char *hostname, int len, + Action *getbyname_imm(Continuation *cont, cb_process_result_pfn process_hostdb_info, const char *hostname, int len, Options const &opt = DEFAULT_OPTIONS); Action *iterate(Continuation *cont); /** Lookup Hostinfo by addr */ - Action * - getbyaddr_re(Continuation *cont, sockaddr const *aip) - { - return getby(cont, nullptr, 0, aip, false, HOST_RES_NONE, 0); - } + Action *getbyaddr_re(Continuation *cont, sockaddr const *aip); /** Set the application information (fire-and-forget). */ void @@ -500,8 +496,7 @@ struct HostDBProcessor : public Processor { HostDBCache *cache(); private: - Action *getby(Continuation *cont, const char *hostname, int len, sockaddr const *ip, bool aforce_dns, HostResStyle host_res_style, - int timeout); + Action *getby(Continuation *cont, cb_process_result_pfn cb_process_result, HostDBHash &hash, Options const &opt); public: /** Set something. diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 3aa0cee28dd..8f2c3d0a327 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -2249,7 +2249,7 @@ HttpSM::state_hostdb_lookup(int event, void *data) opt.host_res_style = ua_txn->get_host_res_style(); Action *dns_lookup_action_handle = - hostDBProcessor.getbyname_imm(this, (process_hostdb_info_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); + hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); if (dns_lookup_action_handle != ACTION_RESULT_DONE) { ink_assert(!pending_action); pending_action = dns_lookup_action_handle; @@ -4047,7 +4047,7 @@ HttpSM::do_hostdb_lookup() opt.timeout = t_state.api_txn_dns_timeout_value; } Action *srv_lookup_action_handle = - hostDBProcessor.getSRVbyname_imm(this, (process_srv_info_pfn)&HttpSM::process_srv_info, d, 0, opt); + hostDBProcessor.getSRVbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_srv_info, d, 0, opt); if (srv_lookup_action_handle != ACTION_RESULT_DONE) { ink_assert(!pending_action); @@ -4064,7 +4064,7 @@ HttpSM::do_hostdb_lookup() opt.host_res_style = ua_txn->get_host_res_style(); Action *dns_lookup_action_handle = - hostDBProcessor.getbyname_imm(this, (process_hostdb_info_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); + hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); if (dns_lookup_action_handle != ACTION_RESULT_DONE) { ink_assert(!pending_action); pending_action = dns_lookup_action_handle; @@ -4098,7 +4098,7 @@ HttpSM::do_hostdb_lookup() opt.timeout = (t_state.api_txn_dns_timeout_value != -1) ? t_state.api_txn_dns_timeout_value : 0; opt.host_res_style = ua_txn->get_host_res_style(); - Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(this, (process_hostdb_info_pfn)&HttpSM::process_hostdb_info, + Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, t_state.dns_info.lookup_name, 0, opt); if (dns_lookup_action_handle != ACTION_RESULT_DONE) { From 331d3788f527659bd0633addfa4ca48b37d92c53 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 17 Jan 2019 15:06:06 -0600 Subject: [PATCH 292/526] TextView: add overflow checking to svto_radix. --- include/tscpp/util/TextView.h | 24 ++++++++++++++++-------- src/tscpp/util/TextView.cc | 7 ++++++- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/include/tscpp/util/TextView.h b/include/tscpp/util/TextView.h index 890ef3071ee..ce620dfc8ed 100644 --- a/include/tscpp/util/TextView.h +++ b/include/tscpp/util/TextView.h @@ -535,13 +535,18 @@ intmax_t svtoi(TextView src, TextView *parsed = nullptr, int base = 0); * @param src The source text. Updated during parsing. * @return The converted numeric value. * - * This is a specialized function useful only where conversion performance is critical. It is used - * inside @c svtoi for the common cases of 8, 10, and 16, therefore normally this isn't much more - * performant in those cases than just @c svtoi. Because of this only positive values are parsed. - * If determining the radix from the text or signed value parsing is needed, used @c svtoi. + * This is a specialized function useful only where conversion performance is critical, or for some + * other reason the numeric text has already been parsed out. The performance gains comes from + * templating the divisor which enables the compiler to optimize the multiplication (e.g., for + * powers of 2 shifts is used). It is used inside @c svtoi and @c svtou for the common cases of 8, + * 10, and 16, therefore normally this isn't much more performant than @c svtoi. Because of this + * only positive values are parsed. If determining the radix from the text or signed value parsing + * is needed, used @c svtoi. * - * @a src is updated in place to indicate what characters were parsed. Parsing stops on the first - * invalid digit, so any leading non-digit characters (e.g. whitespace) must already be removed. + * @a src is updated in place by removing parsed characters. Parsing stops on the first invalid + * digit, so any leading non-digit characters (e.g. whitespace) must already be removed. Overflow + * is detected and the first digit that would overflow is not parsed, and the maximum value is + * returned. */ template uintmax_t @@ -551,8 +556,11 @@ svto_radix(ts::TextView &src) uintmax_t zret{0}; uintmax_t v; while (src.size() && (0 <= (v = ts::svtoi_convert[static_cast(*src)])) && v < N) { - zret *= N; - zret += v; + auto n = zret * N + v; + if (n < zret) { // overflow / wrap + return std::numeric_limits::max(); + } + zret = n; ++src; } return zret; diff --git a/src/tscpp/util/TextView.cc b/src/tscpp/util/TextView.cc index b6b1135eb62..4532b027fa2 100644 --- a/src/tscpp/util/TextView.cc +++ b/src/tscpp/util/TextView.cc @@ -131,7 +131,12 @@ ts::svtoi(TextView src, TextView *out, int base) break; default: while (src.size() && (0 <= (v = svtoi_convert[static_cast(*src)])) && v < base) { - zret = zret * base + v; + auto n = zret * base + v; + if (n < zret) { + zret = std::numeric_limits::max(); + break; // overflow, stop parsing. + } + zret = n; ++src; } break; From b4f8d1c2ca5895c81dd7137b11620e421c1e8582 Mon Sep 17 00:00:00 2001 From: Aaron Canary Date: Mon, 18 Feb 2019 16:12:39 -0600 Subject: [PATCH 293/526] Removing Lazy Buf Alloc from H1 Server session Because Susan did it i her branch. And she said it never really worked right. And I like to KIWF. --- proxy/http/Http1ServerSession.cc | 6 ++---- proxy/http/Http1ServerSession.h | 11 ----------- proxy/http/HttpSM.cc | 5 +---- proxy/http/HttpSM.h | 13 ------------- proxy/http/HttpTunnel.cc | 10 ++++------ 5 files changed, 7 insertions(+), 38 deletions(-) diff --git a/proxy/http/Http1ServerSession.cc b/proxy/http/Http1ServerSession.cc index e70dac262e9..3c2361a0441 100644 --- a/proxy/http/Http1ServerSession.cc +++ b/proxy/http/Http1ServerSession.cc @@ -74,11 +74,9 @@ Http1ServerSession::new_connection(NetVConnection *new_vc) magic = HTTP_SS_MAGIC_ALIVE; HTTP_SUM_GLOBAL_DYN_STAT(http_current_server_connections_stat, 1); // Update the true global stat HTTP_INCREMENT_DYN_STAT(http_total_server_connections_stat); -#ifdef LAZY_BUF_ALLOC - read_buffer = new_empty_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); -#else + read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); -#endif + buf_reader = read_buffer->alloc_reader(); Debug("http_ss", "[%" PRId64 "] session born, netvc %p", con_id, new_vc); state = HSS_INIT; diff --git a/proxy/http/Http1ServerSession.h b/proxy/http/Http1ServerSession.h index a66bf1a64fe..69d3dab9dae 100644 --- a/proxy/http/Http1ServerSession.h +++ b/proxy/http/Http1ServerSession.h @@ -31,17 +31,6 @@ ****************************************************************************/ #pragma once -/* Enable LAZY_BUF_ALLOC to delay allocation of buffers until they - * are actually required. - * Enabling LAZY_BUF_ALLOC, stop Http code from allocation space - * for header buffer and tunnel buffer. The allocation is done by - * the net code in read_from_net when data is actually written into - * the buffer. By allocating memory only when it is required we can - * reduce the memory consumed by TS process. - * - * IMPORTANT NOTE: enable/disable LAZY_BUF_ALLOC in HttpSM.h as well. - */ -#define LAZY_BUF_ALLOC #include "P_Net.h" diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 8f2c3d0a327..1707beb7058 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -6403,10 +6403,7 @@ HttpSM::server_transfer_init(MIOBuffer *buf, int hdr_size) if (server_response_pre_read_bytes == to_copy && server_buffer_reader->read_avail() > 0) { t_state.current.server->keep_alive = HTTP_NO_KEEPALIVE; } -#ifdef LAZY_BUF_ALLOC - // reset the server session buffer - server_session->reset_read_buffer(); -#endif + return nbytes; } diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index 11a40e20a4f..6b55386e006 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -45,19 +45,6 @@ #include "tscore/History.h" //#include "AuthHttpAdapter.h" -/* Enable LAZY_BUF_ALLOC to delay allocation of buffers until they - * are actually required. - * Enabling LAZY_BUF_ALLOC, stop Http code from allocation space - * for header buffer and tunnel buffer. The allocation is done by - * the net code in read_from_net when data is actually written into - * the buffer. By allocating memory only when it is required we can - * reduce the memory consumed by TS process. - * - * IMPORTANT NOTE: enable/disable LAZY_BUF_ALLOC in Http1ServerSession.h - * as well. - */ -#define LAZY_BUF_ALLOC - #define HTTP_API_CONTINUE (INK_API_EVENT_EVENTS_START + 0) #define HTTP_API_ERROR (INK_API_EVENT_EVENTS_START + 1) diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 36d9508331d..9e0d05202ba 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -1272,10 +1272,8 @@ HttpTunnel::consumer_reenable(HttpTunnelConsumer *c) { HttpTunnelProducer *p = c->producer; - if (p && p->alive -#ifndef LAZY_BUF_ALLOC - && p->read_buffer->write_avail() > 0 -#endif + if (p && p->alive && p->read_buffer->write_avail() > 0 + ) { // Only do flow control if enabled and the producer is an external // source. Otherwise disable by making the backlog zero. Because @@ -1398,9 +1396,9 @@ HttpTunnel::consumer_handler(int event, HttpTunnelConsumer *c) // updating the buffer state for the VConnection // that is being reenabled if (p->alive && p->read_vio -#ifndef LAZY_BUF_ALLOC + && p->read_buffer->write_avail() > 0 -#endif + ) { if (p->is_throttled()) { this->consumer_reenable(c); From 3b0a6f07e39e399af4db140d3c147091fd572d58 Mon Sep 17 00:00:00 2001 From: Aaron Canary Date: Tue, 19 Feb 2019 10:28:49 -0600 Subject: [PATCH 294/526] rm unused func reset_read_buffer --- proxy/http/Http1ServerSession.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/proxy/http/Http1ServerSession.h b/proxy/http/Http1ServerSession.h index 69d3dab9dae..fd8e25511f1 100644 --- a/proxy/http/Http1ServerSession.h +++ b/proxy/http/Http1ServerSession.h @@ -77,16 +77,6 @@ class Http1ServerSession : public VConnection */ void enable_outbound_connection_tracking(OutboundConnTrack::Group *group); - void - reset_read_buffer(void) - { - ink_assert(read_buffer->_writer); - ink_assert(buf_reader != nullptr); - read_buffer->dealloc_all_readers(); - read_buffer->_writer = nullptr; - buf_reader = read_buffer->alloc_reader(); - } - IOBufferReader * get_reader() { From 837250cc6af9be61c1b5b495dd40aa08aa341067 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Thu, 21 Feb 2019 11:58:32 +0900 Subject: [PATCH 295/526] Cleanup: Set SSL_OP_NO_TICKET in SSLInitServerContext() --- iocore/net/SSLUtils.cc | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 305317af312..111ec1ba769 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -1872,7 +1872,16 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu goto fail; } } + +#if defined(SSL_OP_NO_TICKET) + // Session tickets are enabled by default. Disable if explicitly requested. + if (sslMultCertSettings->session_ticket_enabled == 0) { + SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET); + Debug("ssl", "ssl session ticket is disabled"); + } +#endif } + if (params->clientCertLevel != 0) { if (params->serverCACertFilename != nullptr && params->serverCACertPath != nullptr) { if ((!SSL_CTX_load_verify_locations(ctx, params->serverCACertFilename, params->serverCACertPath)) || @@ -2030,6 +2039,10 @@ SSLCreateServerContext(const SSLConfigParams *params) return ctx; } +/** + Insert SSLCertContext (SSL_CTX ans options) into SSLCertLookup with key. + Do NOT call SSL_CTX_set_* functions from here. SSL_CTX should be set up by SSLInitServerContext(). + */ static SSL_CTX * ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, const ssl_user_config *sslMultCertSettings) { @@ -2088,14 +2101,6 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons #endif } -#if defined(SSL_OP_NO_TICKET) - // Session tickets are enabled by default. Disable if explicitly requested. - if (sslMultCertSettings->session_ticket_enabled == 0) { - SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET); - Debug("ssl", "ssl session ticket is disabled"); - } -#endif - #ifdef TS_USE_TLS_OCSP if (SSLConfigParams::ssl_ocsp_enabled) { Debug("ssl", "SSL OCSP Stapling is enabled"); From c811aea9e0484433fbdd63e0fa6b9fbab87085eb Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Thu, 21 Feb 2019 15:43:58 +0900 Subject: [PATCH 296/526] Cleanup: Separate SSLStats and SSLDiags from SSLUtils --- iocore/net/Makefile.am | 2 + iocore/net/OCSPStapling.cc | 1 + iocore/net/P_SSLUtils.h | 104 -------- iocore/net/SSLClientUtils.cc | 2 + iocore/net/SSLConfig.cc | 16 +- iocore/net/SSLDiags.cc | 224 +++++++++++++++++ iocore/net/SSLDiags.h | 43 ++++ iocore/net/SSLNetProcessor.cc | 1 + iocore/net/SSLNetVConnection.cc | 16 +- iocore/net/SSLSessionCache.cc | 2 + iocore/net/SSLStats.cc | 239 ++++++++++++++++++ iocore/net/SSLStats.h | 115 +++++++++ iocore/net/SSLUtils.cc | 420 +------------------------------- 13 files changed, 660 insertions(+), 525 deletions(-) create mode 100644 iocore/net/SSLDiags.cc create mode 100644 iocore/net/SSLDiags.h create mode 100644 iocore/net/SSLStats.cc create mode 100644 iocore/net/SSLStats.h diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index 9eb93963b71..0971d5f70d7 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -136,6 +136,7 @@ libinknet_a_SOURCES = \ SSLCertLookup.cc \ SSLSessionCache.cc \ SSLConfig.cc \ + SSLDiags.cc \ SSLInternal.cc \ SSLNetAccept.cc \ SSLNetProcessor.cc \ @@ -143,6 +144,7 @@ libinknet_a_SOURCES = \ SSLNextProtocolAccept.cc \ SSLNextProtocolSet.cc \ SSLSNIConfig.cc \ + SSLStats.cc \ SSLUtils.cc \ SSLClientUtils.cc \ OCSPStapling.cc \ diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index b4fcd77e8a7..26316ce9cf5 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -27,6 +27,7 @@ #include "P_Net.h" #include "P_SSLConfig.h" #include "P_SSLUtils.h" +#include "SSLStats.h" // Maxiumum OCSP stapling response size. // This should be the response for a single certificate and will typically include the responder certificate chain, diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index e70dbc551c4..a61d8bcf435 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -33,92 +33,12 @@ #endif #include -#include - struct SSLConfigParams; struct SSLCertLookup; class SSLNetVConnection; -struct RecRawStatBlock; typedef int ssl_error_t; -enum SSL_Stats { - ssl_origin_server_expired_cert_stat, - ssl_user_agent_expired_cert_stat, - ssl_origin_server_revoked_cert_stat, - ssl_user_agent_revoked_cert_stat, - ssl_origin_server_unknown_cert_stat, - ssl_user_agent_unknown_cert_stat, - ssl_origin_server_cert_verify_failed_stat, - ssl_user_agent_cert_verify_failed_stat, - ssl_origin_server_bad_cert_stat, - ssl_user_agent_bad_cert_stat, - ssl_origin_server_decryption_failed_stat, - ssl_user_agent_decryption_failed_stat, - ssl_origin_server_wrong_version_stat, - ssl_user_agent_wrong_version_stat, - ssl_origin_server_other_errors_stat, - ssl_user_agent_other_errors_stat, - ssl_origin_server_unknown_ca_stat, - ssl_user_agent_unknown_ca_stat, - ssl_user_agent_sessions_stat, - ssl_user_agent_session_hit_stat, - ssl_user_agent_session_miss_stat, - ssl_user_agent_session_timeout_stat, - ssl_total_handshake_time_stat, - ssl_total_success_handshake_count_in_stat, - ssl_total_tickets_created_stat, - ssl_total_tickets_verified_stat, - ssl_total_tickets_verified_old_key_stat, // verified with old key. - ssl_total_ticket_keys_renewed_stat, // number of keys renewed. - ssl_total_tickets_not_found_stat, - ssl_total_tickets_renewed_stat, - ssl_total_dyn_def_tls_record_count, - ssl_total_dyn_max_tls_record_count, - ssl_total_dyn_redo_tls_record_count, - ssl_session_cache_hit, - ssl_session_cache_miss, - ssl_session_cache_eviction, - ssl_session_cache_lock_contention, - ssl_session_cache_new_session, - - /* error stats */ - ssl_error_want_write, - ssl_error_want_read, - ssl_error_want_client_hello_cb, - ssl_error_want_x509_lookup, - ssl_error_syscall, - ssl_error_read_eos, - ssl_error_zero_return, - ssl_error_ssl, - ssl_sni_name_set_failure, - ssl_total_success_handshake_count_out_stat, - - /* ocsp stapling stats */ - ssl_ocsp_revoked_cert_stat, - ssl_ocsp_unknown_cert_stat, - ssl_ocsp_refreshed_cert_stat, - ssl_ocsp_refresh_cert_failure_stat, - - ssl_cipher_stats_start = 100, - ssl_cipher_stats_end = 300, - - Ssl_Stat_Count -}; - -extern RecRawStatBlock *ssl_rsb; - -/* Stats should only be accessed using these macros */ -#define SSL_INCREMENT_DYN_STAT(x) RecIncrRawStat(ssl_rsb, nullptr, (int)x, 1) -#define SSL_DECREMENT_DYN_STAT(x) RecIncrRawStat(ssl_rsb, nullptr, (int)x, -1) -#define SSL_SET_COUNT_DYN_STAT(x, count) RecSetRawStatCount(ssl_rsb, x, count) -#define SSL_INCREMENT_DYN_STAT_EX(x, y) RecIncrRawStat(ssl_rsb, nullptr, (int)x, y) -#define SSL_CLEAR_DYN_STAT(x) \ - do { \ - RecSetRawStatSum(ssl_rsb, (x), 0); \ - RecSetRawStatCount(ssl_rsb, (x), 0); \ - } while (0) - // Create a default SSL server context. SSL_CTX *SSLDefaultServerContext(); @@ -131,9 +51,6 @@ void SSLInitializeLibrary(); // Initialize SSL library based on configuration settings void SSLPostConfigInitialize(); -// Initialize SSL statistics. -void SSLInitializeStatistics(); - // Release SSL_CTX and the associated data. This works for both // client and server contexts and gracefully accepts nullptr. void SSLReleaseContext(SSL_CTX *ctx); @@ -144,27 +61,6 @@ ssl_error_t SSLReadBuffer(SSL *ssl, void *buf, int64_t nbytes, int64_t &nread); ssl_error_t SSLAccept(SSL *ssl); ssl_error_t SSLConnect(SSL *ssl); -// Log an SSL error. -#define SSLError(fmt, ...) SSLDiagnostic(MakeSourceLocation(), false, nullptr, fmt, ##__VA_ARGS__) -#define SSLErrorVC(vc, fmt, ...) SSLDiagnostic(MakeSourceLocation(), false, (vc), fmt, ##__VA_ARGS__) -// Log a SSL diagnostic using the "ssl" diagnostic tag. -#define SSLDebug(fmt, ...) SSLDiagnostic(MakeSourceLocation(), true, nullptr, fmt, ##__VA_ARGS__) -#define SSLVCDebug(vc, fmt, ...) SSLDiagnostic(MakeSourceLocation(), true, (vc), fmt, ##__VA_ARGS__) - -#define SSL_CLR_ERR_INCR_DYN_STAT(vc, x, fmt, ...) \ - do { \ - SSLVCDebug((vc), fmt, ##__VA_ARGS__); \ - RecIncrRawStat(ssl_rsb, nullptr, (int)x, 1); \ - } while (0) - -void SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) TS_PRINTFLIKE(4, 5); - -// Return a static string name for a SSL_ERROR constant. -const char *SSLErrorName(int ssl_error); - -// Log a SSL network buffer. -void SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, const char *message); - // Load the SSL certificate configuration. bool SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *lookup); diff --git a/iocore/net/SSLClientUtils.cc b/iocore/net/SSLClientUtils.cc index 89842f543fd..dd67e88b051 100644 --- a/iocore/net/SSLClientUtils.cc +++ b/iocore/net/SSLClientUtils.cc @@ -23,9 +23,11 @@ #include "records/I_RecHttp.h" #include "tscore/ink_platform.h" #include "tscore/X509HostnameValidator.h" + #include "P_Net.h" #include "P_SSLClientUtils.h" #include "YamlSNIConfig.h" +#include "SSLDiags.h" #include #include diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 2d67244e48c..1fd19557979 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -29,19 +29,23 @@ SSL Configurations ****************************************************************************/ -#include "tscore/ink_platform.h" -#include "tscore/I_Layout.h" +#include "P_SSLConfig.h" #include #include + +#include "tscore/ink_platform.h" +#include "tscore/I_Layout.h" +#include "records/I_RecHttp.h" + +#include "HttpConfig.h" + #include "P_Net.h" -#include "P_SSLConfig.h" -#include "YamlSNIConfig.h" #include "P_SSLUtils.h" #include "P_SSLCertLookup.h" #include "SSLSessionCache.h" -#include -#include +#include "SSLDiags.h" +#include "YamlSNIConfig.h" int SSLConfig::configid = 0; int SSLCertificateConfig::configid = 0; diff --git a/iocore/net/SSLDiags.cc b/iocore/net/SSLDiags.cc new file mode 100644 index 00000000000..549a4d31e38 --- /dev/null +++ b/iocore/net/SSLDiags.cc @@ -0,0 +1,224 @@ +/** @file + + Diags for TLS + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "SSLDiags.h" + +#include + +#include "P_Net.h" +#include "SSLStats.h" + +// return true if we have a stat for the error +static bool +increment_ssl_client_error(unsigned long err) +{ + // we only look for LIB_SSL errors atm + if (ERR_LIB_SSL != ERR_GET_LIB(err)) { + SSL_INCREMENT_DYN_STAT(ssl_user_agent_other_errors_stat); + return false; + } + + // error was in LIB_SSL, now just switch on REASON + // (we ignore FUNCTION with the prejudice that we don't care what function + // the error came from, hope that's ok?) + switch (ERR_GET_REASON(err)) { +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED + case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_expired_cert_stat); + break; +#endif +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED + case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_revoked_cert_stat); + break; +#endif +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN + case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_unknown_cert_stat); + break; +#endif + case SSL_R_CERTIFICATE_VERIFY_FAILED: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_cert_verify_failed_stat); + break; +#ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE + case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_bad_cert_stat); + break; +#endif +#ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED + case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_decryption_failed_stat); + break; +#endif + case SSL_R_WRONG_VERSION_NUMBER: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_wrong_version_stat); + break; +#ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA + case SSL_R_TLSV1_ALERT_UNKNOWN_CA: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_unknown_ca_stat); + break; +#endif + default: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_other_errors_stat); + return false; + } + + return true; +} + +// return true if we have a stat for the error + +static bool +increment_ssl_server_error(unsigned long err) +{ + // we only look for LIB_SSL errors atm + if (ERR_LIB_SSL != ERR_GET_LIB(err)) { + SSL_INCREMENT_DYN_STAT(ssl_origin_server_other_errors_stat); + return false; + } + + // error was in LIB_SSL, now just switch on REASON + // (we ignore FUNCTION with the prejudice that we don't care what function + // the error came from, hope that's ok?) + switch (ERR_GET_REASON(err)) { +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED + case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_expired_cert_stat); + break; +#endif +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED + case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_revoked_cert_stat); + break; +#endif +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN + case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_unknown_cert_stat); + break; +#endif + case SSL_R_CERTIFICATE_VERIFY_FAILED: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_cert_verify_failed_stat); + break; +#ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE + case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_bad_cert_stat); + break; +#endif +#ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED + case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_decryption_failed_stat); + break; +#endif + case SSL_R_WRONG_VERSION_NUMBER: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_wrong_version_stat); + break; +#ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA + case SSL_R_TLSV1_ALERT_UNKNOWN_CA: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_unknown_ca_stat); + break; +#endif + default: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_other_errors_stat); + return false; + } + + return true; +} + +void +SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) +{ + unsigned long l; + char buf[256]; + const char *file, *data; + int line, flags; + unsigned long es; + va_list ap; + ip_text_buffer ip_buf = {'\0'}; + + if (vc) { + ats_ip_ntop(vc->get_remote_addr(), ip_buf, sizeof(ip_buf)); + } + + es = (unsigned long)pthread_self(); + while ((l = ERR_get_error_line_data(&file, &line, &data, &flags)) != 0) { + if (debug) { + if (unlikely(diags->on())) { + diags->log("ssl-diag", DL_Debug, &loc, "SSL::%lu:%s:%s:%d%s%s%s%s", es, ERR_error_string(l, buf), file, line, + (flags & ERR_TXT_STRING) ? ":" : "", (flags & ERR_TXT_STRING) ? data : "", vc ? ": peer address is " : "", + ip_buf); + } + } else { + diags->error(DL_Error, &loc, "SSL::%lu:%s:%s:%d%s%s%s%s", es, ERR_error_string(l, buf), file, line, + (flags & ERR_TXT_STRING) ? ":" : "", (flags & ERR_TXT_STRING) ? data : "", vc ? ": peer address is " : "", + ip_buf); + } + + // Tally desired stats (only client/server connection stats, not init + // issues where vc is nullptr) + if (vc) { + // get_context() == NET_VCONNECTION_OUT if ats is client (we update server stats) + if (vc->get_context() == NET_VCONNECTION_OUT) { + increment_ssl_server_error(l); // update server error stats + } else { + increment_ssl_client_error(l); // update client error stat + } + } + } + + va_start(ap, fmt); + if (debug) { + diags->log_va("ssl-diag", DL_Debug, &loc, fmt, ap); + } else { + diags->error_va(DL_Error, &loc, fmt, ap); + } + va_end(ap); +} + +const char * +SSLErrorName(int ssl_error) +{ + static const char *names[] = { + "SSL_ERROR_NONE", "SSL_ERROR_SSL", "SSL_ERROR_WANT_READ", "SSL_ERROR_WANT_WRITE", "SSL_ERROR_WANT_X509_LOOKUP", + "SSL_ERROR_SYSCALL", "SSL_ERROR_ZERO_RETURN", "SSL_ERROR_WANT_CONNECT", "SSL_ERROR_WANT_ACCEPT"}; + + if (ssl_error < 0 || ssl_error >= (int)countof(names)) { + return "unknown SSL error"; + } + + return names[ssl_error]; +} + +void +SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, const char *message) +{ + if (is_debug_tag_set(tag)) { + if (message != nullptr) { + fprintf(stdout, "%s\n", message); + } + for (unsigned ii = 0; ii < buflen; ii++) { + putc(buffer[ii], stdout); + } + putc('\n', stdout); + } +} diff --git a/iocore/net/SSLDiags.h b/iocore/net/SSLDiags.h new file mode 100644 index 00000000000..52ccd08980f --- /dev/null +++ b/iocore/net/SSLDiags.h @@ -0,0 +1,43 @@ +/** @file + + Diags for TLS + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "tscore/Diags.h" + +class SSLNetVConnection; + +// Log an SSL error. +#define SSLError(fmt, ...) SSLDiagnostic(MakeSourceLocation(), false, nullptr, fmt, ##__VA_ARGS__) +#define SSLErrorVC(vc, fmt, ...) SSLDiagnostic(MakeSourceLocation(), false, (vc), fmt, ##__VA_ARGS__) +// Log a SSL diagnostic using the "ssl" diagnostic tag. +#define SSLDebug(fmt, ...) SSLDiagnostic(MakeSourceLocation(), true, nullptr, fmt, ##__VA_ARGS__) +#define SSLVCDebug(vc, fmt, ...) SSLDiagnostic(MakeSourceLocation(), true, (vc), fmt, ##__VA_ARGS__) + +void SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) TS_PRINTFLIKE(4, 5); + +// Return a static string name for a SSL_ERROR constant. +const char *SSLErrorName(int ssl_error); + +// Log a SSL network buffer. +void SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, const char *message); diff --git a/iocore/net/SSLNetProcessor.cc b/iocore/net/SSLNetProcessor.cc index cff8992429d..90c7a4e622c 100644 --- a/iocore/net/SSLNetProcessor.cc +++ b/iocore/net/SSLNetProcessor.cc @@ -27,6 +27,7 @@ #include "P_SSLUtils.h" #include "P_OCSPStapling.h" #include "P_SSLSNI.h" +#include "SSLStats.h" // // Global Data diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 5d08d3ddb4c..2c555715759 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -20,22 +20,26 @@ See the License for the specific language governing permissions and limitations under the License. */ + #include "tscore/ink_config.h" #include "tscore/EventNotify.h" #include "tscore/I_Layout.h" #include "records/I_RecHttp.h" + +#include "InkAPIInternal.h" // Added to include the ssl_hook definitions +#include "Log.h" +#include "HttpTunnel.h" +#include "ProxyProtocol.h" +#include "HttpConfig.h" + #include "P_Net.h" #include "P_SSLNextProtocolSet.h" #include "P_SSLUtils.h" -#include "InkAPIInternal.h" // Added to include the ssl_hook definitions #include "P_SSLConfig.h" -#include "BIO_fastopen.h" -#include "Log.h" #include "P_SSLClientUtils.h" #include "P_SSLSNI.h" -#include "HttpTunnel.h" -#include "ProxyProtocol.h" -#include +#include "BIO_fastopen.h" +#include "SSLStats.h" #include #include diff --git a/iocore/net/SSLSessionCache.cc b/iocore/net/SSLSessionCache.cc index bbe52a9f241..1238b4baeb0 100644 --- a/iocore/net/SSLSessionCache.cc +++ b/iocore/net/SSLSessionCache.cc @@ -21,6 +21,8 @@ #include "P_SSLConfig.h" #include "SSLSessionCache.h" +#include "SSLStats.h" + #include #define SSLSESSIONCACHE_STRINGIFY0(x) #x diff --git a/iocore/net/SSLStats.cc b/iocore/net/SSLStats.cc new file mode 100644 index 00000000000..c748b94820f --- /dev/null +++ b/iocore/net/SSLStats.cc @@ -0,0 +1,239 @@ +/** @file + + Stats of TLS + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "SSLStats.h" + +#include + +#include "P_SSLConfig.h" + +RecRawStatBlock *ssl_rsb = nullptr; +std::unordered_map cipher_map; + +static int +SSLRecRawStatSyncCount(const char *name, RecDataT data_type, RecData *data, RecRawStatBlock *rsb, int id) +{ + // Grab all the stats we want from OpenSSL and set the stats. This function only needs to be called by one of the + // involved stats, all others *must* call RecRawStatSyncSum. + SSLCertificateConfig::scoped_config certLookup; + + int64_t sessions = 0; + int64_t hits = 0; + int64_t misses = 0; + int64_t timeouts = 0; + + if (certLookup) { + const unsigned ctxCount = certLookup->count(); + for (size_t i = 0; i < ctxCount; i++) { + SSLCertContext *cc = certLookup->get(i); + if (cc && cc->ctx) { + sessions += SSL_CTX_sess_accept_good(cc->ctx); + hits += SSL_CTX_sess_hits(cc->ctx); + misses += SSL_CTX_sess_misses(cc->ctx); + timeouts += SSL_CTX_sess_timeouts(cc->ctx); + } + } + } + + SSL_SET_COUNT_DYN_STAT(ssl_user_agent_sessions_stat, sessions); + SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_hit_stat, hits); + SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_miss_stat, misses); + SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_timeout_stat, timeouts); + + return RecRawStatSyncCount(name, data_type, data, rsb, id); +} + +void +SSLInitializeStatistics() +{ + SSL_CTX *ctx; + SSL *ssl; + STACK_OF(SSL_CIPHER) * ciphers; + + // Allocate SSL statistics block. + ssl_rsb = RecAllocateRawStatBlock((int)Ssl_Stat_Count); + ink_assert(ssl_rsb != nullptr); + + // SSL client errors. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_other_errors", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_other_errors_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_expired_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_expired_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_revoked_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_revoked_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_unknown_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_unknown_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_cert_verify_failed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_cert_verify_failed_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_bad_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_bad_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_decryption_failed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_decryption_failed_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_wrong_version", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_wrong_version_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_unknown_ca", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_unknown_ca_stat, RecRawStatSyncSum); + + // Polled SSL context statistics. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_sessions", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)ssl_user_agent_sessions_stat, + SSLRecRawStatSyncCount); //<- only use this fn once + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_hit", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)ssl_user_agent_session_hit_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_miss", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)ssl_user_agent_session_miss_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_timeout", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)ssl_user_agent_session_timeout_stat, RecRawStatSyncCount); + + // SSL server errors. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_other_errors", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_other_errors_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_expired_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_expired_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_revoked_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_revoked_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_unknown_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_unknown_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_cert_verify_failed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_cert_verify_failed_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_bad_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_bad_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_decryption_failed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_decryption_failed_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_wrong_version", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_wrong_version_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_unknown_ca", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_unknown_ca_stat, RecRawStatSyncSum); + + // SSL handshake time + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_handshake_time", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_handshake_time_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_in", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_success_handshake_count_in_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_out", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_success_handshake_count_out_stat, RecRawStatSyncCount); + + // TLS tickets + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_created", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_created_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_verified", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_verified_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_not_found", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_not_found_stat, RecRawStatSyncCount); + // TODO: ticket renewal is not used right now. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_renewed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_renewed_stat, RecRawStatSyncCount); + // The number of session tickets verified with an "old" key. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_verified_old_key", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_verified_old_key_stat, RecRawStatSyncCount); + // The number of ticket keys renewed. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_ticket_keys_renewed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_ticket_keys_renewed_stat, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_hit", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_hit, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_new_session", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_new_session, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_miss", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_miss, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_eviction", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_eviction, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_lock_contention", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_lock_contention, RecRawStatSyncCount); + + /* Track dynamic record size */ + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.default_record_size_count", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_dyn_def_tls_record_count, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.max_record_size_count", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_dyn_max_tls_record_count, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.redo_record_size_count", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_dyn_redo_tls_record_count, RecRawStatSyncCount); + + /* error stats */ + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_write", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_want_write, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_read", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_want_read, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_x509_lookup", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_want_x509_lookup, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_syscall", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_syscall, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_read_eos", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_read_eos, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_zero_return", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_zero_return, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_ssl", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_error_ssl, + RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_sni_name_set_failure", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_sni_name_set_failure, RecRawStatSyncCount); + + /* ocsp stapling stats */ + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_revoked_cert_stat", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_ocsp_revoked_cert_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_unknown_cert_stat", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_ocsp_unknown_cert_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refreshed_cert", RECD_INT, RECP_PERSISTENT, + (int)ssl_ocsp_refreshed_cert_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refresh_cert_failure", RECD_INT, RECP_PERSISTENT, + (int)ssl_ocsp_refresh_cert_failure_stat, RecRawStatSyncCount); + + // Get and register the SSL cipher stats. Note that we are using the default SSL context to obtain + // the cipher list. This means that the set of ciphers is fixed by the build configuration and not + // filtered by proxy.config.ssl.server.cipher_suite. This keeps the set of cipher suites stable across + // configuration reloads and works for the case where we honor the client cipher preference. + + ctx = SSLDefaultServerContext(); + ssl = SSL_new(ctx); + ciphers = SSL_get_ciphers(ssl); + + // BoringSSL has sk_SSL_CIPHER_num() return a size_t (well, sk_num() is) + for (int index = 0; index < static_cast(sk_SSL_CIPHER_num(ciphers)); index++) { + SSL_CIPHER *cipher = const_cast(sk_SSL_CIPHER_value(ciphers, index)); + const char *cipherName = SSL_CIPHER_get_name(cipher); + std::string statName = "proxy.process.ssl.cipher.user_agent." + std::string(cipherName); + + // If room in allocated space ... + if ((ssl_cipher_stats_start + index) > ssl_cipher_stats_end) { + // Too many ciphers, increase ssl_cipher_stats_end. + SSLError("too many ciphers to register metric '%s', increase SSL_Stats::ssl_cipher_stats_end", statName.c_str()); + continue; + } + + // If not already registered ... + if (cipherName && cipher_map.find(cipherName) == cipher_map.end()) { + cipher_map.emplace(cipherName, (intptr_t)(ssl_cipher_stats_start + index)); + // Register as non-persistent since the order/index is dependent upon configuration. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, statName.c_str(), RECD_INT, RECP_NON_PERSISTENT, + (int)ssl_cipher_stats_start + index, RecRawStatSyncSum); + SSL_CLEAR_DYN_STAT((int)ssl_cipher_stats_start + index); + Debug("ssl", "registering SSL cipher metric '%s'", statName.c_str()); + } + } + + SSL_free(ssl); + SSLReleaseContext(ctx); +} diff --git a/iocore/net/SSLStats.h b/iocore/net/SSLStats.h new file mode 100644 index 00000000000..ff38df0d03f --- /dev/null +++ b/iocore/net/SSLStats.h @@ -0,0 +1,115 @@ +/** @file + + Stats of TLS + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include + +#include "records/I_RecProcess.h" +#include "SSLDiags.h" + +/* Stats should only be accessed using these macros */ +#define SSL_INCREMENT_DYN_STAT(x) RecIncrRawStat(ssl_rsb, nullptr, (int)x, 1) +#define SSL_DECREMENT_DYN_STAT(x) RecIncrRawStat(ssl_rsb, nullptr, (int)x, -1) +#define SSL_SET_COUNT_DYN_STAT(x, count) RecSetRawStatCount(ssl_rsb, x, count) +#define SSL_INCREMENT_DYN_STAT_EX(x, y) RecIncrRawStat(ssl_rsb, nullptr, (int)x, y) +#define SSL_CLEAR_DYN_STAT(x) \ + do { \ + RecSetRawStatSum(ssl_rsb, (x), 0); \ + RecSetRawStatCount(ssl_rsb, (x), 0); \ + } while (0) +#define SSL_CLR_ERR_INCR_DYN_STAT(vc, x, fmt, ...) \ + do { \ + SSLVCDebug((vc), fmt, ##__VA_ARGS__); \ + RecIncrRawStat(ssl_rsb, nullptr, (int)x, 1); \ + } while (0) + +enum SSL_Stats { + ssl_origin_server_expired_cert_stat, + ssl_user_agent_expired_cert_stat, + ssl_origin_server_revoked_cert_stat, + ssl_user_agent_revoked_cert_stat, + ssl_origin_server_unknown_cert_stat, + ssl_user_agent_unknown_cert_stat, + ssl_origin_server_cert_verify_failed_stat, + ssl_user_agent_cert_verify_failed_stat, + ssl_origin_server_bad_cert_stat, + ssl_user_agent_bad_cert_stat, + ssl_origin_server_decryption_failed_stat, + ssl_user_agent_decryption_failed_stat, + ssl_origin_server_wrong_version_stat, + ssl_user_agent_wrong_version_stat, + ssl_origin_server_other_errors_stat, + ssl_user_agent_other_errors_stat, + ssl_origin_server_unknown_ca_stat, + ssl_user_agent_unknown_ca_stat, + ssl_user_agent_sessions_stat, + ssl_user_agent_session_hit_stat, + ssl_user_agent_session_miss_stat, + ssl_user_agent_session_timeout_stat, + ssl_total_handshake_time_stat, + ssl_total_success_handshake_count_in_stat, + ssl_total_tickets_created_stat, + ssl_total_tickets_verified_stat, + ssl_total_tickets_verified_old_key_stat, // verified with old key. + ssl_total_ticket_keys_renewed_stat, // number of keys renewed. + ssl_total_tickets_not_found_stat, + ssl_total_tickets_renewed_stat, + ssl_total_dyn_def_tls_record_count, + ssl_total_dyn_max_tls_record_count, + ssl_total_dyn_redo_tls_record_count, + ssl_session_cache_hit, + ssl_session_cache_miss, + ssl_session_cache_eviction, + ssl_session_cache_lock_contention, + ssl_session_cache_new_session, + + /* error stats */ + ssl_error_want_write, + ssl_error_want_read, + ssl_error_want_client_hello_cb, + ssl_error_want_x509_lookup, + ssl_error_syscall, + ssl_error_read_eos, + ssl_error_zero_return, + ssl_error_ssl, + ssl_sni_name_set_failure, + ssl_total_success_handshake_count_out_stat, + + /* ocsp stapling stats */ + ssl_ocsp_revoked_cert_stat, + ssl_ocsp_unknown_cert_stat, + ssl_ocsp_refreshed_cert_stat, + ssl_ocsp_refresh_cert_failure_stat, + + ssl_cipher_stats_start = 100, + ssl_cipher_stats_end = 300, + + Ssl_Stat_Count +}; + +extern RecRawStatBlock *ssl_rsb; +extern std::unordered_map cipher_map; + +// Initialize SSL statistics. +void SSLInitializeStatistics(); diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 111ec1ba769..ad20188f9c1 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -19,18 +19,25 @@ limitations under the License. */ +#include "P_SSLUtils.h" + #include "tscore/ink_platform.h" #include "tscore/SimpleTokenizer.h" -#include "records/I_RecHttp.h" #include "tscore/I_Layout.h" -#include "P_Net.h" #include "tscore/ink_cap.h" #include "tscore/ink_mutex.h" +#include "records/I_RecHttp.h" + +#include "P_Net.h" +#include "InkAPIInternal.h" + #include "P_OCSPStapling.h" +#include "P_SSLSNI.h" +#include "P_SSLConfig.h" #include "SSLSessionCache.h" -#include "InkAPIInternal.h" #include "SSLDynlock.h" -#include "P_SSLSNI.h" +#include "SSLDiags.h" +#include "SSLStats.h" #include #include @@ -138,9 +145,6 @@ static int ssl_vc_index = -1; static ink_mutex *mutex_buf = nullptr; static bool open_ssl_initialized = false; -RecRawStatBlock *ssl_rsb = nullptr; -std::unordered_map cipher_map; - /* Using pthread thread ID and mutex functions directly, instead of * ATS this_ethread / ProxyMutex, so that other linked libraries * may use pthreads and openssl without confusing us here. (TS-2271). @@ -904,38 +908,6 @@ ssl_private_key_validate_exec(const char *cmdLine) return bReturn; } -static int -SSLRecRawStatSyncCount(const char *name, RecDataT data_type, RecData *data, RecRawStatBlock *rsb, int id) -{ - // Grab all the stats we want from OpenSSL and set the stats. This function only needs to be called by one of the - // involved stats, all others *must* call RecRawStatSyncSum. - SSLCertificateConfig::scoped_config certLookup; - - int64_t sessions = 0; - int64_t hits = 0; - int64_t misses = 0; - int64_t timeouts = 0; - - if (certLookup) { - const unsigned ctxCount = certLookup->count(); - for (size_t i = 0; i < ctxCount; i++) { - SSLCertContext *cc = certLookup->get(i); - if (cc && cc->ctx) { - sessions += SSL_CTX_sess_accept_good(cc->ctx); - hits += SSL_CTX_sess_hits(cc->ctx); - misses += SSL_CTX_sess_misses(cc->ctx); - timeouts += SSL_CTX_sess_timeouts(cc->ctx); - } - } - } - - SSL_SET_COUNT_DYN_STAT(ssl_user_agent_sessions_stat, sessions); - SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_hit_stat, hits); - SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_miss_stat, misses); - SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_timeout_stat, timeouts); - return RecRawStatSyncCount(name, data_type, data, rsb, id); -} - #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) #define ssl_malloc(size, file, line) ssl_malloc(size) #define ssl_realloc(ptr, size, file, line) ssl_realloc(ptr, size) @@ -1058,376 +1030,6 @@ SSLInitializeLibrary() open_ssl_initialized = true; } -void -SSLInitializeStatistics() -{ - SSL_CTX *ctx; - SSL *ssl; - STACK_OF(SSL_CIPHER) * ciphers; - - // Allocate SSL statistics block. - ssl_rsb = RecAllocateRawStatBlock((int)Ssl_Stat_Count); - ink_assert(ssl_rsb != nullptr); - - // SSL client errors. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_other_errors", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_other_errors_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_expired_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_expired_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_revoked_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_revoked_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_unknown_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_unknown_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_cert_verify_failed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_cert_verify_failed_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_bad_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_bad_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_decryption_failed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_decryption_failed_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_wrong_version", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_wrong_version_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_unknown_ca", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_unknown_ca_stat, RecRawStatSyncSum); - - // Polled SSL context statistics. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_sessions", RECD_COUNTER, RECP_NON_PERSISTENT, - (int)ssl_user_agent_sessions_stat, - SSLRecRawStatSyncCount); //<- only use this fn once - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_hit", RECD_COUNTER, RECP_NON_PERSISTENT, - (int)ssl_user_agent_session_hit_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_miss", RECD_COUNTER, RECP_NON_PERSISTENT, - (int)ssl_user_agent_session_miss_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_timeout", RECD_COUNTER, RECP_NON_PERSISTENT, - (int)ssl_user_agent_session_timeout_stat, RecRawStatSyncCount); - - // SSL server errors. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_other_errors", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_other_errors_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_expired_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_expired_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_revoked_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_revoked_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_unknown_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_unknown_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_cert_verify_failed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_cert_verify_failed_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_bad_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_bad_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_decryption_failed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_decryption_failed_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_wrong_version", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_wrong_version_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_unknown_ca", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_unknown_ca_stat, RecRawStatSyncSum); - - // SSL handshake time - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_handshake_time", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_handshake_time_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_in", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_success_handshake_count_in_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_out", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_success_handshake_count_out_stat, RecRawStatSyncCount); - - // TLS tickets - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_created", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_created_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_verified", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_verified_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_not_found", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_not_found_stat, RecRawStatSyncCount); - // TODO: ticket renewal is not used right now. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_renewed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_renewed_stat, RecRawStatSyncCount); - // The number of session tickets verified with an "old" key. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_verified_old_key", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_verified_old_key_stat, RecRawStatSyncCount); - // The number of ticket keys renewed. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_ticket_keys_renewed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_ticket_keys_renewed_stat, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_hit", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_hit, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_new_session", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_new_session, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_miss", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_miss, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_eviction", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_eviction, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_lock_contention", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_lock_contention, RecRawStatSyncCount); - - /* Track dynamic record size */ - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.default_record_size_count", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_dyn_def_tls_record_count, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.max_record_size_count", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_dyn_max_tls_record_count, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.redo_record_size_count", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_dyn_redo_tls_record_count, RecRawStatSyncCount); - - /* error stats */ - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_write", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_want_write, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_read", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_want_read, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_x509_lookup", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_want_x509_lookup, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_syscall", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_syscall, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_read_eos", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_read_eos, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_zero_return", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_zero_return, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_ssl", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_error_ssl, - RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_sni_name_set_failure", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_sni_name_set_failure, RecRawStatSyncCount); - - /* ocsp stapling stats */ - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_revoked_cert_stat", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_ocsp_revoked_cert_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_unknown_cert_stat", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_ocsp_unknown_cert_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refreshed_cert", RECD_INT, RECP_PERSISTENT, - (int)ssl_ocsp_refreshed_cert_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refresh_cert_failure", RECD_INT, RECP_PERSISTENT, - (int)ssl_ocsp_refresh_cert_failure_stat, RecRawStatSyncCount); - - // Get and register the SSL cipher stats. Note that we are using the default SSL context to obtain - // the cipher list. This means that the set of ciphers is fixed by the build configuration and not - // filtered by proxy.config.ssl.server.cipher_suite. This keeps the set of cipher suites stable across - // configuration reloads and works for the case where we honor the client cipher preference. - - ctx = SSLDefaultServerContext(); - ssl = SSL_new(ctx); - ciphers = SSL_get_ciphers(ssl); - - // BoringSSL has sk_SSL_CIPHER_num() return a size_t (well, sk_num() is) - for (int index = 0; index < static_cast(sk_SSL_CIPHER_num(ciphers)); index++) { - SSL_CIPHER *cipher = const_cast(sk_SSL_CIPHER_value(ciphers, index)); - const char *cipherName = SSL_CIPHER_get_name(cipher); - std::string statName = "proxy.process.ssl.cipher.user_agent." + std::string(cipherName); - - // If room in allocated space ... - if ((ssl_cipher_stats_start + index) > ssl_cipher_stats_end) { - // Too many ciphers, increase ssl_cipher_stats_end. - SSLError("too many ciphers to register metric '%s', increase SSL_Stats::ssl_cipher_stats_end", statName.c_str()); - continue; - } - - // If not already registered ... - if (cipherName && cipher_map.find(cipherName) == cipher_map.end()) { - cipher_map.emplace(cipherName, (intptr_t)(ssl_cipher_stats_start + index)); - // Register as non-persistent since the order/index is dependent upon configuration. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, statName.c_str(), RECD_INT, RECP_NON_PERSISTENT, - (int)ssl_cipher_stats_start + index, RecRawStatSyncSum); - SSL_CLEAR_DYN_STAT((int)ssl_cipher_stats_start + index); - Debug("ssl", "registering SSL cipher metric '%s'", statName.c_str()); - } - } - - SSL_free(ssl); - SSLReleaseContext(ctx); -} - -// return true if we have a stat for the error -static bool -increment_ssl_client_error(unsigned long err) -{ - // we only look for LIB_SSL errors atm - if (ERR_LIB_SSL != ERR_GET_LIB(err)) { - SSL_INCREMENT_DYN_STAT(ssl_user_agent_other_errors_stat); - return false; - } - - // error was in LIB_SSL, now just switch on REASON - // (we ignore FUNCTION with the prejudice that we don't care what function - // the error came from, hope that's ok?) - switch (ERR_GET_REASON(err)) { -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED - case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_expired_cert_stat); - break; -#endif -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED - case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_revoked_cert_stat); - break; -#endif -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN - case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_unknown_cert_stat); - break; -#endif - case SSL_R_CERTIFICATE_VERIFY_FAILED: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_cert_verify_failed_stat); - break; -#ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE - case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_bad_cert_stat); - break; -#endif -#ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED - case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_decryption_failed_stat); - break; -#endif - case SSL_R_WRONG_VERSION_NUMBER: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_wrong_version_stat); - break; -#ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA - case SSL_R_TLSV1_ALERT_UNKNOWN_CA: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_unknown_ca_stat); - break; -#endif - default: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_other_errors_stat); - return false; - } - - return true; -} - -// return true if we have a stat for the error - -static bool -increment_ssl_server_error(unsigned long err) -{ - // we only look for LIB_SSL errors atm - if (ERR_LIB_SSL != ERR_GET_LIB(err)) { - SSL_INCREMENT_DYN_STAT(ssl_origin_server_other_errors_stat); - return false; - } - - // error was in LIB_SSL, now just switch on REASON - // (we ignore FUNCTION with the prejudice that we don't care what function - // the error came from, hope that's ok?) - switch (ERR_GET_REASON(err)) { -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED - case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_expired_cert_stat); - break; -#endif -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED - case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_revoked_cert_stat); - break; -#endif -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN - case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_unknown_cert_stat); - break; -#endif - case SSL_R_CERTIFICATE_VERIFY_FAILED: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_cert_verify_failed_stat); - break; -#ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE - case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_bad_cert_stat); - break; -#endif -#ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED - case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_decryption_failed_stat); - break; -#endif - case SSL_R_WRONG_VERSION_NUMBER: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_wrong_version_stat); - break; -#ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA - case SSL_R_TLSV1_ALERT_UNKNOWN_CA: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_unknown_ca_stat); - break; -#endif - default: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_other_errors_stat); - return false; - } - - return true; -} - -void -SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) -{ - unsigned long l; - char buf[256]; - const char *file, *data; - int line, flags; - unsigned long es; - va_list ap; - ip_text_buffer ip_buf = {'\0'}; - - if (vc) { - ats_ip_ntop(vc->get_remote_addr(), ip_buf, sizeof(ip_buf)); - } - - es = (unsigned long)pthread_self(); - while ((l = ERR_get_error_line_data(&file, &line, &data, &flags)) != 0) { - if (debug) { - if (unlikely(diags->on())) { - diags->log("ssl-diag", DL_Debug, &loc, "SSL::%lu:%s:%s:%d%s%s%s%s", es, ERR_error_string(l, buf), file, line, - (flags & ERR_TXT_STRING) ? ":" : "", (flags & ERR_TXT_STRING) ? data : "", vc ? ": peer address is " : "", - ip_buf); - } - } else { - diags->error(DL_Error, &loc, "SSL::%lu:%s:%s:%d%s%s%s%s", es, ERR_error_string(l, buf), file, line, - (flags & ERR_TXT_STRING) ? ":" : "", (flags & ERR_TXT_STRING) ? data : "", vc ? ": peer address is " : "", - ip_buf); - } - - // Tally desired stats (only client/server connection stats, not init - // issues where vc is nullptr) - if (vc) { - // get_context() == NET_VCONNECTION_OUT if ats is client (we update server stats) - if (vc->get_context() == NET_VCONNECTION_OUT) { - increment_ssl_server_error(l); // update server error stats - } else { - increment_ssl_client_error(l); // update client error stat - } - } - } - - va_start(ap, fmt); - if (debug) { - diags->log_va("ssl-diag", DL_Debug, &loc, fmt, ap); - } else { - diags->error_va(DL_Error, &loc, fmt, ap); - } - va_end(ap); -} - -const char * -SSLErrorName(int ssl_error) -{ - static const char *names[] = { - "SSL_ERROR_NONE", "SSL_ERROR_SSL", "SSL_ERROR_WANT_READ", "SSL_ERROR_WANT_WRITE", "SSL_ERROR_WANT_X509_LOOKUP", - "SSL_ERROR_SYSCALL", "SSL_ERROR_ZERO_RETURN", "SSL_ERROR_WANT_CONNECT", "SSL_ERROR_WANT_ACCEPT"}; - - if (ssl_error < 0 || ssl_error >= (int)countof(names)) { - return "unknown SSL error"; - } - - return names[ssl_error]; -} - -void -SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, const char *message) -{ - if (is_debug_tag_set(tag)) { - if (message != nullptr) { - fprintf(stdout, "%s\n", message); - } - for (unsigned ii = 0; ii < buflen; ii++) { - putc(buffer[ii], stdout); - } - putc('\n', stdout); - } -} - SSL_CTX * SSLDefaultServerContext() { From b52c293ef8f2acb9471dbebd46fce7e439695df3 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 18 Feb 2019 23:23:56 -0600 Subject: [PATCH 297/526] IpMap: Add move constructor. --- include/tscore/IpMap.h | 108 +++++++++++++++------------- src/tscore/IpMap.cc | 90 +++++++++++++++-------- src/tscore/unit_tests/test_IpMap.cc | 26 +++++-- 3 files changed, 139 insertions(+), 85 deletions(-) diff --git a/include/tscore/IpMap.h b/include/tscore/IpMap.h index 34ec5dc3175..17ecc956673 100644 --- a/include/tscore/IpMap.h +++ b/include/tscore/IpMap.h @@ -23,6 +23,9 @@ #pragma once +#include +#include + #include "tscore/ink_platform.h" #include "tscore/ink_defs.h" #include "tscore/RbTree.h" @@ -42,8 +45,8 @@ namespace detail typename A = T const & ///< Argument type. > struct Interval { - typedef T Metric; ///< Metric (storage) type. - typedef A ArgType; ///< Type used to pass instances of @c Metric. + using Metric = T; ///< Metric (storage) type. + using ArgType = A; ///< Type used to pass instances of @c Metric. Interval() {} ///< Default constructor. /// Construct with values. @@ -95,7 +98,7 @@ namespace detail class IpMap { public: - typedef IpMap self; ///< Self reference type. + using self_type = IpMap; ///< Self reference type. class iterator; // forward declare. @@ -107,7 +110,7 @@ class IpMap friend class IpMap; public: - typedef Node self; ///< Self reference type. + using self_type = Node; ///< Self reference type. /// Default constructor. Node() : _data(nullptr) {} /// Construct with @a data. @@ -119,7 +122,7 @@ class IpMap return _data; } /// Set client data. - virtual self & + virtual self_type & setData(void *data ///< Client data pointer to store. ) { @@ -146,30 +149,30 @@ class IpMap friend class IpMap; public: - typedef iterator self; ///< Self reference type. - typedef Node value_type; ///< Referenced type for iterator. - typedef int difference_type; ///< Distance type. - typedef Node *pointer; ///< Pointer to referent. - typedef Node &reference; ///< Reference to referent. - typedef std::bidirectional_iterator_tag iterator_category; + using self_type = iterator; ///< Self reference type. + using value_type = Node; ///< Referenced type for iterator. + using difference_type = int; ///< Distance type. + using pointer = Node *; ///< Pointer to referent. + using reference = Node &; ///< Reference to referent. + using iterator_category = std::bidirectional_iterator_tag; /// Default constructor. - iterator() : _tree(nullptr), _node(nullptr) {} + iterator() = default; reference operator*() const; //!< value operator pointer operator->() const; //!< dereference operator - self &operator++(); //!< next node (prefix) - self operator++(int); //!< next node (postfix) - self &operator--(); ///< previous node (prefix) - self operator--(int); ///< next node (postfix) + self_type &operator++(); //!< next node (prefix) + self_type operator++(int); //!< next node (postfix) + self_type &operator--(); ///< previous node (prefix) + self_type operator--(int); ///< next node (postfix) /** Equality. @return @c true if the iterators refer to the same node. */ - bool operator==(self const &that) const; + bool operator==(self_type const &that) const; /** Inequality. @return @c true if the iterators refer to different nodes. */ bool - operator!=(self const &that) const + operator!=(self_type const &that) const { return !(*this == that); } @@ -177,20 +180,25 @@ class IpMap private: /// Construct a valid iterator. iterator(IpMap const *tree, Node *node) : _tree(tree), _node(node) {} - IpMap const *_tree; ///< Container. - Node *_node; //!< Current node. + IpMap const *_tree = nullptr; ///< Container. + Node *_node = nullptr; //!< Current node. }; - IpMap(); ///< Default constructor. + IpMap(); ///< Default constructor. + IpMap(self_type const &that) = delete; + IpMap(self_type &&that) noexcept; ~IpMap(); ///< Destructor. + self_type &operator=(self_type const &that) = delete; + self_type &operator =(self_type &&that); + /** Mark a range. All addresses in the range [ @a min , @a max ] are marked with @a data. @return This object. */ - self &mark(sockaddr const *min, ///< Minimum value in range. - sockaddr const *max, ///< Maximum value in range. - void *data = nullptr ///< Client data payload. + self_type &mark(sockaddr const *min, ///< Minimum value in range. + sockaddr const *max, ///< Maximum value in range. + void *data = nullptr ///< Client data payload. ); /** Mark a range. @@ -198,9 +206,9 @@ class IpMap @note Convenience overload for IPv4 addresses. @return This object. */ - self &mark(in_addr_t min, ///< Minimum address (network order). - in_addr_t max, ///< Maximum address (network order). - void *data = nullptr ///< Client data. + self_type &mark(in_addr_t min, ///< Minimum address (network order). + in_addr_t max, ///< Maximum address (network order). + void *data = nullptr ///< Client data. ); /** Mark a range. @@ -208,9 +216,9 @@ class IpMap @note Convenience overload for IPv4 addresses. @return This object. */ - self &mark(IpAddr const &min, ///< Minimum address (network order). - IpAddr const &max, ///< Maximum address (network order). - void *data = nullptr ///< Client data. + self_type &mark(IpAddr const &min, ///< Minimum address (network order). + IpAddr const &max, ///< Maximum address (network order). + void *data = nullptr ///< Client data. ); /** Mark an IPv4 address @a addr with @a data. @@ -218,8 +226,8 @@ class IpMap @note Convenience overload for IPv4 addresses. @return This object. */ - self &mark(in_addr_t addr, ///< Address (network order). - void *data = nullptr ///< Client data. + self_type &mark(in_addr_t addr, ///< Address (network order). + void *data = nullptr ///< Client data. ); /** Mark a range. @@ -227,9 +235,9 @@ class IpMap @note Convenience overload. @return This object. */ - self &mark(IpEndpoint const *min, ///< Minimum address (network order). - IpEndpoint const *max, ///< Maximum address (network order). - void *data = nullptr ///< Client data. + self_type &mark(IpEndpoint const *min, ///< Minimum address (network order). + IpEndpoint const *max, ///< Maximum address (network order). + void *data = nullptr ///< Client data. ); /** Mark an address @a addr with @a data. @@ -237,8 +245,8 @@ class IpMap @note Convenience overload. @return This object. */ - self &mark(IpEndpoint const *addr, ///< Address (network order). - void *data = nullptr ///< Client data. + self_type &mark(IpEndpoint const *addr, ///< Address (network order). + void *data = nullptr ///< Client data. ); /** Unmark addresses. @@ -248,14 +256,14 @@ class IpMap @return This object. */ - self &unmark(sockaddr const *min, ///< Minimum value. - sockaddr const *max ///< Maximum value. + self_type &unmark(sockaddr const *min, ///< Minimum value. + sockaddr const *max ///< Maximum value. ); /// Unmark addresses (overload). - self &unmark(IpEndpoint const *min, IpEndpoint const *max); + self_type &unmark(IpEndpoint const *min, IpEndpoint const *max); /// Unmark overload. - self &unmark(in_addr_t min, ///< Minimum of range to unmark. - in_addr_t max ///< Maximum of range to unmark. + self_type &unmark(in_addr_t min, ///< Minimum of range to unmark. + in_addr_t max ///< Maximum of range to unmark. ); /** Fill addresses. @@ -269,13 +277,13 @@ class IpMap @return This object. */ - self &fill(sockaddr const *min, sockaddr const *max, void *data = nullptr); + self_type &fill(sockaddr const *min, sockaddr const *max, void *data = nullptr); /// Fill addresses (overload). - self &fill(IpEndpoint const *min, IpEndpoint const *max, void *data = nullptr); + self_type &fill(IpEndpoint const *min, IpEndpoint const *max, void *data = nullptr); /// Fill addresses (overload). - self &fill(IpAddr const &min, IpAddr const &max, void *data = nullptr); + self_type &fill(IpAddr const &min, IpAddr const &max, void *data = nullptr); /// Fill addresses (overload). - self &fill(in_addr_t min, in_addr_t max, void *data = nullptr); + self_type &fill(in_addr_t min, in_addr_t max, void *data = nullptr); /** Test for membership. @@ -328,7 +336,7 @@ class IpMap @note This is much faster than @c unmark. @return This object. */ - self &clear(); + self_type &clear(); /// Iterator for first element. iterator begin() const; @@ -354,8 +362,8 @@ class IpMap /// @return The IPv6 map. ts::detail::Ip6Map *force6(); - ts::detail::Ip4Map *_m4; ///< Map of IPv4 addresses. - ts::detail::Ip6Map *_m6; ///< Map of IPv6 addresses. + ts::detail::Ip4Map *_m4 = nullptr; ///< Map of IPv4 addresses. + ts::detail::Ip6Map *_m6 = nullptr; ///< Map of IPv6 addresses. }; inline IpMap & @@ -437,7 +445,7 @@ IpMap::iterator::operator++(int) inline IpMap::iterator IpMap::iterator::operator--(int) { - self tmp(*this); + self_type tmp(*this); --*this; return tmp; } diff --git a/src/tscore/IpMap.cc b/src/tscore/IpMap.cc index ed206572c3c..d2583f269f1 100644 --- a/src/tscore/IpMap.cc +++ b/src/tscore/IpMap.cc @@ -1,6 +1,3 @@ -#include "tscore/IpMap.h" -#include "tscore/ink_inet.h" - /** @file IP address map support. @@ -46,6 +43,9 @@ before we had IpAddr as a type. */ +#include "tscore/IpMap.h" +#include "tscore/ink_inet.h" + namespace ts { namespace detail @@ -139,8 +139,9 @@ namespace detail using ArgType = typename N::ArgType; ///< Import type. using Metric = typename N::Metric; ///< Import type.g482 - IpMapBase() : _root(nullptr) {} - ~IpMapBase() { this->clear(); } + IpMapBase() = default; + IpMapBase(self_type &&that) : _root(that._root), _list(std::move(that._list)) { that._root = nullptr; } + ~IpMapBase(); /** Mark a range. All addresses in the range [ @a min , @a max ] are marked with @a data. @return This object. @@ -266,7 +267,7 @@ namespace detail return static_cast(_list.tail()); } - N *_root; ///< Root node. + N *_root = nullptr; ///< Root node. /// In order list of nodes. /// For ugly compiler reasons, this is a list of base class pointers /// even though we really store @a N instances on it. @@ -283,7 +284,6 @@ namespace detail } }; using NodeList = ts::IntrusiveDList; - // typedef ts::IntrusiveDList NodeList; /// This keeps track of all allocated nodes in order. /// Iteration depends on this list being maintained. NodeList _list; @@ -460,17 +460,14 @@ namespace detail Metric max_plus = N::deref(max); N::inc(max_plus); - /* Some subtlety - for IPv6 we overload the compare operators to do - the right thing, but we can't overload pointer - comparisons. Therefore we carefully never compare pointers in - this logic. Only @a min and @a max can be pointers, everything - else is an instance or a reference. Since there's no good reason - to compare @a min and @a max this isn't particularly tricky, but - it's good to keep in mind. If we were somewhat more clever, we - would provide static less than and equal operators in the - template class @a N and convert all the comparisons to use only - those two via static function call. - */ + /* Some subtlety - for IPv6 we overload the compare operators to do the right thing, but we + * can't overload pointer comparisons. Therefore we carefully never compare pointers in this + * logic. Only @a min and @a max can be pointers, everything else is an instance or a reference. + * Since there's no good reason to compare @a min and @a max this isn't particularly tricky, but + * it's good to keep in mind. If we were somewhat more clever, we would provide static less than + * and equal operators in the template class @a N and convert all the comparisons to use only + * those two via static function call. + */ /* We have lots of special cases here primarily to minimize memory allocation by re-using an existing node as often as possible. @@ -748,8 +745,10 @@ namespace detail return *this; } + template IpMapBase::~IpMapBase() { this->clear(); } + //---------------------------------------------------------------------------- - typedef Interval Ip4Span; + using Ip4Span = Interval; /** Node for IPv4 map. We store the address in host order in the @a _min and @a _max @@ -911,7 +910,13 @@ namespace detail /// is to use a pointer, not a reference. using ArgType = const ts::detail::Interval::Metric *; - /// Construct from pointers. + /** Construct from the argument type. + * + * @param min Minimum value in the range. + * @param max Maximum value in the range (inclusvie). + * @param data Data to attach to the range. + */ + Ip6Node(ArgType min, ///< Minimum address (network order). ArgType max, ///< Maximum address (network order). void *data ///< Client data. @@ -919,30 +924,36 @@ namespace detail : Node(data), Ip6Span(*min, *max) { } - /// Construct with values. - Ip6Node(Metric const &min, ///< Minimum address (network order). - Metric const &max, ///< Maximum address (network order). - void *data ///< Client data. - ) - : Node(data), Ip6Span(min, max) - { - } + + /** Construct from the underlying @c Metric type @a min to @a max + * + * @param min Minimum value in the range. + * @param max Maximum value in the range (inclusvie). + * @param data Data to attach to the range. + */ + Ip6Node(Metric const &min, Metric const &max, void *data) : Node(data), Ip6Span(min, max) {} + /// @return The minimum value of the interval. sockaddr const * min() const override { return ats_ip_sa_cast(&_min); } + /// @return The maximum value of the interval. sockaddr const * max() const override { return ats_ip_sa_cast(&_max); } - /// Set the client data. + + /** Set the client @a data. + * + * @param data Client data. + * @return @a this + */ self_type & - setData(void *data ///< Client data. - ) override + setData(void *data) override { _data = data; return *this; @@ -1081,6 +1092,23 @@ namespace detail } // namespace detail } // namespace ts //---------------------------------------------------------------------------- +IpMap::IpMap(IpMap::self_type &&that) noexcept : _m4(that._m4), _m6(that._m6) +{ + that._m4 = nullptr; + that._m6 = nullptr; +} + +IpMap::self_type & +IpMap::operator=(IpMap::self_type &&that) +{ + if (&that != this) { + this->clear(); + std::swap(_m4, that._m4); + std::swap(_m6, that._m6); + } + return *this; +} + IpMap::~IpMap() { delete _m4; diff --git a/src/tscore/unit_tests/test_IpMap.cc b/src/tscore/unit_tests/test_IpMap.cc index f8a288663b4..20a8231c608 100644 --- a/src/tscore/unit_tests/test_IpMap.cc +++ b/src/tscore/unit_tests/test_IpMap.cc @@ -540,11 +540,11 @@ TEST_CASE("IpMap CloseIntersection", "[libts][ipmap]") void *const markB = reinterpret_cast(2); void *const markC = reinterpret_cast(3); void *const markD = reinterpret_cast(4); - // void *mark; // for retrieval IpEndpoint a_1_l, a_1_u, a_2_l, a_2_u, a_3_l, a_3_u, a_4_l, a_4_u, a_5_l, a_5_u, a_6_l, a_6_u, a_7_l, a_7_u; IpEndpoint b_1_l, b_1_u; IpEndpoint c_1_l, c_1_u, c_2_l, c_2_u, c_3_l, c_3_u; + IpEndpoint c_3_m; IpEndpoint d_1_l, d_1_u, d_2_l, d_2_u; IpEndpoint a_1_m; @@ -573,6 +573,7 @@ TEST_CASE("IpMap CloseIntersection", "[libts][ipmap]") ats_ip_pton("123.90.112.0", &c_2_l); ats_ip_pton("123.90.119.255", &c_2_u); ats_ip_pton("123.90.132.0", &c_3_l); + ats_ip_pton("123.90.134.157", &c_3_m); ats_ip_pton("123.90.135.255", &c_3_u); ats_ip_pton("123.82.196.0", &d_1_l); @@ -590,16 +591,33 @@ TEST_CASE("IpMap CloseIntersection", "[libts][ipmap]") CHECK_THAT(map, IsMarkedAt(a_1_m)); map.mark(b_1_l, b_1_u, markB); - CHECK_THAT(map, IsMarkedAt(a_1_m)); + CHECK_THAT(map, IsMarkedWith(a_1_m, markA)); map.mark(c_1_l, c_1_u, markC); map.mark(c_2_l, c_2_u, markC); map.mark(c_3_l, c_3_u, markC); - CHECK_THAT(map, IsMarkedAt(a_1_m)); + CHECK_THAT(map, IsMarkedWith(a_1_m, markA)); map.mark(d_1_l, d_1_u, markD); map.mark(d_2_l, d_2_u, markD); CHECK_THAT(map, IsMarkedAt(a_1_m)); + CHECK_THAT(map, IsMarkedWith(b_1_u, markB)); + CHECK_THAT(map, IsMarkedWith(c_3_m, markC)); + CHECK_THAT(map, IsMarkedWith(d_2_l, markD)); CHECK(map.count() == 13); -} + + // Check move constructor. + IpMap m2{std::move(map)}; + // Original map should be empty. + REQUIRE(map.count() == 0); + // Do all these again on the destination map. + CHECK_THAT(m2, IsMarkedWith(a_1_m, markA)); + CHECK_THAT(m2, IsMarkedWith(a_1_m, markA)); + CHECK_THAT(m2, IsMarkedWith(a_1_m, markA)); + CHECK_THAT(m2, IsMarkedWith(a_1_m, markA)); + CHECK_THAT(m2, IsMarkedWith(b_1_u, markB)); + CHECK_THAT(m2, IsMarkedWith(c_3_m, markC)); + CHECK_THAT(m2, IsMarkedWith(d_2_l, markD)); + CHECK(m2.count() == 13); +}; From 426640346798589dcd7d5b87c9f64e748208ceaf Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Thu, 21 Feb 2019 15:18:47 -0600 Subject: [PATCH 298/526] set thread affinity using type info --- iocore/eventsystem/P_UnixEventProcessor.h | 3 +++ src/traffic_server/InkAPI.cc | 14 ++-------- .../cont_schedule/gold/thread_affinity.gold | 4 +-- tests/tools/plugins/cont_schedule.cc | 26 +++++++++++-------- 4 files changed, 22 insertions(+), 25 deletions(-) diff --git a/iocore/eventsystem/P_UnixEventProcessor.h b/iocore/eventsystem/P_UnixEventProcessor.h index 90f08b95165..003d425e37e 100644 --- a/iocore/eventsystem/P_UnixEventProcessor.h +++ b/iocore/eventsystem/P_UnixEventProcessor.h @@ -71,6 +71,9 @@ EventProcessor::schedule(Event *e, EventType etype, bool fast_signal) e->ethread = ethread; } else { e->ethread = assign_thread(etype); + if (ethread == nullptr) { + e->continuation->setThreadAffinity(e->ethread); + } } if (e->continuation->mutex) { diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 5b63750b6e6..e7d108093ec 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -4409,8 +4409,7 @@ TSContSchedule(TSCont contp, TSHRTime timeout) EThread *eth = i->getThreadAffinity(); if (eth == nullptr) { - eth = this_event_thread(); - i->setThreadAffinity(eth); + return nullptr; } TSAction action; @@ -4438,10 +4437,6 @@ TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) ink_assert(!"not reached"); } - if (i->getThreadAffinity() == nullptr) { - i->setThreadAffinity(this_event_thread()); - } - EventType etype; switch (tp) { @@ -4527,8 +4522,7 @@ TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */) EThread *eth = i->getThreadAffinity(); if (eth == nullptr) { - eth = this_event_thread(); - i->setThreadAffinity(eth); + return nullptr; } TSAction action = reinterpret_cast(eth->schedule_every(i, HRTIME_MSECONDS(every))); @@ -4551,10 +4545,6 @@ TSContScheduleEveryOnPool(TSCont contp, TSHRTime every, TSThreadPool tp) ink_assert(!"not reached"); } - if (i->getThreadAffinity() == nullptr) { - i->setThreadAffinity(this_event_thread()); - } - EventType etype; switch (tp) { diff --git a/tests/gold_tests/cont_schedule/gold/thread_affinity.gold b/tests/gold_tests/cont_schedule/gold/thread_affinity.gold index b81b4f5b76d..09a79cf83ea 100644 --- a/tests/gold_tests/cont_schedule/gold/thread_affinity.gold +++ b/tests/gold_tests/cont_schedule/gold/thread_affinity.gold @@ -1,5 +1,5 @@ `` -``pass [affinity thread is null] -``pass [affinity thread is set] +``pass [affinity thread is not null] ``pass [affinity thread is cleared] +``pass [affinity thread is set] `` diff --git a/tests/tools/plugins/cont_schedule.cc b/tests/tools/plugins/cont_schedule.cc index accf3db958a..abc8066de5c 100644 --- a/tests/tools/plugins/cont_schedule.cc +++ b/tests/tools/plugins/cont_schedule.cc @@ -175,22 +175,22 @@ TSContThreadAffinity_handler(TSCont contp, TSEvent event, void *edata) test_thread = TSEventThreadSelf(); - if (TSContThreadAffinityGet(contp) == nullptr) { - TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is null]"); - TSContThreadAffinitySet(contp, TSEventThreadSelf()); - if (TSContThreadAffinityGet(contp) == test_thread) { - TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is set]"); - TSContThreadAffinityClear(contp); - if (TSContThreadAffinityGet(contp) == nullptr) { - TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is cleared]"); + if (TSContThreadAffinityGet(contp) != nullptr) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is not null]"); + TSContThreadAffinityClear(contp); + if (TSContThreadAffinityGet(contp) == nullptr) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is cleared]"); + TSContThreadAffinitySet(contp, TSEventThreadSelf()); + if (TSContThreadAffinityGet(contp) == test_thread) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is set]"); } else { - TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not cleared]"); + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not set]"); } } else { - TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not set]"); + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not cleared]"); } } else { - TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not null]"); + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is null]"); } return 0; @@ -207,6 +207,7 @@ TSContSchedule_test() } else { TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + TSContThreadAffinityClear(contp); TSContScheduleOnPool(contp, 200, TS_THREAD_POOL_NET); } } @@ -223,8 +224,10 @@ TSContScheduleOnPool_test() } else { TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); TSContScheduleOnPool(contp_1, 0, TS_THREAD_POOL_NET); + TSContThreadAffinityClear(contp_1); TSContScheduleOnPool(contp_1, 100, TS_THREAD_POOL_NET); TSContScheduleOnPool(contp_2, 200, TS_THREAD_POOL_TASK); + TSContThreadAffinityClear(contp_2); TSContScheduleOnPool(contp_2, 300, TS_THREAD_POOL_TASK); } } @@ -240,6 +243,7 @@ TSContScheduleOnThread_test() } else { TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + TSContThreadAffinityClear(contp); TSContScheduleOnPool(contp, 200, TS_THREAD_POOL_NET); } } From ac25c6c0cb99735f9a45c75e4c32b03bf2bae786 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Fri, 22 Feb 2019 09:59:07 -0800 Subject: [PATCH 299/526] Added connect_end to the slow log report --- tools/slow_log_report.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/slow_log_report.pl b/tools/slow_log_report.pl index 597b2f525f7..9207b5906af 100755 --- a/tools/slow_log_report.pl +++ b/tools/slow_log_report.pl @@ -37,7 +37,7 @@ ($) my($stats) = @_; printf("%25s %10s %10s %10s %10s %10s %10s %10s %10s\n", 'key', 'total', 'count', 'mean', 'median', '95th', '99th', 'min', 'max'); - foreach my $key ('ua_begin', 'ua_first_read', 'ua_read_header_done', 'cache_open_read_begin', 'cache_open_read_end', 'dns_lookup_begin', 'dns_lookup_end', 'server_connect', 'server_first_read', 'server_read_header_done', 'server_close', 'ua_close', 'sm_finish') { + foreach my $key ('ua_begin', 'ua_first_read', 'ua_read_header_done', 'cache_open_read_begin', 'cache_open_read_end', 'dns_lookup_begin', 'dns_lookup_end', 'server_connect', 'server_connect_end', 'server_first_read', 'server_read_header_done', 'server_close', 'ua_close', 'sm_finish') { my $count = $stats->{$key}->{count}; my $total = $stats->{$key}->{total}; From 0be4e7326b89670dea1ffe4c7f21c36bfd2bdf59 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 6 Feb 2019 11:10:22 -0600 Subject: [PATCH 300/526] MIMEScanner: Make MIMEScanner a class, not a POD with free functions. --- proxy/hdrs/HTTP.cc | 27 ++- proxy/hdrs/HdrTest.cc | 79 ------ proxy/hdrs/HdrTest.h | 1 - proxy/hdrs/MIME.cc | 303 +++++++++--------------- proxy/hdrs/MIME.h | 82 ++++++- proxy/hdrs/Makefile.am | 1 + proxy/hdrs/unit_tests/test_Hdrs.cc | 86 +++++++ proxy/hdrs/unit_tests/unit_test_main.cc | 21 +- 8 files changed, 305 insertions(+), 295 deletions(-) create mode 100644 proxy/hdrs/unit_tests/test_Hdrs.cc diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 41bf41b2284..eebf4ad8741 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -894,17 +894,21 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const const char *version_start; const char *version_end; + ts::TextView text, parsed; + real_end = end; start: hh->m_polarity = HTTP_TYPE_REQUEST; // Make sure the line is not longer than 64K - if (scanner->m_line_length >= UINT16_MAX) { + if (scanner->get_buffered_line_size() >= UINT16_MAX) { return PARSE_RESULT_ERROR; } - err = mime_scanner_get(scanner, start, real_end, &line_start, &end, &line_is_real, eof, MIME_SCANNER_TYPE_LINE); + text.assign(*start, real_end); + err = scanner->get(text, parsed, line_is_real, eof, MIMEScanner::LINE); + *start = text.data(); if (err < 0) { return err; } @@ -917,9 +921,9 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const return err; } - cur = line_start; - ink_assert((end - cur) >= 0); - ink_assert((end - cur) < UINT16_MAX); + ink_assert(parsed.size() < UINT16_MAX); + line_start = cur = parsed.data(); + end = parsed.data_end(); must_copy_strings = (must_copy_strings || (!line_is_real)); @@ -1244,11 +1248,14 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const hh->m_polarity = HTTP_TYPE_RESPONSE; // Make sure the line is not longer than 64K - if (scanner->m_line_length >= UINT16_MAX) { + if (scanner->get_buffered_line_size() >= UINT16_MAX) { return PARSE_RESULT_ERROR; } - err = mime_scanner_get(scanner, start, real_end, &line_start, &end, &line_is_real, eof, MIME_SCANNER_TYPE_LINE); + ts::TextView text{*start, real_end}; + ts::TextView parsed; + err = scanner->get(text, parsed, line_is_real, eof, MIMEScanner::LINE); + *start = text.data(); if (err < 0) { return err; } @@ -1256,9 +1263,9 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const return err; } - cur = line_start; - ink_assert((end - cur) >= 0); - ink_assert((end - cur) < UINT16_MAX); + ink_assert(parsed.size() < UINT16_MAX); + line_start = cur = parsed.data(); + end = parsed.data_end(); must_copy_strings = (must_copy_strings || (!line_is_real)); diff --git a/proxy/hdrs/HdrTest.cc b/proxy/hdrs/HdrTest.cc index ee3f93db134..875e4e1a738 100644 --- a/proxy/hdrs/HdrTest.cc +++ b/proxy/hdrs/HdrTest.cc @@ -74,7 +74,6 @@ HdrTest::go(RegressionTest *t, int /* atype ATS_UNUSED */) status = status & test_url(); status = status & test_arena(); status = status & test_regex(); - status = status & test_http_parser_eos_boundary_cases(); status = status & test_http_mutation(); status = status & test_mime(); status = status & test_http(); @@ -517,84 +516,6 @@ HdrTest::test_mime() return (failures_to_status("test_mime", 0)); } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_http_parser_eos_boundary_cases() -{ - struct { - const char *msg; - int expected_result; - int expected_bytes_consumed; - } tests[] = { - {"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26}, - {"GET /index.html HTTP/1.0\r\n\r\n***BODY****", PARSE_RESULT_DONE, 28}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n***BODY****", PARSE_RESULT_DONE, 48}, - {"GET", PARSE_RESULT_ERROR, 3}, - {"GET /index.html", PARSE_RESULT_ERROR, 15}, - {"GET /index.html\r\n", PARSE_RESULT_ERROR, 17}, - {"GET /index.html HTTP/1.0", PARSE_RESULT_ERROR, 24}, - {"GET /index.html HTTP/1.0\r", PARSE_RESULT_ERROR, 25}, - {"GET /index.html HTTP/1.0\n", PARSE_RESULT_DONE, 25}, - {"GET /index.html HTTP/1.0\n\n", PARSE_RESULT_DONE, 26}, - {"GET /index.html HTTP/1.0\r\n\r\n", PARSE_RESULT_DONE, 28}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar", PARSE_RESULT_ERROR, 44}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 45}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n", PARSE_RESULT_DONE, 48}, - {"GET /index.html HTTP/1.0\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 44}, - {"GET /index.html HTTP/1.0\nUser-Agent: foobar\nBoo: foo\n", PARSE_RESULT_DONE, 53}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46}, - {"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26}, - {"", PARSE_RESULT_ERROR, 0}, - {nullptr, 0, 0}, - }; - - int i, ret, bytes_consumed; - const char *orig_start; - const char *start; - const char *end; - HTTPParser parser; - - int failures = 0; - - bri_box("test_http_parser_eos_boundary_cases"); - - http_parser_init(&parser); - - for (i = 0; tests[i].msg != nullptr; i++) { - HTTPHdr req_hdr; - - start = tests[i].msg; - end = start + strlen(start); // 1 character past end of string - - req_hdr.create(HTTP_TYPE_REQUEST); - - http_parser_clear(&parser); - - orig_start = start; - ret = req_hdr.parse_req(&parser, &start, end, true); - bytes_consumed = (int)(start - orig_start); - - printf("======== test %d (length=%d, consumed=%d)\n", i, (int)strlen(tests[i].msg), bytes_consumed); - printf("[%s]\n", tests[i].msg); - printf("\n["); - req_hdr.print(nullptr, 0, nullptr, nullptr); - printf("]\n\n"); - - if ((ret != tests[i].expected_result) || (bytes_consumed != tests[i].expected_bytes_consumed)) { - ++failures; - printf("FAILED: test %d: retval , eaten \n\n", i, tests[i].expected_result, ret, - tests[i].expected_bytes_consumed, bytes_consumed); - } - - req_hdr.destroy(); - } - - return (failures_to_status("test_http_parser_eos_boundary_cases", failures)); -} - /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/hdrs/HdrTest.h b/proxy/hdrs/HdrTest.h index 91a979c9c4a..7f8dc839b78 100644 --- a/proxy/hdrs/HdrTest.h +++ b/proxy/hdrs/HdrTest.h @@ -51,7 +51,6 @@ class HdrTest int test_parse_date(); int test_format_date(); int test_url(); - int test_http_parser_eos_boundary_cases(); int test_arena(); int test_regex(); int test_accept_language_match(); diff --git a/proxy/hdrs/MIME.cc b/proxy/hdrs/MIME.cc index 81c462223b7..d130b202feb 100644 --- a/proxy/hdrs/MIME.cc +++ b/proxy/hdrs/MIME.cc @@ -34,6 +34,8 @@ #include "HdrUtils.h" #include "HttpCompat.h" +using ts::TextView; + /*********************************************************************** * * * C O M P I L E O P T I O N S * @@ -2302,215 +2304,161 @@ MIMEHdr::get_host_port_values(const char **host_ptr, ///< Pointer to host. * P A R S E R * * * ***********************************************************************/ -void -_mime_scanner_init(MIMEScanner *scanner) -{ - scanner->m_line = nullptr; - scanner->m_line_size = 0; - scanner->m_line_length = 0; - scanner->m_state = MIME_PARSE_BEFORE; -} -////////////////////////////////////////////////////// -// init first time structure setup // -// clear resets an already-initialized structure // -////////////////////////////////////////////////////// void -mime_scanner_init(MIMEScanner *scanner) +MIMEScanner::init() { - _mime_scanner_init(scanner); + m_state = INITIAL_PARSE_STATE; + // Ugly, but required because of how proxy allocation works - that leaves the instance in a + // random state, so even assigning to it can crash. Because this method substitutes for a real + // constructor in the proxy allocation system, call the CTOR here. Any memory that gets allocated + // is supposed to be cleaned up by calling @c clear on this object. + new (&m_line) std::string; } -// clear is to reset an already initialized structure -void -mime_scanner_clear(MIMEScanner *scanner) +MIMEScanner & +MIMEScanner::append(TextView text) { - ats_free(scanner->m_line); - _mime_scanner_init(scanner); -} - -void -mime_scanner_append(MIMEScanner *scanner, const char *data, int data_size) -{ - int free_size = scanner->m_line_size - scanner->m_line_length; - - ////////////////////////////////////////////////////// - // if not enough space, allocate or grow the buffer // - ////////////////////////////////////////////////////// - if (data_size > free_size) { // need to allocate/grow the buffer - if (scanner->m_line_size == 0) { // buffer should be at least 128 bytes - scanner->m_line_size = 128; - } - - while (free_size < data_size) { // grow buffer by powers of 2 - scanner->m_line_size *= 2; - free_size = scanner->m_line_size - scanner->m_line_length; - } - - if (scanner->m_line == nullptr) { // if no buffer yet, allocate one - scanner->m_line = (char *)ats_malloc(scanner->m_line_size); - } else { - scanner->m_line = (char *)ats_realloc(scanner->m_line, scanner->m_line_size); - } - } - //////////////////////////////////////////////// - // append new data onto the end of the buffer // - //////////////////////////////////////////////// - - memcpy(&(scanner->m_line[scanner->m_line_length]), data, data_size); - scanner->m_line_length += data_size; + m_line += text; + return *this; } ParseResult -mime_scanner_get(MIMEScanner *S, const char **raw_input_s, const char *raw_input_e, const char **output_s, const char **output_e, - bool *output_shares_raw_input, - bool raw_input_eof, ///< All data has been received for this header. - int raw_input_scan_type) +MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, bool eof_p, ScanType scan_type) { - const char *raw_input_c, *lf_ptr; ParseResult zret = PARSE_RESULT_CONT; // Need this for handling dangling CR. - static const char RAW_CR = ParseRules::CHAR_CR; - - ink_assert((raw_input_s != nullptr) && (*raw_input_s != nullptr)); - ink_assert(raw_input_e != nullptr); + static const char RAW_CR{ParseRules::CHAR_CR}; - raw_input_c = *raw_input_s; - - while (PARSE_RESULT_CONT == zret && raw_input_c < raw_input_e) { - ptrdiff_t runway = raw_input_e - raw_input_c; // remaining input. - switch (S->m_state) { + auto text = input; + while (PARSE_RESULT_CONT == zret && !text.empty()) { + switch (m_state) { case MIME_PARSE_BEFORE: // waiting to find a field. - if (ParseRules::is_cr(*raw_input_c)) { - ++raw_input_c; - if (runway >= 2 && ParseRules::is_lf(*raw_input_c)) { + if (ParseRules::is_cr(*text)) { + ++text; + if (!text.empty() && ParseRules::is_lf(*text)) { // optimize a bit - this happens >99% of the time after a CR. - ++raw_input_c; + ++text; zret = PARSE_RESULT_DONE; } else { - S->m_state = MIME_PARSE_FOUND_CR; + m_state = MIME_PARSE_FOUND_CR; } - } else if (ParseRules::is_lf(*raw_input_c)) { - ++raw_input_c; + } else if (ParseRules::is_lf(*text)) { + ++text; zret = PARSE_RESULT_DONE; // Required by regression test. } else { // consume this character in the next state. - S->m_state = MIME_PARSE_INSIDE; + m_state = MIME_PARSE_INSIDE; } break; case MIME_PARSE_FOUND_CR: - // Looking for a field and found a CR, which should mean terminating - // the header. Note that we've left the CR in the input so we have - // to skip over it. - if (ParseRules::is_lf(*raw_input_c)) { - // Header terminated. - ++raw_input_c; + // Looking for a field and found a CR, which should mean terminating the header. + if (ParseRules::is_lf(*text)) { + ++text; zret = PARSE_RESULT_DONE; } else { - // This really should be an error (spec doesn't permit lone CR) - // but the regression tests require it. - mime_scanner_append(S, &RAW_CR, 1); - S->m_state = MIME_PARSE_INSIDE; + // This really should be an error (spec doesn't permit lone CR) but the regression tests + // require it. + this->append({&RAW_CR, 1}); + m_state = MIME_PARSE_INSIDE; } break; - case MIME_PARSE_INSIDE: - lf_ptr = static_cast(memchr(raw_input_c, ParseRules::CHAR_LF, runway)); - if (lf_ptr) { - raw_input_c = lf_ptr + 1; - if (MIME_SCANNER_TYPE_LINE == raw_input_scan_type) { - zret = PARSE_RESULT_OK; - S->m_state = MIME_PARSE_BEFORE; + case MIME_PARSE_INSIDE: { + auto lf_off = text.find(ParseRules::CHAR_LF); + if (lf_off != TextView::npos) { + text.remove_prefix(lf_off + 1); // drop up to and including LF + if (LINE == scan_type) { + zret = PARSE_RESULT_OK; + m_state = MIME_PARSE_BEFORE; } else { - S->m_state = MIME_PARSE_AFTER; + m_state = MIME_PARSE_AFTER; // looking for line folding. } - } else { - raw_input_c = raw_input_e; // grab all that's available. + } else { // no EOL, consume all text without changing state. + text.remove_prefix(text.size()); } - break; + } break; case MIME_PARSE_AFTER: - // After a LF. Might be the end or a continuation. - if (ParseRules::is_ws(*raw_input_c)) { - char *unfold = const_cast(raw_input_c - 1); - - *unfold-- = ' '; + // After a LF, the next line might be a continuation / folded line. That's indicated by a + // starting whitespace. If that's the case, back up over the preceding CR/LF with space and + // pretend it's the same line. + if (ParseRules::is_ws(*text)) { // folded line. + char *unfold = const_cast(text.data() - 1); + *unfold-- = ' '; if (ParseRules::is_cr(*unfold)) { *unfold = ' '; } - S->m_state = MIME_PARSE_INSIDE; // back inside the field. + m_state = MIME_PARSE_INSIDE; // back inside the field. } else { - S->m_state = MIME_PARSE_BEFORE; // field terminated. - zret = PARSE_RESULT_OK; + m_state = MIME_PARSE_BEFORE; // field terminated. + zret = PARSE_RESULT_OK; } break; } } - ptrdiff_t data_size = raw_input_c - *raw_input_s; + TextView parsed_text{input.data(), text.data()}; + bool save_parsed_text_p = !parsed_text.empty(); if (PARSE_RESULT_CONT == zret) { - // data ran out before we got a clear final result. - // There a number of things we need to check and possibly adjust - // that result. It's less complex to do this cleanup than handle - // in the parser state machine. - if (raw_input_eof) { + // data ran out before we got a clear final result. There a number of things we need to check + // and possibly adjust that result. It's less complex to do this cleanup than handle all of + // these checks in the parser state machine. + if (eof_p) { // Should never return PARSE_CONT if we've hit EOF. - if (0 == data_size) { + if (parsed_text.empty()) { // all input previously consumed. If we're between fields, that's cool. - if (MIME_PARSE_INSIDE != S->m_state) { - S->m_state = MIME_PARSE_BEFORE; // probably not needed... - zret = PARSE_RESULT_DONE; + if (MIME_PARSE_INSIDE != m_state) { + m_state = MIME_PARSE_BEFORE; // probably not needed... + zret = PARSE_RESULT_DONE; } else { zret = PARSE_RESULT_ERROR; // unterminated field. } - } else if (MIME_PARSE_AFTER == S->m_state) { - // Special case it seems - need to accept the final field - // even if there's no header terminating CR LF. We check for - // absolute end of input because otherwise this might be - // a multiline field where we haven't seen the next leading space. - S->m_state = MIME_PARSE_BEFORE; - zret = PARSE_RESULT_OK; + } else if (MIME_PARSE_AFTER == m_state) { + // Special case it seems - need to accept the final field even if there's no header + // terminating CR LF. This is only reasonable after absolute end of input (EOF) because + // otherwise this might be a multiline field where we haven't seen the next leading space. + m_state = MIME_PARSE_BEFORE; + zret = PARSE_RESULT_OK; } else { // Partial input, no field / line CR LF zret = PARSE_RESULT_ERROR; // Unterminated field. } - } else if (data_size) { - if (MIME_PARSE_INSIDE == S->m_state) { + } else if (!parsed_text.empty()) { + if (MIME_PARSE_INSIDE == m_state) { // Inside a field but more data is expected. Save what we've got. - mime_scanner_append(S, *raw_input_s, data_size); - data_size = 0; // Don't append again. - } else if (MIME_PARSE_AFTER == S->m_state) { + this->append(parsed_text); // Do this here to force appending. + save_parsed_text_p = false; // don't double append. + } else if (MIME_PARSE_AFTER == m_state) { // After a field but we still have data. Need to parse it too. - S->m_state = MIME_PARSE_BEFORE; - zret = PARSE_RESULT_OK; + m_state = MIME_PARSE_BEFORE; + zret = PARSE_RESULT_OK; } } } - if (data_size && S->m_line_length) { + if (save_parsed_text_p && !m_line.empty()) { // If we're already accumulating, continue to do so if we have data. - mime_scanner_append(S, *raw_input_s, data_size); + this->append(parsed_text); } - // No sharing if we've accumulated data (really, force this to make compiler shut up). - *output_shares_raw_input = 0 == S->m_line_length; // adjust out arguments. if (PARSE_RESULT_CONT != zret) { - if (0 != S->m_line_length) { - *output_s = S->m_line; - *output_e = *output_s + S->m_line_length; - S->m_line_length = 0; + if (!m_line.empty()) { + output = m_line; + m_line.resize(0); // depending resize(0) not deallocating internal string memory. + output_shares_input = false; } else { - *output_s = *raw_input_s; - *output_e = raw_input_c; + output = parsed_text; + output_shares_input = true; } } - // Make sure there are no '\0' in the input scanned so far - if (zret != PARSE_RESULT_ERROR && memchr(*raw_input_s, '\0', raw_input_c - *raw_input_s) != nullptr) { + // Make sure there are no null characters in the input scanned so far + if (zret != PARSE_RESULT_ERROR && TextView::npos != parsed_text.find('\0')) { zret = PARSE_RESULT_ERROR; } - *raw_input_s = raw_input_c; // mark input consumed. + input.remove_prefix(parsed_text.size()); return zret; } @@ -2528,14 +2476,14 @@ _mime_parser_init(MIMEParser *parser) void mime_parser_init(MIMEParser *parser) { - mime_scanner_init(&parser->m_scanner); + parser->m_scanner.init(); _mime_parser_init(parser); } void mime_parser_clear(MIMEParser *parser) { - mime_scanner_clear(&parser->m_scanner); + parser->m_scanner.clear(); _mime_parser_init(parser); } @@ -2545,17 +2493,6 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char { ParseResult err; bool line_is_real; - const char *colon; - const char *line_c; - const char *line_s = nullptr; - const char *line_e = nullptr; - const char *field_name_first; - const char *field_name_last; - const char *field_value_first; - const char *field_value_last; - const char *field_line_first; - const char *field_line_last; - int field_name_length, field_value_length; MIMEScanner *scanner = &parser->m_scanner; @@ -2564,22 +2501,23 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // get a name:value line, with all continuation lines glued into one line // //////////////////////////////////////////////////////////////////////////// - err = mime_scanner_get(scanner, real_s, real_e, &line_s, &line_e, &line_is_real, eof, MIME_SCANNER_TYPE_FIELD); + TextView text{*real_s, real_e}; + TextView parsed; + err = scanner->get(text, parsed, line_is_real, eof, MIMEScanner::FIELD); + *real_s = text.data(); if (err != PARSE_RESULT_OK) { return err; } - line_c = line_s; - ////////////////////////////////////////////////// // if got a LF or CR on its own, end the header // ////////////////////////////////////////////////// - if ((line_e - line_c >= 2) && (line_c[0] == ParseRules::CHAR_CR) && (line_c[1] == ParseRules::CHAR_LF)) { + if ((parsed.size() >= 2) && (parsed[0] == ParseRules::CHAR_CR) && (parsed[1] == ParseRules::CHAR_LF)) { return PARSE_RESULT_DONE; } - if ((line_e - line_c >= 1) && (line_c[0] == ParseRules::CHAR_LF)) { + if ((parsed.size() >= 1) && (parsed[0] == ParseRules::CHAR_LF)) { return PARSE_RESULT_DONE; } @@ -2587,27 +2525,23 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // find pointers into the name:value field // ///////////////////////////////////////////// - field_line_first = line_c; - field_line_last = line_e - 1; - - // find name first - field_name_first = line_c; /** * Fix for INKqa09141. The is_token function fails for '@' character. * Header names starting with '@' signs are valid headers. Hence we * have to add one more check to see if the first parameter is '@' * character then, the header name is valid. **/ - if ((!ParseRules::is_token(*field_name_first)) && (*field_name_first != '@')) { + if ((!ParseRules::is_token(*parsed)) && (*parsed != '@')) { continue; // toss away garbage line } // find name last - colon = (char *)memchr(line_c, ':', (line_e - line_c)); - if (!colon) { + auto field_value = parsed; // need parsed as is later on. + auto field_name = field_value.split_prefix_at(':'); + if (field_name.empty()) { continue; // toss away garbage line } - field_name_last = colon - 1; + // RFC7230 section 3.2.4: // No whitespace is allowed between the header field-name and colon. In // the past, differences in the handling of such whitespace have led to @@ -2615,57 +2549,44 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // server MUST reject any received request message that contains // whitespace between a header field-name and colon with a response code // of 400 (Bad Request). - if ((field_name_last >= field_name_first) && is_ws(*field_name_last)) { + if (is_ws(field_name.back())) { return PARSE_RESULT_ERROR; } // find value first - field_value_first = colon + 1; - while ((field_value_first < line_e) && is_ws(*field_value_first)) { - ++field_value_first; - } - - // find_value_last - field_value_last = line_e - 1; - while ((field_value_last >= field_value_first) && ParseRules::is_wslfcr(*field_value_last)) { - --field_value_last; - } - - field_name_length = (int)(field_name_last - field_name_first + 1); - field_value_length = (int)(field_value_last - field_value_first + 1); + field_value.ltrim_if(&ParseRules::is_ws); + field_value.rtrim_if(&ParseRules::is_wslfcr); // Make sure the name or value is not longer than 64K - if (field_name_length >= UINT16_MAX || field_value_length >= UINT16_MAX) { + if (field_name.size() >= UINT16_MAX || field_value.size() >= UINT16_MAX) { return PARSE_RESULT_ERROR; } - int total_line_length = (int)(field_line_last - field_line_first + 1); + // int total_line_length = (int)(field_line_last - field_line_first + 1); ////////////////////////////////////////////////////////////////////// // if we can't leave the name & value in the real buffer, copy them // ////////////////////////////////////////////////////////////////////// if (must_copy_strings || (!line_is_real)) { - int length = total_line_length; - char *dup = heap->duplicate_str(field_name_first, length); - intptr_t delta = dup - field_name_first; - - field_name_first += delta; - field_value_first += delta; + char *dup = heap->duplicate_str(parsed.data(), parsed.size()); + intptr_t delta = dup - parsed.data(); + field_name.assign(field_name.data() + delta, field_name.size()); + field_value.assign(field_value.data() + delta, field_value.size()); } /////////////////////// // tokenize the name // /////////////////////// - int field_name_wks_idx = hdrtoken_tokenize(field_name_first, field_name_length); + int field_name_wks_idx = hdrtoken_tokenize(field_name.data(), field_name.size()); /////////////////////////////////////////// // build and insert the new field object // /////////////////////////////////////////// MIMEField *field = mime_field_create(heap, mh); - mime_field_name_value_set(heap, mh, field, field_name_wks_idx, field_name_first, field_name_length, field_value_first, - field_value_length, true, total_line_length, false); + mime_field_name_value_set(heap, mh, field, field_name_wks_idx, field_name.data(), field_name.size(), field_value.data(), + field_value.size(), true, parsed.size(), false); mime_hdr_field_attach(mh, field, 1, nullptr); } } diff --git a/proxy/hdrs/MIME.h b/proxy/hdrs/MIME.h index 5e78834efd7..972af091572 100644 --- a/proxy/hdrs/MIME.h +++ b/proxy/hdrs/MIME.h @@ -25,6 +25,7 @@ #include #include +#include #include "tscore/ink_assert.h" #include "tscore/ink_apidefs.h" @@ -33,6 +34,8 @@ #include "HdrHeap.h" #include "HdrToken.h" +#include "tscpp/util/TextView.h" + /*********************************************************************** * * * Defines * @@ -58,9 +61,6 @@ enum MimeParseState { MIME_PARSE_AFTER, ///< After a field. }; -#define MIME_SCANNER_TYPE_LINE 0 -#define MIME_SCANNER_TYPE_FIELD 1 - /*********************************************************************** * * * Assertions * @@ -283,13 +283,75 @@ struct MIMEHdrImpl : public HdrHeapObjImpl { * * ***********************************************************************/ +/** A pre-parser used to extract MIME "lines" from raw input for further parsing. + * + * This maintains an internal line buffer which is used to keeping content between calls + * when the parse has not yet completed. + * + */ struct MIMEScanner { - char *m_line; // buffered line being built up - int m_line_length; // size of real live data in buffer - int m_line_size; // total allocated size of buffer - MimeParseState m_state; ///< Parsing machine state. + using self_type = MIMEScanner; ///< Self reference type. +public: + /// Type of input scanning. + enum ScanType { + LINE = 0, ///< Scan a single line. + FIELD = 1, ///< Scan with line folding enabled. + }; + + void init(); ///< Pseudo-constructor required by proxy allocation. + void clear(); ///< Pseudo-destructor required by proxy allocation. + + /// @return The size of the internal line buffer. + size_t get_buffered_line_size() const; + + /** Scan @a input for MIME data delimited by CR/LF end of line markers. + * + * @param input [in,out] Text to scan. + * @param output [out] Parsed text from @a input, if any. + * @param output_shares_input [out] Whether @a output is in @a input. + * @param eof_p [in] The source for @a input is done, no more data will ever be available. + * @param scan_type [in] Whether to check for line folding. + * @return The result of scanning. + * + * @a input is updated to remove text that was scanned. @a output is updated to be a view of the + * scanned @a input. This is separate because @a output may be a view of @a input or a view of the + * internal line buffer. Which of these cases obtains is returned in @a output_shares_input. This + * is @c true if @a output is a view of @a input, and @c false if @a output is a view of the + * internal buffer, but is only set if the result is not @c PARSE_RESULT_CONT (that is, it is not + * set until scanning has completed). If @a scan_type is @c FIELD then folded lines are + * accumulated in to a single line stored in the internal buffer. Otherwise the scanning + * terminates at the first CR/LF. + */ + ParseResult get(ts::TextView &input, ts::TextView &output, bool &output_shares_input, bool eof_p, ScanType scan_type); + +protected: + /** Append @a text to the internal buffer. + * + * @param text Text to append. + * @return @a this + * + * A copy of @a text is appended to the internal line buffer. + */ + self_type &append(ts::TextView text); + + static constexpr MimeParseState INITIAL_PARSE_STATE = MIME_PARSE_BEFORE; + std::string m_line; ///< Internally buffered line data for field coalescence. + MimeParseState m_state{INITIAL_PARSE_STATE}; ///< Parsing machine state. }; +inline size_t +MIMEScanner::get_buffered_line_size() const +{ + return m_line.size(); +} + +inline void +MIMEScanner::clear() +{ + m_line.clear(); + m_state = INITIAL_PARSE_STATE; +} + struct MIMEParser { MIMEScanner m_scanner; int32_t m_field; @@ -692,12 +754,6 @@ void mime_field_name_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, void mime_field_value_append(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, const char *value, int length, bool prepend_comma, const char separator); -void mime_scanner_init(MIMEScanner *scanner); -void mime_scanner_clear(MIMEScanner *scanner); -void mime_scanner_append(MIMEScanner *scanner, const char *data, int data_size); -ParseResult mime_scanner_get(MIMEScanner *S, const char **raw_input_s, const char *raw_input_e, const char **output_s, - const char **output_e, bool *output_shares_raw_input, bool raw_input_eof, int raw_input_scan_type); - void mime_parser_init(MIMEParser *parser); void mime_parser_clear(MIMEParser *parser); ParseResult mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char **real_s, const char *real_e, diff --git a/proxy/hdrs/Makefile.am b/proxy/hdrs/Makefile.am index 172c0d83e46..75c33860eb4 100644 --- a/proxy/hdrs/Makefile.am +++ b/proxy/hdrs/Makefile.am @@ -82,6 +82,7 @@ test_proxy_hdrs_CPPFLAGS = $(AM_CPPFLAGS)\ test_proxy_hdrs_SOURCES = \ unit_tests/unit_test_main.cc \ + unit_tests/test_Hdrs.cc \ unit_tests/test_HdrUtils.cc test_proxy_hdrs_LDADD = \ diff --git a/proxy/hdrs/unit_tests/test_Hdrs.cc b/proxy/hdrs/unit_tests/test_Hdrs.cc new file mode 100644 index 00000000000..a43503a64f6 --- /dev/null +++ b/proxy/hdrs/unit_tests/test_Hdrs.cc @@ -0,0 +1,86 @@ +/** @file + + Catch-based unit tests for various header logic. + This replaces the old regression tests in HdrTest.cc. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "catch.hpp" + +#include "HTTP.h" + +// replaces test_http_parser_eos_boundary_cases +TEST_CASE("HdrTest", "[proxy][hdrtest]") +{ + struct Test { + ts::TextView msg; + int expected_result; + int expected_bytes_consumed; + }; + static const std::array tests = {{ + {"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26}, + {"GET /index.html HTTP/1.0\r\n\r\n***BODY****", PARSE_RESULT_DONE, 28}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n***BODY****", PARSE_RESULT_DONE, 48}, + {"GET", PARSE_RESULT_ERROR, 3}, + {"GET /index.html", PARSE_RESULT_ERROR, 15}, + {"GET /index.html\r\n", PARSE_RESULT_ERROR, 17}, + {"GET /index.html HTTP/1.0", PARSE_RESULT_ERROR, 24}, + {"GET /index.html HTTP/1.0\r", PARSE_RESULT_ERROR, 25}, + {"GET /index.html HTTP/1.0\n", PARSE_RESULT_DONE, 25}, + {"GET /index.html HTTP/1.0\n\n", PARSE_RESULT_DONE, 26}, + {"GET /index.html HTTP/1.0\r\n\r\n", PARSE_RESULT_DONE, 28}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar", PARSE_RESULT_ERROR, 44}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 45}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n", PARSE_RESULT_DONE, 48}, + {"GET /index.html HTTP/1.0\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 44}, + {"GET /index.html HTTP/1.0\nUser-Agent: foobar\nBoo: foo\n", PARSE_RESULT_DONE, 53}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46}, + {"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26}, + {"", PARSE_RESULT_ERROR, 0}, + }}; + + HTTPParser parser; + + http_parser_init(&parser); + + for (auto const &test : tests) { + HTTPHdr req_hdr; + HdrHeap *heap = new_HdrHeap(HDR_HEAP_DEFAULT_SIZE + 64); // extra to prevent proxy allocation. + + req_hdr.create(HTTP_TYPE_REQUEST, heap); + + http_parser_clear(&parser); + + auto start = test.msg.data(); + auto ret = req_hdr.parse_req(&parser, &start, test.msg.data_end(), true); + auto bytes_consumed = start - test.msg.data(); + + REQUIRE(bytes_consumed == test.expected_bytes_consumed); + REQUIRE(ret == test.expected_result); + + req_hdr.destroy(); + } +} diff --git a/proxy/hdrs/unit_tests/unit_test_main.cc b/proxy/hdrs/unit_tests/unit_test_main.cc index 6aed3a63097..636c5ccbe5d 100644 --- a/proxy/hdrs/unit_tests/unit_test_main.cc +++ b/proxy/hdrs/unit_tests/unit_test_main.cc @@ -21,5 +21,24 @@ limitations under the License. */ -#define CATCH_CONFIG_MAIN +#include "HTTP.h" + +#define CATCH_CONFIG_RUNNER #include "catch.hpp" + +extern int cmd_disable_pfreelist; + +int +main(int argc, char *argv[]) +{ + // No thread setup, forbid use of thread local allocators. + cmd_disable_pfreelist = true; + // Get all of the HTTP WKS items populated. + http_init(); + + int result = Catch::Session().run(argc, argv); + + // global clean-up... + + return result; +} From d9dc0f42e9f161f8a943483ab8dc38d178b18e16 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Wed, 13 Feb 2019 20:49:23 +0000 Subject: [PATCH 301/526] Implement nbf claim in Uri Signing Plugin --- plugins/experimental/uri_signing/jwt.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index a38565c5385..69a07e3714c 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -97,12 +97,6 @@ unsupported_string_claim(const char *str) return !str; } -bool -unsupported_date_claim(double t) -{ - return isnan(t); -} - bool jwt_validate(struct jwt *jwt) { @@ -126,8 +120,8 @@ jwt_validate(struct jwt *jwt) return false; } - if (!unsupported_date_claim(jwt->nbf)) { - PluginDebug("Initial JWT Failure: nbf unsupported"); + if (now() < jwt->nbf) { + PluginDebug("Initial JWT Failure: nbf claim violated"); return false; } From b4e53199f0d93eecd82db13a3f9841760d1913a0 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 20 Feb 2019 16:15:50 +0900 Subject: [PATCH 302/526] Cleanup: remove duplicated SSL_CTX_set_tlsext_status_cb calls for OCSP Stapling --- iocore/net/SSLUtils.cc | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index ad20188f9c1..653cec49cbf 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -1285,7 +1285,7 @@ setClientCertLevel(SSL *ssl, uint8_t certLevel) } SSL_CTX * -SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMultCertSettings, std::vector &certList) +SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMultCertSettings, std::vector &cert_list) { int server_verify_client; SSL_CTX *ctx = SSLDefaultServerContext(); @@ -1416,7 +1416,7 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu goto fail; } - certList.push_back(cert); + cert_list.push_back(cert); if (SSLConfigParams::load_ssl_file_cb) { SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str(), CONFIG_FLAG_UNVERSIONED); } @@ -1605,6 +1605,12 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu if (SSLConfigParams::ssl_ocsp_enabled) { Debug("ssl", "SSL OCSP Stapling is enabled"); SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); + + for (auto cert : cert_list) { + if (!ssl_stapling_init_cert(ctx, cert, setting_cert)) { + Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", setting_cert); + } + } } else { Debug("ssl", "SSL OCSP Stapling is disabled"); } @@ -1625,7 +1631,7 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } SSL_CLEAR_PW_REFERENCES(ctx) SSLReleaseContext(ctx); - for (auto cert : certList) { + for (auto cert : cert_list) { X509_free(cert); } @@ -1703,24 +1709,6 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons #endif } -#ifdef TS_USE_TLS_OCSP - if (SSLConfigParams::ssl_ocsp_enabled) { - Debug("ssl", "SSL OCSP Stapling is enabled"); - SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); - for (auto cert : cert_list) { - if (!ssl_stapling_init_cert(ctx, cert, certname)) { - Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", (const char *)certname); - } - } - } else { - Debug("ssl", "SSL OCSP Stapling is disabled"); - } -#else - if (SSLConfigParams::ssl_ocsp_enabled) { - Warning("failed to enable SSL OCSP Stapling; this version of OpenSSL does not support it"); - } -#endif /* TS_USE_TLS_OCSP */ - // Insert additional mappings. Note that this maps multiple keys to the same value, so when // this code is updated to reconfigure the SSL certificates, it will need some sort of // refcounting or alternate way of avoiding double frees. From 65653961072d9e6d56570e5a744b7499ee6b1b22 Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Mon, 25 Feb 2019 15:30:46 -0800 Subject: [PATCH 303/526] HdrHeap default size unit test fix. After refactoring "HdrHeap refresh" in PR #4953 unit test need to be updated as well. Changing HDR_HEAP_DEFAULT_SIZE to HdrHeap::DEFAULT_SIZE --- doc/developer-guide/core-architecture/heap.en.rst | 4 ++-- proxy/hdrs/unit_tests/test_Hdrs.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/developer-guide/core-architecture/heap.en.rst b/doc/developer-guide/core-architecture/heap.en.rst index e03b5ad77a7..31bee10df8f 100644 --- a/doc/developer-guide/core-architecture/heap.en.rst +++ b/doc/developer-guide/core-architecture/heap.en.rst @@ -132,10 +132,10 @@ Classes .. function:: HdrHeap * new_HdrHeap(int n) - Create and return a new instance of :class:`HdrHeap`. If :arg:`n` is less than ``HDR_HEAP_DEFAULT_SIZE`` + Create and return a new instance of :class:`HdrHeap`. If :arg:`n` is less than ``HdrHeap::DEFAULT_SIZE`` it is increased to that value. - If the allocated size is ``HDR_HEAP_DEFAULT_SIZE`` (or smaller and upsized to that value) then + If the allocated size is ``HdrHeap::DEFAULT_SIZE`` (or smaller and upsized to that value) then the instance is allocated from a thread local pool via :code:`hdrHeapAllocator`. If larger it is allocated from global memory via :code:`ats_malloc`. diff --git a/proxy/hdrs/unit_tests/test_Hdrs.cc b/proxy/hdrs/unit_tests/test_Hdrs.cc index a43503a64f6..790418c45c4 100644 --- a/proxy/hdrs/unit_tests/test_Hdrs.cc +++ b/proxy/hdrs/unit_tests/test_Hdrs.cc @@ -68,7 +68,7 @@ TEST_CASE("HdrTest", "[proxy][hdrtest]") for (auto const &test : tests) { HTTPHdr req_hdr; - HdrHeap *heap = new_HdrHeap(HDR_HEAP_DEFAULT_SIZE + 64); // extra to prevent proxy allocation. + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); // extra to prevent proxy allocation. req_hdr.create(HTTP_TYPE_REQUEST, heap); From 902d5ad96a446f7485716e8ec72f7f06551418b6 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 26 Feb 2019 10:55:14 +0900 Subject: [PATCH 304/526] Ignore test_librecords It is check program introduced by c83061d68dfbe9e25210a953ea640063a0525ab7. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ec01fe126c4..0aeedac9e89 100644 --- a/.gitignore +++ b/.gitignore @@ -87,6 +87,7 @@ src/tscore/test_Regex src/tscore/test_X509HostnameValidator src/tscore/test_tscore src/tscpp/util/test_tscpputil +lib/records/test_librecords lib/perl/lib/Apache/TS.pm iocore/net/test_certlookup From c15daac8ca95eeecf47a50d8d96cb9d25149760e Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Sat, 23 Feb 2019 13:55:09 -0600 Subject: [PATCH 305/526] IP support: Make IpAddr constexpr constructible, define min/max addresses in IpMap. --- include/tscore/IpMap.h | 21 +++++++++++++++++++++ include/tscore/ink_inet.h | 34 +++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/include/tscore/IpMap.h b/include/tscore/IpMap.h index 17ecc956673..ae57329ef6e 100644 --- a/include/tscore/IpMap.h +++ b/include/tscore/IpMap.h @@ -102,6 +102,16 @@ class IpMap class iterator; // forward declare. + static constexpr in_addr_t RAW_IP4_MIN_ADDR = 0; + static constexpr IpAddr IP4_MIN_ADDR{RAW_IP4_MIN_ADDR}; + static constexpr in_addr_t RAW_IP4_MAX_ADDR = ~0; + static constexpr IpAddr IP4_MAX_ADDR{RAW_IP4_MAX_ADDR}; + + static constexpr in6_addr RAW_IP6_MIN_ADDR = {{{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}}; + static constexpr IpAddr IP6_MIN_ADDR{RAW_IP6_MIN_ADDR}; + static constexpr in6_addr RAW_IP6_MAX_ADDR = {{{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}}; + static constexpr IpAddr IP6_MAX_ADDR{RAW_IP6_MAX_ADDR}; + /** Public API for intervals in the map. */ class Node : protected ts::detail::RBNode @@ -260,6 +270,8 @@ class IpMap sockaddr const *max ///< Maximum value. ); /// Unmark addresses (overload). + self_type &unmark(IpAddr const &min, IpAddr const &max); + /// Unmark addresses (overload). self_type &unmark(IpEndpoint const *min, IpEndpoint const *max); /// Unmark overload. self_type &unmark(in_addr_t min, ///< Minimum of range to unmark. @@ -399,6 +411,15 @@ IpMap::unmark(IpEndpoint const *min, IpEndpoint const *max) return this->unmark(&min->sa, &max->sa); } +inline IpMap & +IpMap::unmark(IpAddr const &min, IpAddr const &max) +{ + IpEndpoint x, y; + x.assign(min); + y.assign(max); + return this->unmark(&x.sa, &y.sa); +} + inline IpMap & IpMap::fill(IpEndpoint const *min, IpEndpoint const *max, void *data) { diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index 364d6be4278..3de20baff58 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -1150,20 +1150,19 @@ struct IpAddr { /// Default construct (invalid address). IpAddr() : _family(AF_UNSPEC) {} - /// Construct as IPv4 @a addr. - explicit IpAddr(in_addr_t addr ///< Address to assign. - ) - : _family(AF_INET) - { - _addr._ip4 = addr; - } - /// Construct as IPv6 @a addr. - explicit IpAddr(in6_addr const &addr ///< Address to assign. - ) - : _family(AF_INET6) - { - _addr._ip6 = addr; - } + + /** Construct from IPv4 address. + * + * @param addr Source address. + */ + explicit constexpr IpAddr(in_addr_t addr) : _family(AF_INET), _addr(addr) {} + + /** Construct from IPv6 address. + * + * @param addr Source address. + */ + explicit constexpr IpAddr(in6_addr const &addr) : _family(AF_INET6), _addr(addr) {} + /// Construct from @c sockaddr. explicit IpAddr(sockaddr const *addr) { this->assign(addr); } /// Construct from @c sockaddr_in6. @@ -1287,12 +1286,17 @@ struct IpAddr { uint16_t _family; ///< Protocol family. /// Address data. - union { + union Addr { in_addr_t _ip4; ///< IPv4 address storage. in6_addr _ip6; ///< IPv6 address storage. uint8_t _byte[TS_IP6_SIZE]; ///< As raw bytes. uint32_t _u32[TS_IP6_SIZE / (sizeof(uint32_t) / sizeof(uint8_t))]; ///< As 32 bit chunks. uint64_t _u64[TS_IP6_SIZE / (sizeof(uint64_t) / sizeof(uint8_t))]; ///< As 64 bit chunks. + + // This is required by the @c constexpr constructor. + constexpr Addr() : _ip4(0) {} + constexpr Addr(in_addr_t addr) : _ip4(addr) {} + constexpr Addr(in6_addr const &addr) : _ip6(addr) {} } _addr; ///< Pre-constructed invalid instance. From fdf76885b5cf5403cfc008c307d6a2034b248422 Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Fri, 8 Feb 2019 16:17:19 -0800 Subject: [PATCH 306/526] Rewrite URL before all remap plugins run Rewriting the url *before* running all plugins instead of *after* which would guarantee that: - all plugins would get the same TSRemapRequestInfo::reqiestUrl (first plugin in the chain would not be special) - all plugins would treat TSRemapRequestInfo::reqiestUrl the same way consistently as a *remapped* URL which makes the first plugin really not different from the rest - there would be a remapped URL default in case the remap rule had no plugins OR none of the plugins modifed the mapped URL Also turning off url_sig and cookie_remap plugin unit-tests impacted by this not backwards compatible change. --- proxy/http/remap/RemapPlugins.cc | 11 +++++------ .../pluginTest/cookie_remap/collapseslashes.test.py | 1 + .../pluginTest/cookie_remap/connector.test.py | 1 + .../pluginTest/cookie_remap/matrixparams.test.py | 1 + .../pluginTest/cookie_remap/subcookie.test.py | 1 + .../pluginTest/cookie_remap/substitute.test.py | 1 + tests/gold_tests/pluginTest/url_sig/url_sig.test.py | 2 ++ 7 files changed, 12 insertions(+), 6 deletions(-) diff --git a/proxy/http/remap/RemapPlugins.cc b/proxy/http/remap/RemapPlugins.cc index 8900e545bcb..7a3293ba0eb 100644 --- a/proxy/http/remap/RemapPlugins.cc +++ b/proxy/http/remap/RemapPlugins.cc @@ -88,6 +88,11 @@ RemapPlugins::run_single_remap() Debug("url_rewrite", "running single remap rule id %d for the %d%s time", map->map_id, _cur, _cur == 1 ? "st" : _cur == 2 ? "nd" : _cur == 3 ? "rd" : "th"); + if (0 == _cur) { + Debug("url_rewrite", "setting the remapped url by copying from mapping rule"); + url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); + } + // There might not be a plugin if we are a regular non-plugin map rule. In that case, we will fall through // and do the default mapping and then stop. if (plugin) { @@ -110,12 +115,6 @@ RemapPlugins::run_single_remap() Debug("url_rewrite", "completed single remap, attempting another via immediate callback"); zret = false; // not done yet. } - - // If the chain is finished, and the URL hasn't been rewritten, do the rule remap. - if (zret && 0 == _rewritten) { - Debug("url_rewrite", "plugins did not change host, port or path, copying from mapping rule"); - url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); - } } return zret; } diff --git a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py index 7f6db2abb44..77d823a7649 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py @@ -27,6 +27,7 @@ ) Test.ContinueOnFail = True Test.testName = "cookie_remap: plugin collapses consecutive slashes" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) # Define default ATS ts = Test.MakeATSProcess("ts") diff --git a/tests/gold_tests/pluginTest/cookie_remap/connector.test.py b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py index 2f7c18ee219..24edae3047d 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/connector.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py @@ -27,6 +27,7 @@ ) Test.ContinueOnFail = True Test.testName = "cookie_remap: test connector" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) # Define default ATS ts = Test.MakeATSProcess("ts") diff --git a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py index 68529ba9fcb..45ecb7b491e 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py @@ -27,6 +27,7 @@ ) Test.ContinueOnFail = True Test.testName = "cookie_remap: Tests when matrix parameters are present" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) # Define default ATS ts = Test.MakeATSProcess("ts") diff --git a/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py index f8d3d17fdd6..23384f67f01 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py @@ -27,6 +27,7 @@ ) Test.ContinueOnFail = True Test.testName = "cookie_remap: test connector" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) # Define default ATS ts = Test.MakeATSProcess("ts") diff --git a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py index 273016c160f..bea8400cf10 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py @@ -27,6 +27,7 @@ ) Test.ContinueOnFail = True Test.testName = "cookie_remap: Substitute variables" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) # Define default ATS ts = Test.MakeATSProcess("ts") diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py index 19af447f0d6..3d0dade02cd 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py @@ -25,6 +25,8 @@ Test.SkipUnless( Condition.HasATSFeature('TS_USE_TLS_ALPN'), ) +Test.ContinueOnFail = True +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) # Skip if plugins not present. Test.SkipUnless(Condition.PluginExists('url_sig.so')) From d47812d988929c71da744c73287e8cf44903fbe7 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Fri, 22 Feb 2019 15:25:49 -0800 Subject: [PATCH 307/526] Fixed use after free when running regression under ASAN --- src/traffic_server/traffic_server.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index eea54b40094..2c09bf18451 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -1376,6 +1376,7 @@ struct RegressionCont : public Continuation { return EVENT_CONT; } + shutdown_event_system = true; fprintf(stderr, "REGRESSION_TEST DONE: %s\n", regression_status_string(res)); ::exit(res == REGRESSION_TEST_PASSED ? 0 : 1); return EVENT_CONT; From 5b86099e66221d2786ba09b5a432f8d73619cf35 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 19 Feb 2019 21:11:43 +0900 Subject: [PATCH 308/526] Cleanup: use string_view for ssl_multicert.config field tags --- iocore/net/SSLUtils.cc | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 653cec49cbf..c2e7061494b 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -21,6 +21,7 @@ #include "P_SSLUtils.h" +#include "tscpp/util/TextView.h" #include "tscore/ink_platform.h" #include "tscore/SimpleTokenizer.h" #include "tscore/I_Layout.h" @@ -70,17 +71,19 @@ #include #endif +using namespace std::literals; + // ssl_multicert.config field names: -#define SSL_IP_TAG "dest_ip" -#define SSL_CERT_TAG "ssl_cert_name" -#define SSL_PRIVATE_KEY_TAG "ssl_key_name" -#define SSL_CA_TAG "ssl_ca_name" -#define SSL_ACTION_TAG "action" -#define SSL_ACTION_TUNNEL_TAG "tunnel" -#define SSL_SESSION_TICKET_ENABLED "ssl_ticket_enabled" -#define SSL_KEY_DIALOG "ssl_key_dialog" -#define SSL_SERVERNAME "dest_fqdn" -#define SSL_CERT_SEPARATE_DELIM ',' +static constexpr std::string_view SSL_IP_TAG("dest_ip"sv); +static constexpr std::string_view SSL_CERT_TAG("ssl_cert_name"sv); +static constexpr std::string_view SSL_PRIVATE_KEY_TAG("ssl_key_name"sv); +static constexpr std::string_view SSL_CA_TAG("ssl_ca_name"sv); +static constexpr std::string_view SSL_ACTION_TAG("action"sv); +static constexpr std::string_view SSL_ACTION_TUNNEL_TAG("tunnel"sv); +static constexpr std::string_view SSL_SESSION_TICKET_ENABLED("ssl_ticket_enabled"sv); +static constexpr std::string_view SSL_KEY_DIALOG("ssl_key_dialog"sv); +static constexpr std::string_view SSL_SERVERNAME("dest_fqdn"sv); +static constexpr char SSL_CERT_SEPARATE_DELIM = ','; // openssl version must be 0.9.4 or greater #if (OPENSSL_VERSION_NUMBER < 0x00090400L) @@ -1364,7 +1367,7 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } else if (strcmp(sslMultCertSettings->dialog, "builtin") == 0) { passwd_cb = ssl_private_key_passphrase_callback_builtin; } else { // unknown config - SSLError("unknown " SSL_KEY_DIALOG " configuration value '%s'", (const char *)sslMultCertSettings->dialog); + SSLError("unknown %s configuration value '%s'", SSL_KEY_DIALOG.data(), (const char *)sslMultCertSettings->dialog); memset(static_cast(&ud), 0, sizeof(ud)); goto fail; } @@ -1783,7 +1786,7 @@ ssl_extract_certificate(const matcher_line *line_info, ssl_user_config &sslMultC if (strcasecmp(SSL_ACTION_TUNNEL_TAG, value) == 0) { sslMultCertSettings.opt = SSLCertContext::OPT_TUNNEL; } else { - Error("Unrecognized action for " SSL_ACTION_TAG); + Error("Unrecognized action for %s", SSL_ACTION_TAG.data()); return false; } } From 3b611dcefe2f0362db7ecbcd233ea828a74faa3b Mon Sep 17 00:00:00 2001 From: scw00 Date: Thu, 21 Feb 2019 18:40:57 +0800 Subject: [PATCH 309/526] Fixed compiler error with std::string_view --- mgmt/MgmtDefs.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mgmt/MgmtDefs.h b/mgmt/MgmtDefs.h index 69328682fe0..868ec25b86d 100644 --- a/mgmt/MgmtDefs.h +++ b/mgmt/MgmtDefs.h @@ -31,6 +31,7 @@ #include "tscore/ink_defs.h" #include "tscpp/util/MemSpan.h" +#include "tscpp/util/TextView.h" typedef int64_t MgmtIntCounter; typedef int64_t MgmtInt; @@ -142,4 +143,4 @@ inline MgmtConverter::MgmtConverter(MgmtInt (*_load_int)(void *), void (*_store_ { } -constexpr std::string_view LM_CONNECTION_SERVER{"processerver.sock"}; +constexpr ts::TextView LM_CONNECTION_SERVER{"processerver.sock"}; From 763cd6d0140f53eb149e944c0a9e7198fcc80624 Mon Sep 17 00:00:00 2001 From: scw00 Date: Sat, 23 Feb 2019 09:53:27 +0800 Subject: [PATCH 310/526] Assertion when cache_config_target_fragment_size is large than MAX_FRAG_SIZE --- iocore/cache/Cache.cc | 19 +++++++++++++++++-- iocore/cache/CacheWrite.cc | 4 +++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index e1416fa7fe4..87b6f1898a8 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -3154,6 +3154,18 @@ register_cache_stats(RecRawStatBlock *rsb, const char *prefix) REG_INT("span.online", cache_span_online_stat); } +int +FragmentSizeUpdateCb(const char * /* name ATS_UNUSED */, RecDataT /* data_type ATS_UNUSED */, RecData data, void *cookie) +{ + if (sizeof(Doc) >= static_cast(data.rec_int) || static_cast(data.rec_int) - sizeof(Doc) > MAX_FRAG_SIZE) { + Warning("The fragments size exceed the limitation, ignore: %" PRId64 ", %d", data.rec_int, cache_config_target_fragment_size); + return 0; + } + + cache_config_target_fragment_size = data.rec_int; + return 0; +} + void ink_cache_init(ts::ModuleVersion v) { @@ -3206,9 +3218,12 @@ ink_cache_init(ts::ModuleVersion v) Debug("cache_init", "proxy.config.cache.hit_evacuate_size_limit = %d", cache_config_hit_evacuate_size_limit); REC_EstablishStaticConfigInt32(cache_config_force_sector_size, "proxy.config.cache.force_sector_size"); - REC_EstablishStaticConfigInt32(cache_config_target_fragment_size, "proxy.config.cache.target_fragment_size"); - if (cache_config_target_fragment_size == 0) { + ink_assert(REC_RegisterConfigUpdateFunc("proxy.config.cache.target_fragment_size", FragmentSizeUpdateCb, nullptr) != + REC_ERR_FAIL); + REC_ReadConfigInt32(cache_config_target_fragment_size, "proxy.config.cache.target_fragment_size"); + + if (cache_config_target_fragment_size == 0 || cache_config_target_fragment_size - sizeof(Doc) > MAX_FRAG_SIZE) { cache_config_target_fragment_size = DEFAULT_TARGET_FRAGMENT_SIZE; } diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index f81cf8da4e7..178ec08ddb7 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -1369,7 +1369,9 @@ CacheVC::openWriteWriteDone(int event, Event *e) static inline int target_fragment_size() { - return cache_config_target_fragment_size - sizeof(Doc); + uint64_t value = cache_config_target_fragment_size - sizeof(Doc); + ink_release_assert(value <= MAX_FRAG_SIZE); + return value; } int From e6231b3faecafffa84f72bd3a82bcaf974b86a31 Mon Sep 17 00:00:00 2001 From: dyrock Date: Thu, 21 Feb 2019 17:06:41 -0600 Subject: [PATCH 311/526] Changed client context mapping to 2-level. Tested against a TLS server to verify client context is created and found. --- iocore/net/P_SSLConfig.h | 10 +++++--- iocore/net/SSLConfig.cc | 54 +++++++++++++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index df178660511..14b7b9ff1f1 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -120,10 +120,12 @@ struct SSLConfigParams : public ConfigInfo { SSL_CTX *client_ctx; - // Making this mutable since this is a updatable - // cache on an otherwise immutable config object - // The ctx_map owns the client SSL_CTX objects and is responseible for cleaning them up - mutable std::unordered_map ctx_map; + // Client contexts are held by 2-level map: + // The first level maps from CA bundle file&path to next level map; + // The second level maps from cert&key to actual SSL_CTX; + // The second level map owns the client SSL_CTX objects and is responsible for cleaning them up + using CTX_MAP = std::unordered_map; + mutable std::unordered_map top_level_ctx_map; mutable ink_mutex ctxMapLock; SSL_CTX *getClientSSL_CTX(void) const; diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 1fd19557979..26a65a41303 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -670,12 +670,26 @@ SSL_CTX * SSLConfigParams::getCTX(const char *client_cert, const char *key_file, const char *ca_bundle_file, const char *ca_bundle_path) const { SSL_CTX *client_ctx = nullptr; - std::string key; - ts::bwprint(key, "{}:{}:{}:{}", client_cert, key_file, ca_bundle_file, ca_bundle_path); + CTX_MAP *ctx_map = nullptr; + std::string top_level_key, ctx_key; + ts::bwprint(top_level_key, "{}:{}", ca_bundle_file, ca_bundle_path); + ts::bwprint(ctx_key, "{}:{}", client_cert, key_file, ca_bundle_file, ca_bundle_path); ink_mutex_acquire(&ctxMapLock); - auto iter = ctx_map.find(key); - if (iter != ctx_map.end()) { + // Do first level searching and create new CTX_MAP as second level if not exists. + auto top_iter = top_level_ctx_map.find(top_level_key); + if (top_iter != top_level_ctx_map.end()) { + if (top_iter->second == nullptr) { + top_iter->second = new CTX_MAP; + } + ctx_map = top_iter->second; + } else { + ctx_map = new CTX_MAP; + top_level_ctx_map.insert(std::make_pair(top_level_key, ctx_map)); + } + // Do second level searching and return client ctx if found + auto iter = ctx_map->find(ctx_key); + if (iter != ctx_map->end()) { client_ctx = iter->second; ink_mutex_release(&ctxMapLock); return client_ctx; @@ -717,12 +731,22 @@ SSLConfigParams::getCTX(const char *client_cert, const char *key_file, const cha } ink_mutex_acquire(&ctxMapLock); - iter = ctx_map.find(key); - if (iter != ctx_map.end()) { + top_iter = top_level_ctx_map.find(top_level_key); + if (top_iter != top_level_ctx_map.end()) { + if (top_iter->second == nullptr) { + top_iter->second = new CTX_MAP; + } + ctx_map = top_iter->second; + } else { + ctx_map = new CTX_MAP; + top_level_ctx_map.insert(std::make_pair(top_level_key, ctx_map)); + } + iter = ctx_map->find(ctx_key); + if (iter != ctx_map->end()) { SSL_CTX_free(client_ctx); client_ctx = iter->second; } else { - ctx_map.insert(std::make_pair(key, client_ctx)); + ctx_map->insert(std::make_pair(ctx_key, client_ctx)); } ink_mutex_release(&ctxMapLock); return client_ctx; @@ -738,11 +762,17 @@ void SSLConfigParams::cleanupCTXTable() { ink_mutex_acquire(&ctxMapLock); - auto iter = ctx_map.begin(); - while (iter != ctx_map.end()) { - SSL_CTX_free(iter->second); - ++iter; + CTX_MAP *ctx_map = nullptr; + for (auto &top_pair : top_level_ctx_map) { + ctx_map = top_pair.second; + if (ctx_map) { + for (auto &pair : (*ctx_map)) { + SSL_CTX_free(pair.second); + } + ctx_map->clear(); + delete ctx_map; + } } - ctx_map.clear(); + top_level_ctx_map.clear(); ink_mutex_release(&ctxMapLock); } From e3ee3e4aedac2e7af77a3f6af0fc2eb7091e310b Mon Sep 17 00:00:00 2001 From: dyrock Date: Mon, 14 Jan 2019 16:40:08 -0600 Subject: [PATCH 312/526] JA3 fingerprint and documentation --- configure.ac | 20 + doc/admin-guide/plugins/index.en.rst | 4 + .../plugins/ja3_fingerprint.en.rst | 49 ++ plugins/Makefile.am | 4 + .../experimental/ja3_fingerprint/Makefile.inc | 19 + plugins/experimental/ja3_fingerprint/README | 26 + .../ja3_fingerprint/ja3_fingerprint.cc | 493 ++++++++++++++++++ 7 files changed, 615 insertions(+) create mode 100644 doc/admin-guide/plugins/ja3_fingerprint.en.rst create mode 100644 plugins/experimental/ja3_fingerprint/Makefile.inc create mode 100644 plugins/experimental/ja3_fingerprint/README create mode 100644 plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc diff --git a/configure.ac b/configure.ac index bec8ef593bd..5ecc6b6b59d 100644 --- a/configure.ac +++ b/configure.ac @@ -1247,6 +1247,26 @@ AC_CHECK_FUNC([EVP_MD_CTX_free], [], LIBS="$saved_LIBS" +# +# Check OpenSSL version for JA3 Fingerprint +# +AC_MSG_CHECKING([for JA3 compatible OpenSSL version]) +AC_EGREP_CPP(yes, [ + #include + #if (OPENSSL_VERSION_NUMBER < 0x010100000L) + yes + #elif (OPENSSL_VERSION_NUMBER >= 0x010101000L) + yes + #endif + ], [ + AC_MSG_RESULT(yes) + AS_IF([test "x${enable_experimental_plugins}" = "xyes"], [ + enable_ja3_plugin=yes + ]) + ], [AC_MSG_RESULT(no)]) + +AM_CONDITIONAL([BUILD_JA3_PLUGIN], [test "x${enable_ja3_plugin}" = "xyes"]) + # # Check for zlib presence and usability TS_CHECK_ZLIB diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index d27e931560b..1df7cadf26b 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -153,6 +153,7 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Header Frequency HIPES Hook Trace + JA3 Fingerprint Memcache Metalink Money Trace @@ -195,6 +196,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`HIPES ` Adds support for HTTP Pipes. +:doc:`JA3 Fingerprint ` + Calculates JA3 Fingerprints for incoming SSL traffic. + :doc:`Memcache ` Implements the memcache protocol for cache contents. diff --git a/doc/admin-guide/plugins/ja3_fingerprint.en.rst b/doc/admin-guide/plugins/ja3_fingerprint.en.rst new file mode 100644 index 00000000000..9be99f55d94 --- /dev/null +++ b/doc/admin-guide/plugins/ja3_fingerprint.en.rst @@ -0,0 +1,49 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. include:: ../../common.defs + +.. _admin-plugins-ja3_fingerprint: + + +JA3 Fingerprint Plugin +******************* + +Description +=========== + +``JA3 Fingerprint`` calculates JA3 fingerprints for incoming SSL traffic. "JA3 is a method for creating SSL/TLS client fingerprints" by concatenating values in ClientHello packet and MD5 hash the result to produce a 32 character fingerprint. Malwares tend to use the same encryption code/client, which makes it an effective way to detect malicious clients. More info about ja3 is available: https://github.com/salesforce/ja3. + +The calculated JA3 fingerprints are then appended to upstream request (to be processed at upstream) and/or logged locally (depending on the config). + +Plugin Configuration +==================== +.. program:: ja3_fingerprint.so + +* ``ja3_fingerprint`` can be used as a global/remap plugin and is configured via :file:`plugin.config` or :file:`remap.config`. + .. option:: --ja3raw + + (`optional`, default:unused) - enables raw fingerprints header. With this option, the plugin will append additional header `X-JA3-Raw` to proxy request. + + .. option:: --ja3log + + (`optional`, default:unused) - enables local logging. With this option, the plugin will log JA3 info to :file:`ja3_fingerprint.log` in the standard logging directory. The format is: [time] [client IP] [JA3 string] [JA3 hash] + +Requirement +============= +Won't compile against OpenSSL 1.1.0 due to API changes and opaque structures. diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 9eee0dd61e4..c00a4fb10bf 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -84,6 +84,10 @@ include experimental/tls_bridge/Makefile.inc include experimental/url_sig/Makefile.inc include experimental/prefetch/Makefile.inc +if BUILD_JA3_PLUGIN +include experimental/ja3_fingerprint/Makefile.inc +endif + if BUILD_URI_SIGNING_PLUGIN include experimental/uri_signing/Makefile.inc endif diff --git a/plugins/experimental/ja3_fingerprint/Makefile.inc b/plugins/experimental/ja3_fingerprint/Makefile.inc new file mode 100644 index 00000000000..b29be9f7d59 --- /dev/null +++ b/plugins/experimental/ja3_fingerprint/Makefile.inc @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/ja3_fingerprint/ja3_fingerprint.la + +experimental_ja3_fingerprint_ja3_fingerprint_la_SOURCES = experimental/ja3_fingerprint/ja3_fingerprint.cc diff --git a/plugins/experimental/ja3_fingerprint/README b/plugins/experimental/ja3_fingerprint/README new file mode 100644 index 00000000000..fdb85c78265 --- /dev/null +++ b/plugins/experimental/ja3_fingerprint/README @@ -0,0 +1,26 @@ +ATS (Apache Traffic Server) JA3 Fingerprint Plugin + +General description +-------------------- +1. JA3 +This plugin looks at all incoming SSL/TLS clientHello and calculates JA3 fingerprint for each client. +It then performs +1) logging JA3 string and its MD5 hash to `ja3_fingerprint.log` in the standard logging directory; +2) appending `X-JA3-Sig` and/or `X-JA3-Raw` headers to upstream request (depending on the config) + +The log file format is as follows: + +[time] [client IP] [JA3 string] [MD5 Hash] + +2. plugin.config +In plugin.config, supply name of the plugin and options +Example: ja3_fingerprint.so --ja3raw --ja3log +Add flag --ja3raw if `X-JA3-Raw` is desired other than only `X-JA3-Sig`. +Add flag --ja3log if local logging in standard logging directory is desired. + +3. remap.config +This plugin can also be used as a remap plugin. For each remap rule, add plugin and parameter field. For example: +map http://from.com http://to.com @plugin=ja3_fingerprint.so [@pparam=--ja3raw] [@pparam=--ja3log] + +4. Requirement +Won't compile against OpenSSL 1.1.0 due to APIs and opaque structures. diff --git a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc new file mode 100644 index 00000000000..16589c4b1b6 --- /dev/null +++ b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc @@ -0,0 +1,493 @@ +/** @ja3_fingerprint.cc + Plugin JA3 Fingerprint calculates JA3 signatures for incoming SSL traffic. + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ts/ts.h" +#include "ts/remap.h" + +#ifdef OPENSSL_NO_SSL_INTERN +#undef OPENSSL_NO_SSL_INTERN +#endif + +#include +#include +#include + +// Get 16bit big endian order and update pointer +#define n2s(c, s) ((s = (((unsigned int)(c[0])) << 8) | (((unsigned int)(c[1])))), c += 2) + +const char *PLUGIN_NAME = "ja3_fingerprint"; +static TSTextLogObject pluginlog; +static int ja3_idx = -1; +static int enable_raw = 0; +static int enable_log = 0; + +// GREASE table as in ja3 +static const std::unordered_set GREASE_table = {0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a, + 0x8a8a, 0x9a9a, 0xaaaa, 0xbaba, 0xcaca, 0xdada, 0xeaea, 0xfafa}; + +struct ja3_data { + std::string ja3_string; + char md5String[33]; + char ip_addr[INET6_ADDRSTRLEN]; +}; + +struct ja3_remap_info { + int raw = false; + int log = false; + TSCont handler = nullptr; + + ~ja3_remap_info() + { + if (handler) { + TSContDestroy(handler); + handler = nullptr; + } + } +}; + +static int +custom_get_ja3_prefixed(int unit, const unsigned char *&data, int len, std::string &result) +{ + int cnt, tmp; + bool first = true; + // Extract each entry and append to result string + for (cnt = 0; cnt < len; cnt += unit) { + if (unit == 1) { + tmp = *(data++); + } else { + n2s(data, tmp); + } + + // Check for GREASE for 16-bit values, append only if non-GREASE + if (unit != 2 || GREASE_table.find(tmp) == GREASE_table.end()) { + if (!first) { + result += '-'; + } + first = false; + result += std::to_string(tmp); + } + } + return 0; +} + +char * +getIP(sockaddr const *s_sockaddr, char res[INET6_ADDRSTRLEN]) +{ + res[0] = '\0'; + + if (s_sockaddr == nullptr) { + return nullptr; + } + + switch (s_sockaddr->sa_family) { + case AF_INET: { + const struct sockaddr_in *s_sockaddr_in = reinterpret_cast(s_sockaddr); + inet_ntop(AF_INET, &s_sockaddr_in->sin_addr, res, INET_ADDRSTRLEN); + } break; + case AF_INET6: { + const struct sockaddr_in6 *s_sockaddr_in6 = reinterpret_cast(s_sockaddr); + inet_ntop(AF_INET6, &s_sockaddr_in6->sin6_addr, res, INET6_ADDRSTRLEN); + } break; + default: + return nullptr; + } + + return res[0] ? res : nullptr; +} + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +// Parsing clientHello to get ja3 string +// No error checking or handling because this should be called after openSSL has done all checks and +// returned successfully +static std::string +custom_get_ja3(SSL *s) +{ + TSDebug(PLUGIN_NAME, "Entering custom_get_ja3()..."); + std::string ja3; + const unsigned char *p, *d; + int i, j, len; + + // ClientHello buf and len + d = p = (unsigned char *)s->init_msg; + long n = s->init_num; + + // Get version + int version = (((int)p[0]) << 8) | (int)p[1]; + ja3 += std::to_string(version) + ','; + p += 2; + + // Skip client random + p += SSL3_RANDOM_SIZE; + + // Skip session id + j = *(p++); + p += j; + + // No DTLS handling + + // Get cipher suites + n2s(p, len); + custom_get_ja3_prefixed(2, p, len, ja3); + ja3 += ','; + + // Skip compression + i = *(p++); + p += i; + + // Get extensions + uint16_t type; + int size; + std::string eclist, ecpflist; + + // Skip length blob + p += 2; + bool first = true; + while (p < d + n) { + // Each extension blob is comprised of [2bytes] type + [2bytes] size + [size bytes] data + n2s(p, type); + n2s(p, size); + + // Elliptic curve points + if (type == 0x0a) { + const unsigned char *sdata = p; + n2s(sdata, len); + custom_get_ja3_prefixed(2, sdata, len, eclist); + } + // Elliptic curve point formats + else if (type == 0x0b) { + const unsigned char *sdata = p; + len = *(sdata++); + custom_get_ja3_prefixed(1, sdata, len, ecpflist); + } + + // Update pointer + p += size; + + // Update ja3 string with valid extension type + if (GREASE_table.find(type) == GREASE_table.end()) { + if (!first) { + ja3 += '-'; + } + first = false; + ja3 += std::to_string(type); + } + } + + // Append eclist and ecpflist + ja3 += "," + eclist + "," + ecpflist; + TSDebug(PLUGIN_NAME, "ja3 string: %s", ja3.c_str()); + return ja3; +} +#elif OPENSSL_VERSION_NUMBER >= 0x10101000L +static std::string +custom_get_ja3(SSL *s) +{ + std::string ja3; + size_t len; + const unsigned char *p; + + // Get version + unsigned int version = SSL_client_hello_get0_legacy_version(s); + ja3 += std::to_string(version) + ','; + + // Get cipher suites + len = SSL_client_hello_get0_ciphers(s, &p); + custom_get_ja3_prefixed(2, p, len, ja3); + + // Get extenstions + int *o; + std::string eclist, ecpflist; + if (SSL_client_hello_get0_ext(s, 0x0a, &p, &len) == 1) { + custom_get_ja3_prefixed(2, p, len, eclist); + } + if (SSL_client_hello_get0_ext(s, 0x0b, &p, &len) == 1) { + custom_get_ja3_prefixed(1, p, len, ecpflist); + } + if (SSL_client_hello_get1_extensions_present(s, &o, &len) == 1) { + bool first = true; + for (size_t i = 0; i < len; i++) { + int type = o[i]; + if (GREASE_table.find(type) == GREASE_table.end()) { + if (!first) { + ja3 += '-'; + } + first = false; + ja3 += std::to_string(type); + } + } + } + ja3 += "," + eclist + "," + ecpflist; + return ja3; +} +#else +#error OpenSSL cannot be 1.1.0 +#endif + +static int +client_hello_ja3_handler(TSCont contp, TSEvent event, void *edata) +{ + TSVConn ssl_vc = reinterpret_cast(edata); + switch (event) { + case TS_EVENT_SSL_CLIENT_HELLO: { + TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + + // OpenSSL handle + SSL *ssl = reinterpret_cast(sslobj); + + ja3_data *data = new ja3_data; + data->ja3_string.append(custom_get_ja3(ssl)); + getIP(TSNetVConnRemoteAddrGet(ssl_vc), data->ip_addr); + + TSVConnArgSet(ssl_vc, ja3_idx, static_cast(data)); + TSDebug(PLUGIN_NAME, "client_hello_ja3_handler(): JA3: %s", data->ja3_string.c_str()); + + // MD5 hash + unsigned char digest[MD5_DIGEST_LENGTH]; + MD5((unsigned char *)data->ja3_string.c_str(), data->ja3_string.length(), digest); + + for (int i = 0; i < 16; i++) { + sprintf(&(data->md5String[i * 2]), "%02x", (unsigned int)digest[i]); + } + TSDebug(PLUGIN_NAME, "Fingerprint: %s", data->md5String); + break; + } + case TS_EVENT_VCONN_CLOSE: { + // Clean up + ja3_data *data = static_cast(TSVConnArgGet(ssl_vc, ja3_idx)); + + if (data == nullptr) { + TSDebug(PLUGIN_NAME, "client_hello_ja3_handler(): Failed to retrieve ja3 data at VCONN_CLOSE."); + return TS_ERROR; + } + + TSVConnArgSet(ssl_vc, ja3_idx, nullptr); + + delete data; + break; + } + default: { + TSDebug(PLUGIN_NAME, "client_hello_ja3_handler(): Unexpected event."); + break; + } + } + TSVConnReenable(ssl_vc); + return TS_SUCCESS; +} + +static int +req_hdr_ja3_handler(TSCont contp, TSEvent event, void *edata) +{ + TSHttpTxn txnp = nullptr; + TSHttpSsn ssnp = nullptr; + TSVConn vconn = nullptr; + if ((txnp = static_cast(edata)) == nullptr || (ssnp = TSHttpTxnSsnGet(txnp)) == nullptr || + (vconn = TSHttpSsnClientVConnGet(ssnp)) == nullptr) { + TSDebug(PLUGIN_NAME, "req_hdr_ja3_handler(): Failure to retrieve txn/ssn/vconn object."); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return TS_SUCCESS; + } + + // Retrieve ja3_data from vconn args + ja3_data *data = static_cast(TSVConnArgGet(vconn, ja3_idx)); + if (data) { + // Decide global or remap + ja3_remap_info *info = static_cast(TSContDataGet(contp)); + bool raw_flag = info ? info->raw : enable_raw; + bool log_flag = info ? info->log : enable_log; + TSDebug(PLUGIN_NAME, "req_hdr_ja3_handler(): Found ja3 string."); + + // Get handle to headers + TSMBuffer bufp; + TSMLoc hdr_loc, field_loc; + TSAssert(TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &bufp, &hdr_loc)); + + // Add JA3 md5 fingerprints + TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "X-JA3-Sig", 9, &field_loc); + TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, data->md5String, 32); + TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc); + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + + // If raw string is configured, added JA3 raw string to header as well + if (raw_flag) { + TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "X-JA3-Raw", 9, &field_loc); + TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, data->ja3_string.data(), data->ja3_string.size()); + TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc); + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); + + // Write to logfile + if (log_flag) { + TSTextLogObjectWrite(pluginlog, "Client IP: %s\tJA3: %.*s\tMD5: %.*s", data->ip_addr, + static_cast(data->ja3_string.size()), data->ja3_string.data(), 32, data->md5String); + } + } else { + TSDebug(PLUGIN_NAME, "req_hdr_ja3_handler(): ja3 data not set. Not SSL vconn. Abort."); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return TS_SUCCESS; +} + +static bool +read_config_option(int argc, const char *argv[], int &raw, int &log) +{ + static const struct option longopts[] = {{"ja3raw", no_argument, &raw, 1}, {"ja3log", no_argument, &log, 1}, {0, 0, 0, 0}}; + + int opt = 0; + while ((opt = getopt_long(argc, (char *const *)argv, "", longopts, nullptr)) >= 0) { + switch (opt) { + case '?': + TSDebug(PLUGIN_NAME, "read_config_option(): Unrecognized command arguments."); + case 0: + case -1: + break; + default: + TSDebug(PLUGIN_NAME, "read_config_option(): Unexpected options error."); + return false; + } + } + + TSDebug(PLUGIN_NAME, "read_config_option(): ja3 raw is %s", (raw == 1) ? "enabled" : "disabled"); + TSDebug(PLUGIN_NAME, "read_config_option(): ja3 logging is %s", (log == 1) ? "enabled" : "disabled"); + return true; +} + +void +TSPluginInit(int argc, const char *argv[]) +{ + TSDebug(PLUGIN_NAME, "Initializing plugin"); + + TSPluginRegistrationInfo info; + + info.plugin_name = PLUGIN_NAME; + info.vendor_name = "Oath"; + info.support_email = "zeyuany@oath.com"; + + // Options + if (!read_config_option(argc, argv, enable_raw, enable_log)) { + return; + } + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[%s] Unable to initialize plugin. Failed to register.", PLUGIN_NAME); + } else { + if (enable_log && !pluginlog) { + TSAssert(TS_SUCCESS == TSTextLogObjectCreate(PLUGIN_NAME, TS_LOG_MODE_ADD_TIMESTAMP, &pluginlog)); + TSDebug(PLUGIN_NAME, "log object created successfully"); + } + // SNI handler + TSCont ja3_cont = TSContCreate(client_hello_ja3_handler, nullptr); + TSVConnArgIndexReserve(PLUGIN_NAME, "used to pass ja3", &ja3_idx); + TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, ja3_cont); + TSHttpHookAdd(TS_VCONN_CLOSE_HOOK, ja3_cont); + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, TSContCreate(req_hdr_ja3_handler, nullptr)); + } + + return; +} + +// Remap Part +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) +{ + TSDebug(PLUGIN_NAME, "JA3 Remap Plugin initializing.."); + + // Check if there is config conflict as both global and remap plugin + if (ja3_idx >= 0) { + TSError(PLUGIN_NAME, "TSRemapInit(): JA3 configured as both global and remap. Check plugin.config."); + return TS_ERROR; + } + + // Set up SNI handler for all TLS connections + TSCont ja3_cont = TSContCreate(client_hello_ja3_handler, nullptr); + TSVConnArgIndexReserve(PLUGIN_NAME, "Used to pass ja3", &ja3_idx); + TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, ja3_cont); + TSHttpHookAdd(TS_VCONN_CLOSE_HOOK, ja3_cont); + + return TS_SUCCESS; +} + +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSED */, int /* errbuf_size ATS_UNUSED */) +{ + TSDebug(PLUGIN_NAME, "New instance for client matching %s to %s", argv[0], argv[1]); + ja3_remap_info *pri = new ja3_remap_info; + + // Parse parameters + if (!read_config_option(argc - 1, const_cast(argv + 1), pri->raw, pri->log)) { + TSDebug(PLUGIN_NAME, "TSRemapNewInstance(): Bad arguments"); + return TS_ERROR; + } + + if (pri->log && !pluginlog) { + TSAssert(TS_SUCCESS == TSTextLogObjectCreate(PLUGIN_NAME, TS_LOG_MODE_ADD_TIMESTAMP, &pluginlog)); + TSDebug(PLUGIN_NAME, "log object created successfully"); + } + + // Create continuation + pri->handler = TSContCreate(req_hdr_ja3_handler, nullptr); + TSContDataSet(pri->handler, pri); + + // Pass to other remap plugin functions + *ih = static_cast(pri); + return TS_SUCCESS; +} + +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) +{ + auto pri = static_cast(ih); + + // On remap, set up handler at send req hook to send JA3 data as header + if (!pri || !rri || !(pri->handler)) { + TSError("[%s] TSRemapDoRemap(): Invalid private data or RRI or handler.", PLUGIN_NAME); + } else { + TSHttpTxnHookAdd(rh, TS_HTTP_SEND_REQUEST_HDR_HOOK, pri->handler); + } + + return TSREMAP_NO_REMAP; +} + +void +TSRemapDeleteInstance(void *ih) +{ + auto pri = static_cast(ih); + if (pri) { + delete pri; + } + ih = nullptr; +} From 321c1e1c7d236aed8f8aa14db2066c58feb4b2e8 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 25 Feb 2019 10:57:10 -0600 Subject: [PATCH 313/526] RemapPluginInfo Refresh. --- proxy/ReverseProxy.cc | 3 +- proxy/ReverseProxy.h | 1 - proxy/http/HttpTransact.h | 4 +- proxy/http/remap/RemapConfig.cc | 64 +++++++++----------- proxy/http/remap/RemapPluginInfo.cc | 92 ++++++++--------------------- proxy/http/remap/RemapPluginInfo.h | 90 ++++++++++++++++------------ proxy/http/remap/RemapPlugins.cc | 8 +-- proxy/http/remap/RemapPlugins.h | 2 +- proxy/http/remap/UrlMapping.cc | 12 ++-- proxy/http/remap/UrlMapping.h | 6 +- 10 files changed, 121 insertions(+), 161 deletions(-) diff --git a/proxy/ReverseProxy.cc b/proxy/ReverseProxy.cc index 8fe9da8f933..0164da968fd 100644 --- a/proxy/ReverseProxy.cc +++ b/proxy/ReverseProxy.cc @@ -43,8 +43,7 @@ // Global Ptrs static Ptr reconfig_mutex; -UrlRewrite *rewrite_table = nullptr; -remap_plugin_info *remap_pi_list = nullptr; // We never reload the remap plugins, just append to 'em. +UrlRewrite *rewrite_table = nullptr; // Tokens for the Callback function #define FILE_CHANGED 0 diff --git a/proxy/ReverseProxy.h b/proxy/ReverseProxy.h index dd205cfdd04..49f2e21ac58 100644 --- a/proxy/ReverseProxy.h +++ b/proxy/ReverseProxy.h @@ -46,7 +46,6 @@ class url_mapping; struct host_hdr_info; extern UrlRewrite *rewrite_table; -extern remap_plugin_info *remap_pi_list; // API Functions int init_reverse_proxy(); diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 5315cdba690..73d5e123889 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -768,8 +768,8 @@ class HttpTransact // INK API/Remap API plugin interface void *remap_plugin_instance = nullptr; void *user_args[TS_HTTP_MAX_USER_ARG]; - remap_plugin_info::_tsremap_os_response *fp_tsremap_os_response = nullptr; - HTTPStatus http_return_code = HTTP_STATUS_NONE; + RemapPluginInfo::OS_Response_F *fp_tsremap_os_response = nullptr; + HTTPStatus http_return_code = HTTP_STATUS_NONE; int api_txn_active_timeout_value = -1; int api_txn_connect_timeout_value = -1; diff --git a/proxy/http/remap/RemapConfig.cc b/proxy/http/remap/RemapConfig.cc index c0eab6effc9..872eb6d11ff 100644 --- a/proxy/http/remap/RemapConfig.cc +++ b/proxy/http/remap/RemapConfig.cc @@ -714,7 +714,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in { TSRemapInterface ri; struct stat stat_buf; - remap_plugin_info *pi; + RemapPluginInfo *pi; char *c, *err, tmpbuf[2048], default_path[PATH_NAME_MAX]; const char *new_argv[1024]; char *parv[1024]; @@ -775,13 +775,9 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in Debug("remap_plugin", "using path %s for plugin", c); - if (!remap_pi_list || (pi = remap_pi_list->find_by_path(c)) == nullptr) { - pi = new remap_plugin_info(c); - if (!remap_pi_list) { - remap_pi_list = pi; - } else { - remap_pi_list->add_to_list(pi); - } + if ((pi = RemapPluginInfo::find_by_path(c)) == nullptr) { + pi = new RemapPluginInfo(ts::file::path(c)); + RemapPluginInfo::add_to_list(pi); Debug("remap_plugin", "New remap plugin info created for \"%s\"", c); { @@ -789,7 +785,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in REC_ReadConfigInteger(elevate_access, "proxy.config.plugin.load_elevated"); ElevateAccess access(elevate_access ? ElevateAccess::FILE_PRIVILEGE : 0); - if ((pi->dlh = dlopen(c, RTLD_NOW)) == nullptr) { + if ((pi->dl_handle = dlopen(c, RTLD_NOW)) == nullptr) { #if defined(freebsd) || defined(openbsd) err = (char *)dlerror(); #else @@ -798,30 +794,28 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in snprintf(errbuf, errbufsize, "Can't load plugin \"%s\" - %s", c, err ? err : "Unknown dlopen() error"); return -4; } - pi->fp_tsremap_init = reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_INIT)); - pi->fp_tsremap_config_reload = - reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_CONFIG_RELOAD)); - pi->fp_tsremap_done = reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_DONE)); - pi->fp_tsremap_new_instance = - reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_NEW_INSTANCE)); - pi->fp_tsremap_delete_instance = - reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_DELETE_INSTANCE)); - pi->fp_tsremap_do_remap = reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_DO_REMAP)); - pi->fp_tsremap_os_response = - reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_OS_RESPONSE)); - - if (!pi->fp_tsremap_init) { + pi->init_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_INIT)); + pi->config_reload_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_CONFIG_RELOAD)); + pi->done_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_DONE)); + pi->new_instance_cb = + reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_NEW_INSTANCE)); + pi->delete_instance_cb = + reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_DELETE_INSTANCE)); + pi->do_remap_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_DO_REMAP)); + pi->os_response_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_OS_RESPONSE)); + + if (!pi->init_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s")", TSREMAP_FUNCNAME_INIT, c); retcode = -10; - } else if (!pi->fp_tsremap_new_instance && pi->fp_tsremap_delete_instance) { + } else if (!pi->new_instance_cb && pi->delete_instance_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s" which is required if "%s" function exists)", TSREMAP_FUNCNAME_NEW_INSTANCE, c, TSREMAP_FUNCNAME_DELETE_INSTANCE); retcode = -11; - } else if (!pi->fp_tsremap_do_remap) { + } else if (!pi->do_remap_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s")", TSREMAP_FUNCNAME_DO_REMAP, c); retcode = -12; - } else if (pi->fp_tsremap_new_instance && !pi->fp_tsremap_delete_instance) { + } else if (pi->new_instance_cb && !pi->delete_instance_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s" which is required if "%s" function exists)", TSREMAP_FUNCNAME_DELETE_INSTANCE, c, TSREMAP_FUNCNAME_NEW_INSTANCE); @@ -831,16 +825,16 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in if (errbuf && errbufsize > 0) { Debug("remap_plugin", "%s", errbuf); } - dlclose(pi->dlh); - pi->dlh = nullptr; + dlclose(pi->dl_handle); + pi->dl_handle = nullptr; return retcode; } memset(&ri, 0, sizeof(ri)); ri.size = sizeof(ri); ri.tsremap_version = TSREMAP_VERSION; - if (pi->fp_tsremap_init(&ri, tmpbuf, sizeof(tmpbuf) - 1) != TS_SUCCESS) { - snprintf(errbuf, errbufsize, "Failed to initialize plugin \"%s\": %s", pi->path, + if (pi->init_cb(&ri, tmpbuf, sizeof(tmpbuf) - 1) != TS_SUCCESS) { + snprintf(errbuf, errbufsize, "Failed to initialize plugin \"%s\": %s", pi->path.c_str(), tmpbuf[0] ? tmpbuf : "Unknown plugin error"); return -5; } @@ -848,7 +842,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in Debug("remap_plugin", "Remap plugin \"%s\" - initialization completed", c); } - if (!pi->dlh) { + if (!pi->dl_handle) { snprintf(errbuf, errbufsize, "Can't load plugin \"%s\"", c); return -6; } @@ -889,7 +883,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in Debug("url_rewrite", "Argument %d: %s", k, argv[k]); } - Debug("url_rewrite", "Viewing parsed plugin parameters for %s: [%d]", pi->path, *plugin_found_at); + Debug("url_rewrite", "Viewing parsed plugin parameters for %s: [%d]", pi->path.c_str(), *plugin_found_at); for (int k = 0; k < parc; k++) { Debug("url_rewrite", "Argument %d: %s", k, parv[k]); } @@ -898,7 +892,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in void *ih = nullptr; TSReturnCode res = TS_SUCCESS; - if (pi->fp_tsremap_new_instance) { + if (pi->new_instance_cb) { #if (!defined(kfreebsd) && defined(freebsd)) || defined(darwin) optreset = 1; #endif @@ -910,7 +904,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in opterr = 0; optarg = nullptr; - res = pi->fp_tsremap_new_instance(parc, parv, &ih, tmpbuf, sizeof(tmpbuf) - 1); + res = pi->new_instance_cb(parc, parv, &ih, tmpbuf, sizeof(tmpbuf) - 1); } Debug("remap_plugin", "done creating new plugin instance"); @@ -1426,9 +1420,7 @@ remap_parse_config(const char *path, UrlRewrite *rewrite) // If this happens to be a config reload, the list of loaded remap plugins is non-empty, and we // can signal all these plugins that a reload has begun. - if (remap_pi_list) { - remap_pi_list->indicate_reload(); - } + RemapPluginInfo::indicate_reload(); bti.rewrite = rewrite; return remap_parse_config_bti(path, &bti); } diff --git a/proxy/http/remap/RemapPluginInfo.cc b/proxy/http/remap/RemapPluginInfo.cc index ad0b06c9e96..db6dfe59498 100644 --- a/proxy/http/remap/RemapPluginInfo.cc +++ b/proxy/http/remap/RemapPluginInfo.cc @@ -1,6 +1,6 @@ /** @file - A brief file description + Information about remap plugin libraries. @section license License @@ -25,102 +25,56 @@ #include "tscore/ink_string.h" #include "tscore/ink_memory.h" -remap_plugin_info::remap_plugin_info(char *_path) - : next(nullptr), - path(nullptr), - path_size(0), - dlh(nullptr), - fp_tsremap_init(nullptr), - fp_tsremap_config_reload(nullptr), - fp_tsremap_done(nullptr), - fp_tsremap_new_instance(nullptr), - fp_tsremap_delete_instance(nullptr), - fp_tsremap_do_remap(nullptr), - fp_tsremap_os_response(nullptr) -{ - // coverity did not see ats_free - // coverity[ctor_dtor_leak] - if (_path && likely((path = ats_strdup(_path)) != nullptr)) { - path_size = strlen(path); - } -} +RemapPluginInfo::List RemapPluginInfo::g_list; -remap_plugin_info::~remap_plugin_info() +RemapPluginInfo::RemapPluginInfo(ts::file::path &&library_path) : path(std::move(library_path)) {} + +RemapPluginInfo::~RemapPluginInfo() { - ats_free(path); - if (dlh) { - dlclose(dlh); + if (dl_handle) { + dlclose(dl_handle); } } // // Find a plugin by path from our linked list // -remap_plugin_info * -remap_plugin_info::find_by_path(char *_path) +RemapPluginInfo * +RemapPluginInfo::find_by_path(std::string_view library_path) { - int _path_size = 0; - remap_plugin_info *pi = nullptr; - - if (likely(_path && (_path_size = strlen(_path)) > 0)) { - for (pi = this; pi; pi = pi->next) { - if (pi->path && pi->path_size == _path_size && !strcmp(pi->path, _path)) { - break; - } - } - } - - return pi; + auto spot = std::find_if(g_list.begin(), g_list.end(), + [&](self_type const &info) -> bool { return 0 == library_path.compare(info.path.view()); }); + return spot == g_list.end() ? nullptr : static_cast(spot); } // // Add a plugin to the linked list // void -remap_plugin_info::add_to_list(remap_plugin_info *pi) +RemapPluginInfo::add_to_list(RemapPluginInfo *pi) { - remap_plugin_info *p = this; - - if (likely(pi)) { - while (p->next) { - p = p->next; - } - - p->next = pi; - pi->next = nullptr; - } + g_list.append(pi); } // -// Remove and delete all plugins from a list, including ourselves. +// Remove and delete all plugins from a list. // void -remap_plugin_info::delete_my_list() +RemapPluginInfo::delete_list() { - remap_plugin_info *p = this->next; - - while (p) { - remap_plugin_info *tmp = p; - - p = p->next; - delete tmp; - } - - delete this; + g_list.apply([](self_type *info) -> void { delete info; }); + g_list.clear(); } // // Tell all plugins (that so wish) that remap.config is being reloaded // void -remap_plugin_info::indicate_reload() +RemapPluginInfo::indicate_reload() { - remap_plugin_info *p = this; - - while (p) { - if (p->fp_tsremap_config_reload) { - p->fp_tsremap_config_reload(); + g_list.apply([](self_type *info) -> void { + if (info->config_reload_cb) { + info->config_reload_cb(); } - p = p->next; - } + }); } diff --git a/proxy/http/remap/RemapPluginInfo.h b/proxy/http/remap/RemapPluginInfo.h index a83a48500a9..a9ac0b76420 100644 --- a/proxy/http/remap/RemapPluginInfo.h +++ b/proxy/http/remap/RemapPluginInfo.h @@ -1,6 +1,6 @@ /** @file - A brief file description + Information about remap plugins. @section license License @@ -23,51 +23,67 @@ #pragma once #include "tscore/ink_platform.h" - +#include "tscpp/util/IntrusiveDList.h" +#include "tscore/ts_file.h" #include "ts/apidefs.h" #include "ts/remap.h" -#define TSREMAP_FUNCNAME_INIT "TSRemapInit" -#define TSREMAP_FUNCNAME_CONFIG_RELOAD "TSRemapConfigReload" -#define TSREMAP_FUNCNAME_DONE "TSRemapDone" -#define TSREMAP_FUNCNAME_NEW_INSTANCE "TSRemapNewInstance" -#define TSREMAP_FUNCNAME_DELETE_INSTANCE "TSRemapDeleteInstance" -#define TSREMAP_FUNCNAME_DO_REMAP "TSRemapDoRemap" -#define TSREMAP_FUNCNAME_OS_RESPONSE "TSRemapOSResponse" +static constexpr const char *const TSREMAP_FUNCNAME_INIT = "TSRemapInit"; +static constexpr const char *const TSREMAP_FUNCNAME_CONFIG_RELOAD = "TSRemapConfigReload"; +static constexpr const char *const TSREMAP_FUNCNAME_DONE = "TSRemapDone"; +static constexpr const char *const TSREMAP_FUNCNAME_NEW_INSTANCE = "TSRemapNewInstance"; +static constexpr const char *const TSREMAP_FUNCNAME_DELETE_INSTANCE = "TSRemapDeleteInstance"; +static constexpr const char *const TSREMAP_FUNCNAME_DO_REMAP = "TSRemapDoRemap"; +static constexpr const char *const TSREMAP_FUNCNAME_OS_RESPONSE = "TSRemapOSResponse"; -/** - * - **/ -class remap_plugin_info +/** Information for a remap plugin. + * This stores the name of the library file and the callback entry points. + */ +class RemapPluginInfo { public: - typedef TSReturnCode _tsremap_init(TSRemapInterface *api_info, char *errbuf, int errbuf_size); - typedef void _tsremap_config_reload(); - typedef void _tsremap_done(void); - typedef TSReturnCode _tsremap_new_instance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size); - typedef void _tsremap_delete_instance(void *); - typedef TSRemapStatus _tsremap_do_remap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri); - typedef void _tsremap_os_response(void *ih, TSHttpTxn rh, int os_response_type); + using self_type = RemapPluginInfo; ///< Self reference type. + + /// Initialization function, called on library load. + using Init_F = TSReturnCode(TSRemapInterface *api_info, char *errbuf, int errbuf_size); + /// Reload function, called to inform the plugin of a configuration reload. + using Reload_F = void(); + /// Called when remapping for a transaction has finished. + using Done_F = void(); + /// Create an rule instance. + using New_Instance_F = TSReturnCode(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size); + /// Delete a rule instance. + using Delete_Instance_F = void(void *ih); + /// Perform remap. + using Do_Remap_F = TSRemapStatus(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri); + /// I have no idea what this is for. + using OS_Response_F = void(void *ih, TSHttpTxn rh, int os_response_type); + + self_type *_next = nullptr; + self_type *_prev = nullptr; + using Linkage = ts::IntrusiveLinkage; + using List = ts::IntrusiveDList; + + ts::file::path path; + void *dl_handle = nullptr; /* "handle" for the dynamic library */ + Init_F *init_cb = nullptr; + Reload_F *config_reload_cb = nullptr; + Done_F *done_cb = nullptr; + New_Instance_F *new_instance_cb = nullptr; + Delete_Instance_F *delete_instance_cb = nullptr; + Do_Remap_F *do_remap_cb = nullptr; + OS_Response_F *os_response_cb = nullptr; - remap_plugin_info *next; - char *path; - int path_size; - void *dlh; /* "handle" for the dynamic library */ - _tsremap_init *fp_tsremap_init; - _tsremap_config_reload *fp_tsremap_config_reload; - _tsremap_done *fp_tsremap_done; - _tsremap_new_instance *fp_tsremap_new_instance; - _tsremap_delete_instance *fp_tsremap_delete_instance; - _tsremap_do_remap *fp_tsremap_do_remap; - _tsremap_os_response *fp_tsremap_os_response; + RemapPluginInfo(ts::file::path &&library_path); + ~RemapPluginInfo(); - remap_plugin_info(char *_path); - ~remap_plugin_info(); + static self_type *find_by_path(std::string_view library_path); + static void add_to_list(self_type *pi); + static void delete_list(); + static void indicate_reload(); - remap_plugin_info *find_by_path(char *_path); - void add_to_list(remap_plugin_info *pi); - void delete_my_list(); - void indicate_reload(); + /// Singleton list of remap plugin info instances. + static List g_list; }; /** diff --git a/proxy/http/remap/RemapPlugins.cc b/proxy/http/remap/RemapPlugins.cc index 7a3293ba0eb..6a664450fb8 100644 --- a/proxy/http/remap/RemapPlugins.cc +++ b/proxy/http/remap/RemapPlugins.cc @@ -26,7 +26,7 @@ ClassAllocator pluginAllocator("RemapPluginsAlloc"); TSRemapStatus -RemapPlugins::run_plugin(remap_plugin_info *plugin) +RemapPlugins::run_plugin(RemapPluginInfo *plugin) { ink_assert(_s); @@ -51,11 +51,11 @@ RemapPlugins::run_plugin(remap_plugin_info *plugin) // Prepare State for the future if (_cur == 0) { - _s->fp_tsremap_os_response = plugin->fp_tsremap_os_response; + _s->fp_tsremap_os_response = plugin->os_response_cb; _s->remap_plugin_instance = ih; } - plugin_retcode = plugin->fp_tsremap_do_remap(ih, reinterpret_cast(_s->state_machine), &rri); + plugin_retcode = plugin->do_remap_cb(ih, reinterpret_cast(_s->state_machine), &rri); // TODO: Deal with negative return codes here if (plugin_retcode < 0) { plugin_retcode = TSREMAP_NO_REMAP; @@ -82,7 +82,7 @@ bool RemapPlugins::run_single_remap() { url_mapping *map = _s->url_map.getMapping(); - remap_plugin_info *plugin = map->get_plugin(_cur); // get the nth plugin in our list of plugins + RemapPluginInfo *plugin = map->get_plugin(_cur); // get the nth plugin in our list of plugins TSRemapStatus plugin_retcode = TSREMAP_NO_REMAP; bool zret = true; // default - last iteration. Debug("url_rewrite", "running single remap rule id %d for the %d%s time", map->map_id, _cur, diff --git a/proxy/http/remap/RemapPlugins.h b/proxy/http/remap/RemapPlugins.h index eafff41c402..421b55ca51e 100644 --- a/proxy/http/remap/RemapPlugins.h +++ b/proxy/http/remap/RemapPlugins.h @@ -68,7 +68,7 @@ struct RemapPlugins : public Continuation { int run_remap(int event, Event *e); bool run_single_remap(); - TSRemapStatus run_plugin(remap_plugin_info *plugin); + TSRemapStatus run_plugin(RemapPluginInfo *plugin); Action action; diff --git a/proxy/http/remap/UrlMapping.cc b/proxy/http/remap/UrlMapping.cc index 5ff1d541772..5979490ec87 100644 --- a/proxy/http/remap/UrlMapping.cc +++ b/proxy/http/remap/UrlMapping.cc @@ -30,7 +30,7 @@ * **/ bool -url_mapping::add_plugin(remap_plugin_info *i, void *ih) +url_mapping::add_plugin(RemapPluginInfo *i, void *ih) { _plugin_list.push_back(i); _instance_data.push_back(ih); @@ -41,7 +41,7 @@ url_mapping::add_plugin(remap_plugin_info *i, void *ih) /** * **/ -remap_plugin_info * +RemapPluginInfo * url_mapping::get_plugin(std::size_t index) const { Debug("url_rewrite", "get_plugin says we have %zu plugins and asking for plugin %zu", plugin_count(), index); @@ -66,11 +66,11 @@ url_mapping::get_instance(std::size_t index) const void url_mapping::delete_instance(unsigned int index) { - void *ih = get_instance(index); - remap_plugin_info *p = get_plugin(index); + void *ih = get_instance(index); + RemapPluginInfo *p = get_plugin(index); - if (ih && p && p->fp_tsremap_delete_instance) { - p->fp_tsremap_delete_instance(ih); + if (ih && p && p->delete_instance_cb) { + p->delete_instance_cb(ih); } } diff --git a/proxy/http/remap/UrlMapping.h b/proxy/http/remap/UrlMapping.h index 269d0143ef7..63f7ca01e7d 100644 --- a/proxy/http/remap/UrlMapping.h +++ b/proxy/http/remap/UrlMapping.h @@ -79,8 +79,8 @@ class url_mapping public: ~url_mapping(); - bool add_plugin(remap_plugin_info *i, void *ih); - remap_plugin_info *get_plugin(std::size_t) const; + bool add_plugin(RemapPluginInfo *i, void *ih); + RemapPluginInfo *get_plugin(std::size_t) const; void *get_instance(std::size_t) const; std::size_t @@ -122,7 +122,7 @@ class url_mapping }; private: - std::vector _plugin_list; + std::vector _plugin_list; std::vector _instance_data; int _rank = 0; }; From 040ad1869e46a3aae905bf259b8d1da0de46a49d Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 19 Feb 2019 16:38:28 -0600 Subject: [PATCH 314/526] ProcessManager: Add string_view message overload. --- mgmt/ProcessManager.cc | 21 +++++++++++++++++++++ mgmt/ProcessManager.h | 15 ++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc index ad0c3f7dee0..e8e6866fc00 100644 --- a/mgmt/ProcessManager.cc +++ b/mgmt/ProcessManager.cc @@ -243,7 +243,28 @@ ProcessManager::signalManager(int msg_id, const char *data_raw, int data_len) mh->msg_id = msg_id; mh->data_len = data_len; memcpy((char *)mh + sizeof(MgmtMessageHdr), data_raw, data_len); + this->signalManager(mh); +} + +void +ProcessManager::signalManager(int msg_id, std::string_view text) +{ + MgmtMessageHdr *mh; + // Make space for the extra null terminator. + mh = static_cast(ats_malloc(sizeof(MgmtMessageHdr) + text.size() + 1)); + auto body = reinterpret_cast(mh + 1); // start of the message body. + mh->msg_id = msg_id; + mh->data_len = text.size() + 1; + memcpy(body, text.data(), text.size()); + body[text.size()] = '\0'; + + this->signalManager(mh); +} + +void +ProcessManager::signalManager(MgmtMessageHdr *mh) +{ ink_release_assert(::enqueue(mgmt_signal_queue, mh)); #if HAVE_EVENTFD diff --git a/mgmt/ProcessManager.h b/mgmt/ProcessManager.h index dcfeb91db15..cc79fbc1436 100644 --- a/mgmt/ProcessManager.h +++ b/mgmt/ProcessManager.h @@ -25,12 +25,14 @@ #pragma once +#include +#include + #include "MgmtUtils.h" #include "BaseManager.h" #include "tscore/ink_sock.h" #include "tscore/ink_apidefs.h" -#include #if HAVE_EVENTFD #include @@ -56,6 +58,17 @@ class ProcessManager : public BaseManager inkcoreapi void signalManager(int msg_id, const char *data_str); inkcoreapi void signalManager(int msg_id, const char *data_raw, int data_len); + /** Send a management message of type @a msg_id with @a text. + * + * @param msg_id ID for the message. + * @param text Content for the message. + * + * A terminating null character is added automatically. + */ + inkcoreapi void signalManager(int msg_id, std::string_view text); + + inkcoreapi void signalManager(MgmtMessageHdr *mh); + void reconfigure(); void initLMConnection(); void handleMgmtMsgFromLM(MgmtMessageHdr *mh); From 6be18d59f9c6169ba42404e8637e35b63fe1dbcf Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 20 Feb 2019 11:54:37 -0600 Subject: [PATCH 315/526] IpMap: Add bwformat. --- include/tscore/IpMap.h | 22 +++++++--- src/tscore/IpMap.cc | 66 +++++++++++++++++++++++------ src/tscore/unit_tests/test_IpMap.cc | 10 +++++ 3 files changed, 79 insertions(+), 19 deletions(-) diff --git a/include/tscore/IpMap.h b/include/tscore/IpMap.h index ae57329ef6e..754f50630b8 100644 --- a/include/tscore/IpMap.h +++ b/include/tscore/IpMap.h @@ -32,6 +32,7 @@ #include "tscore/ink_inet.h" #include "tscpp/util/IntrusiveDList.h" #include "tscore/ink_assert.h" +#include "tscore/BufferWriterForward.h" namespace ts { @@ -194,7 +195,7 @@ class IpMap Node *_node = nullptr; //!< Current node. }; - IpMap(); ///< Default constructor. + IpMap() = default; ///< Default constructor. IpMap(self_type const &that) = delete; IpMap(self_type &&that) noexcept; ~IpMap(); ///< Destructor. @@ -362,9 +363,13 @@ class IpMap */ void validate(); - /// Print all spans. - /// @return This map. - // self& print(); + /** Generate formatted output. + * + * @param w Destination of formatted output. + * @param spec Formatting specification. + * @return @a w + */ + ts::BufferWriter &describe(ts::BufferWriter &w, ts::BWFSpec const &spec) const; protected: /// Force the IPv4 map to exist. @@ -487,4 +492,11 @@ inline IpMap::iterator::pointer IpMap::iterator::operator->() const return _node; } -inline IpMap::IpMap() : _m4(nullptr), _m6(nullptr) {} +namespace ts +{ +inline BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, IpMap const &map) +{ + return map.describe(w, spec); +} +} // namespace ts diff --git a/src/tscore/IpMap.cc b/src/tscore/IpMap.cc index d2583f269f1..4a51962ca68 100644 --- a/src/tscore/IpMap.cc +++ b/src/tscore/IpMap.cc @@ -45,6 +45,7 @@ #include "tscore/IpMap.h" #include "tscore/ink_inet.h" +#include "tscore/BufferWriter.h" namespace ts { @@ -226,9 +227,13 @@ namespace detail /// @return The number of distinct ranges. size_t count() const; - /// Print all spans. - /// @return This map. - self_type &print(); + /** Generate formatted output. + * + * @param w Destination of the output. + * @param spec Format specification. + * @return @a w. + */ + ts::BufferWriter &describe(ts::BufferWriter &w, ts::BWFSpec const &spec) const; // Helper methods. N * @@ -731,18 +736,21 @@ namespace detail } template - IpMapBase & - IpMapBase::print() + ts::BufferWriter & + IpMapBase::describe(ts::BufferWriter &w, ts::BWFSpec const &spec) const { -#if 0 - for ( Node* n = _list.head() ; n ; n = n->_next ) { - std::cout - << n << ": " << n->_min << '-' << n->_max << " [" << n->_data << "] " - << (n->_color == Node::BLACK ? "Black " : "Red ") << "P=" << n->_parent << " L=" << n->_left << " R=" << n->_right - << std::endl; - } -#endif - return *this; + auto pos = w.extent(); + for (auto const &rb_node : _list) { + N const &n{static_cast(rb_node)}; + if (w.extent() > pos) { + w.write(','); + } + w.print("{::a}-{::a}={}", n.min(), n.max(), n._data); + if (std::string_view::npos != spec._ext.find('x')) { + w.print("[{};^{};<{};>{}]", n._color == N::BLACK ? "Black" : "Red", n._parent, n._left, n._right); + } + } + return w; } template IpMapBase::~IpMapBase() { this->clear(); } @@ -1090,6 +1098,14 @@ namespace detail friend class ::IpMap; }; } // namespace detail + +template +inline BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, detail::IpMapBase const &map) +{ + return map.describe(w, spec); +} + } // namespace ts //---------------------------------------------------------------------------- IpMap::IpMap(IpMap::self_type &&that) noexcept : _m4(that._m4), _m6(that._m6) @@ -1290,5 +1306,27 @@ IpMap::iterator::operator--() return *this; } +ts::BufferWriter & +IpMap::describe(ts::BufferWriter &w, ts::BWFSpec const &spec) const +{ + w.write("IPv4 "); + if (_m4) { + bwformat(w, spec, *_m4); + } else { + w.write("N/A"); + } + w.write("\n"); + + w.write("IPv6 "); + if (_m6) { + bwformat(w, spec, *_m6); + } else { + w.write("N/A"); + } + w.write("\n"); + + return w; +} + //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- diff --git a/src/tscore/unit_tests/test_IpMap.cc b/src/tscore/unit_tests/test_IpMap.cc index 20a8231c608..07debf88b06 100644 --- a/src/tscore/unit_tests/test_IpMap.cc +++ b/src/tscore/unit_tests/test_IpMap.cc @@ -24,6 +24,7 @@ #include "tscore/IpMap.h" #include #include +#include std::ostream & operator<<(std::ostream &s, IpEndpoint const &addr) @@ -620,4 +621,13 @@ TEST_CASE("IpMap CloseIntersection", "[libts][ipmap]") CHECK_THAT(m2, IsMarkedWith(c_3_m, markC)); CHECK_THAT(m2, IsMarkedWith(d_2_l, markD)); CHECK(m2.count() == 13); + +#if 0 + ts::LocalBufferWriter<1024> w; + std::cout << "Basic map dump" << std::endl; + std::cout << w.print("{}", m2).view() << std::endl; + w.reset(); + std::cout << "With tree detail" << std::endl; + std::cout << w.print("{::x}", m2).view() << std::endl; +#endif }; From a8e7e03dae4f971676ab65cfe2ba34216a3b9f97 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Mon, 25 Feb 2019 18:03:55 -0800 Subject: [PATCH 316/526] Doc: Clarify how to update the ssl sessions ticket file --- doc/admin-guide/files/records.config.en.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 01e6d110d50..3abea79396d 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3321,7 +3321,8 @@ SSL Termination The filename of the default and global ticket key for SSL sessions. The location is relative to the :ts:cv:`proxy.config.ssl.server.cert.path` directory. One way to generate this would be to run ``head -c48 /dev/urandom | openssl enc -base64 | head -c48 > file.ticket``. Also - note that OpenSSL session tickets are sensitive to the version of the ca-certificates. + note that OpenSSL session tickets are sensitive to the version of the ca-certificates. Once the + file is changed with new tickets, use :option:`traffic_ctl config reload` to begin using them. .. ts:cv:: CONFIG proxy.config.ssl.servername.filename STRING ssl_server_name.yaml From b8d546c131495bde56dfc1ff7449604c6d3385da Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Mon, 25 Feb 2019 11:03:16 +0900 Subject: [PATCH 317/526] Cleanup: Separate TLS SessionTicket from SSLUtils --- iocore/net/Makefile.am | 5 +- iocore/net/SSLCertLookup.cc | 26 ++++---- iocore/net/SSLConfig.cc | 14 ++-- iocore/net/SSLSessionTicket.cc | 117 +++++++++++++++++++++++++++++++++ iocore/net/SSLSessionTicket.h | 42 ++++++++++++ iocore/net/SSLUtils.cc | 107 +++--------------------------- 6 files changed, 186 insertions(+), 125 deletions(-) create mode 100644 iocore/net/SSLSessionTicket.cc create mode 100644 iocore/net/SSLSessionTicket.h diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index 0971d5f70d7..7e27fe5a894 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -134,7 +134,7 @@ libinknet_a_SOURCES = \ ProxyProtocol.cc \ Socks.cc \ SSLCertLookup.cc \ - SSLSessionCache.cc \ + SSLClientUtils.cc \ SSLConfig.cc \ SSLDiags.cc \ SSLInternal.cc \ @@ -145,8 +145,9 @@ libinknet_a_SOURCES = \ SSLNextProtocolSet.cc \ SSLSNIConfig.cc \ SSLStats.cc \ + SSLSessionCache.cc \ + SSLSessionTicket.cc \ SSLUtils.cc \ - SSLClientUtils.cc \ OCSPStapling.cc \ Socks.cc \ UDPIOEvent.cc \ diff --git a/iocore/net/SSLCertLookup.cc b/iocore/net/SSLCertLookup.cc index f94f5150986..2b6e91dd976 100644 --- a/iocore/net/SSLCertLookup.cc +++ b/iocore/net/SSLCertLookup.cc @@ -21,12 +21,9 @@ limitations under the License. */ -#include "tscore/ink_config.h" - #include "P_SSLCertLookup.h" -#include "P_SSLUtils.h" -#include "P_SSLConfig.h" -#include "I_EventSystem.h" + +#include "tscore/ink_config.h" #include "tscore/I_Layout.h" #include "tscore/MatcherUtils.h" #include "tscore/Regex.h" @@ -35,17 +32,16 @@ #include "tscore/bwf_std_format.h" #include "tscore/TestBox.h" +#include "I_EventSystem.h" + +#include "P_SSLUtils.h" +#include "P_SSLConfig.h" +#include "SSLSessionTicket.h" + #include #include #include -// Check if the ticket_key callback #define is available, and if so, enable session tickets. -#ifdef SSL_CTX_set_tlsext_ticket_key_cb - -#define HAVE_OPENSSL_SESSION_TICKETS 1 - -#endif /* SSL_CTX_set_tlsext_ticket_key_cb */ - struct SSLAddressLookupKey { explicit SSLAddressLookupKey(const IpEndpoint &ip) { @@ -201,7 +197,7 @@ ticket_block_create(char *ticket_key_data, int ticket_key_len) ssl_ticket_key_block * ssl_create_ticket_keyblock(const char *ticket_key_path) { -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS ats_scoped_str ticket_key_data; int ticket_key_len; ssl_ticket_key_block *keyblock = nullptr; @@ -226,10 +222,10 @@ ssl_create_ticket_keyblock(const char *ticket_key_path) ticket_block_free(keyblock); return nullptr; -#else /* !HAVE_OPENSSL_SESSION_TICKETS */ +#else /* !TS_HAVE_OPENSSL_SESSION_TICKETS */ (void)ticket_key_path; return nullptr; -#endif /* HAVE_OPENSSL_SESSION_TICKETS */ +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ } void SSLCertContext::release() diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 26a65a41303..d822ce41b4e 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -43,8 +43,9 @@ #include "P_Net.h" #include "P_SSLUtils.h" #include "P_SSLCertLookup.h" -#include "SSLSessionCache.h" #include "SSLDiags.h" +#include "SSLSessionCache.h" +#include "SSLSessionTicket.h" #include "YamlSNIConfig.h" int SSLConfig::configid = 0; @@ -71,13 +72,6 @@ static std::unique_ptr> sslCertUpdate; static std::unique_ptr> sslConfigUpdate; static std::unique_ptr> sslTicketKey; -// Check if the ticket_key callback #define is available, and if so, enable session tickets. -#ifdef SSL_CTX_set_tlsext_ticket_key_cb - -#define HAVE_OPENSSL_SESSION_TICKETS 1 - -#endif /* SSL_CTX_set_tlsext_ticket_key_cb */ - SSLConfigParams::SSLConfigParams() { ink_mutex_init(&ctxMapLock); @@ -563,7 +557,7 @@ SSLTicketParams::LoadTicket() { cleanup(); -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS ssl_ticket_key_block *keyblock = nullptr; SSLConfig::scoped_config params; @@ -611,7 +605,7 @@ void SSLTicketParams::LoadTicketData(char *ticket_data, int ticket_data_len) { cleanup(); -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS if (ticket_data != nullptr && ticket_data_len > 0) { default_global_keyblock = ticket_block_create(ticket_data, ticket_data_len); } else { diff --git a/iocore/net/SSLSessionTicket.cc b/iocore/net/SSLSessionTicket.cc new file mode 100644 index 00000000000..151c64a95a1 --- /dev/null +++ b/iocore/net/SSLSessionTicket.cc @@ -0,0 +1,117 @@ +/** @file + + SessionTicket TLS Extension + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "SSLSessionTicket.h" + +#if TS_HAVE_OPENSSL_SESSION_TICKETS + +#include "tscore/MatcherUtils.h" + +#include "P_Net.h" +#include "SSLStats.h" +#include "P_SSLConfig.h" + +// Remvoe this when drop OpenSSL 1.0.2 support +#ifndef evp_md_func +#ifdef OPENSSL_NO_SHA256 +#define evp_md_func EVP_sha1() +#else +#define evp_md_func EVP_sha256() +#endif +#endif + +void +ssl_session_ticket_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) +{ + ticket_block_free((struct ssl_ticket_key_block *)ptr); +} + +/* + * RFC 5077. Create session ticket to resume SSL session without requiring session-specific state at the TLS server. + * Specifically, it distributes the encrypted session-state information to the client in the form of a ticket and + * a mechanism to present the ticket back to the server. + * */ +int +ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx, + int enc) +{ + SSLCertificateConfig::scoped_config lookup; + SSLTicketKeyConfig::scoped_config params; + SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + + // Get the IP address to look up the keyblock + IpEndpoint ip; + int namelen = sizeof(ip); + SSLCertContext *cc = nullptr; + if (0 == safe_getsockname(netvc->get_socket(), &ip.sa, &namelen)) { + cc = lookup->find(ip); + } + ssl_ticket_key_block *keyblock = nullptr; + if (cc == nullptr || cc->keyblock == nullptr) { + // Try the default + keyblock = params->default_global_keyblock; + } else { + keyblock = cc->keyblock; + } + ink_release_assert(keyblock != nullptr && keyblock->num_keys > 0); + + if (enc == 1) { + const ssl_ticket_key_t &most_recent_key = keyblock->keys[0]; + memcpy(keyname, most_recent_key.key_name, sizeof(most_recent_key.key_name)); + RAND_bytes(iv, EVP_MAX_IV_LENGTH); + EVP_EncryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, most_recent_key.aes_key, iv); + HMAC_Init_ex(hctx, most_recent_key.hmac_secret, sizeof(most_recent_key.hmac_secret), evp_md_func, nullptr); + + Debug("ssl", "create ticket for a new session."); + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_created_stat); + return 1; + } else if (enc == 0) { + for (unsigned i = 0; i < keyblock->num_keys; ++i) { + if (memcmp(keyname, keyblock->keys[i].key_name, sizeof(keyblock->keys[i].key_name)) == 0) { + EVP_DecryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, keyblock->keys[i].aes_key, iv); + HMAC_Init_ex(hctx, keyblock->keys[i].hmac_secret, sizeof(keyblock->keys[i].hmac_secret), evp_md_func, nullptr); + + Debug("ssl", "verify the ticket for an existing session."); + // Increase the total number of decrypted tickets. + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_stat); + + if (i != 0) { // The number of tickets decrypted with "older" keys. + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_old_key_stat); + } + + SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + netvc->setSSLSessionCacheHit(true); + // When we decrypt with an "older" key, encrypt the ticket again with the most recent key. + return (i == 0) ? 1 : 2; + } + } + + Debug("ssl", "keyname is not consistent."); + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_not_found_stat); + return 0; + } + + return -1; +} + +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ diff --git a/iocore/net/SSLSessionTicket.h b/iocore/net/SSLSessionTicket.h new file mode 100644 index 00000000000..be520cbfaf4 --- /dev/null +++ b/iocore/net/SSLSessionTicket.h @@ -0,0 +1,42 @@ +/** @file + + SessionTicket TLS Extension + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include + +// Check if the ticket_key callback #define is available, and if so, enable session tickets. +#ifdef SSL_CTX_set_tlsext_ticket_key_cb +#define TS_HAVE_OPENSSL_SESSION_TICKETS 1 +#endif + +#ifdef TS_HAVE_OPENSSL_SESSION_TICKETS + +#include +#include + +void ssl_session_ticket_free(void *, void *, CRYPTO_EX_DATA *, int, long, void *); +int ssl_callback_session_ticket(SSL *, unsigned char *, unsigned char *, EVP_CIPHER_CTX *, HMAC_CTX *, int); + +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index c2e7061494b..1e48bf29e11 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -36,6 +36,7 @@ #include "P_SSLSNI.h" #include "P_SSLConfig.h" #include "SSLSessionCache.h" +#include "SSLSessionTicket.h" #include "SSLDynlock.h" #include "SSLDiags.h" #include "SSLStats.h" @@ -59,10 +60,6 @@ #include #endif -#if HAVE_OPENSSL_HMAC_H -#include -#endif - #if HAVE_OPENSSL_TS_H #include #endif @@ -130,16 +127,7 @@ struct ssl_user_config { SSLSessionCache *session_cache; // declared extern in P_SSLConfig.h -// Check if the ticket_key callback #define is available, and if so, enable session tickets. -#ifdef SSL_CTX_set_tlsext_ticket_key_cb - -#define HAVE_OPENSSL_SESSION_TICKETS 1 - -static void session_ticket_free(void *, void *, CRYPTO_EX_DATA *, int, long, void *); -static int ssl_callback_session_ticket(SSL *, unsigned char *, unsigned char *, EVP_CIPHER_CTX *, HMAC_CTX *, int); -#endif /* SSL_CTX_set_tlsext_ticket_key_cb */ - -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS static int ssl_session_ticket_index = -1; #endif @@ -376,7 +364,7 @@ set_context_cert(SSL *ssl) if (ctx != nullptr) { SSL_set_SSL_CTX(ssl, ctx); -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS // Reset the ticket callback if needed SSL_CTX_set_tlsext_ticket_key_cb(ctx, ssl_callback_session_ticket); #endif @@ -714,7 +702,7 @@ ssl_context_enable_ecdh(SSL_CTX *ctx) static ssl_ticket_key_block * ssl_context_enable_tickets(SSL_CTX *ctx, const char *ticket_key_path) { -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS ssl_ticket_key_block *keyblock = nullptr; keyblock = ssl_create_ticket_keyblock(ticket_key_path); @@ -736,10 +724,10 @@ ssl_context_enable_tickets(SSL_CTX *ctx, const char *ticket_key_path) SSL_CTX_clear_options(ctx, SSL_OP_NO_TICKET); return keyblock; -#else /* !HAVE_OPENSSL_SESSION_TICKETS */ +#else /* !TS_HAVE_OPENSSL_SESSION_TICKETS */ (void)ticket_key_path; return nullptr; -#endif /* HAVE_OPENSSL_SESSION_TICKETS */ +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ } struct passphrase_cb_userdata { @@ -1015,8 +1003,8 @@ SSLInitializeLibrary() CRYPTO_set_dynlock_destroy_callback(ssl_dyn_destroy_callback); } -#ifdef SSL_CTX_set_tlsext_ticket_key_cb - ssl_session_ticket_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, session_ticket_free); +#ifdef TS_HAVE_OPENSSL_SESSION_TICKETS + ssl_session_ticket_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, ssl_session_ticket_free); if (ssl_session_ticket_index == -1) { SSLError("failed to create session ticket index"); } @@ -1705,7 +1693,7 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons } } if (!inserted) { -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS if (keyblock != nullptr) { ticket_block_free(keyblock); } @@ -1879,83 +1867,6 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l return true; } -#if HAVE_OPENSSL_SESSION_TICKETS - -static void -session_ticket_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) -{ - ticket_block_free((struct ssl_ticket_key_block *)ptr); -} - -/* - * RFC 5077. Create session ticket to resume SSL session without requiring session-specific state at the TLS server. - * Specifically, it distributes the encrypted session-state information to the client in the form of a ticket and - * a mechanism to present the ticket back to the server. - * */ -static int -ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx, - int enc) -{ - SSLCertificateConfig::scoped_config lookup; - SSLTicketKeyConfig::scoped_config params; - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - - // Get the IP address to look up the keyblock - IpEndpoint ip; - int namelen = sizeof(ip); - SSLCertContext *cc = nullptr; - if (0 == safe_getsockname(netvc->get_socket(), &ip.sa, &namelen)) { - cc = lookup->find(ip); - } - ssl_ticket_key_block *keyblock = nullptr; - if (cc == nullptr || cc->keyblock == nullptr) { - // Try the default - keyblock = params->default_global_keyblock; - } else { - keyblock = cc->keyblock; - } - ink_release_assert(keyblock != nullptr && keyblock->num_keys > 0); - - if (enc == 1) { - const ssl_ticket_key_t &most_recent_key = keyblock->keys[0]; - memcpy(keyname, most_recent_key.key_name, sizeof(most_recent_key.key_name)); - RAND_bytes(iv, EVP_MAX_IV_LENGTH); - EVP_EncryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, most_recent_key.aes_key, iv); - HMAC_Init_ex(hctx, most_recent_key.hmac_secret, sizeof(most_recent_key.hmac_secret), evp_md_func, nullptr); - - Debug("ssl", "create ticket for a new session."); - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_created_stat); - return 1; - } else if (enc == 0) { - for (unsigned i = 0; i < keyblock->num_keys; ++i) { - if (memcmp(keyname, keyblock->keys[i].key_name, sizeof(keyblock->keys[i].key_name)) == 0) { - EVP_DecryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, keyblock->keys[i].aes_key, iv); - HMAC_Init_ex(hctx, keyblock->keys[i].hmac_secret, sizeof(keyblock->keys[i].hmac_secret), evp_md_func, nullptr); - - Debug("ssl", "verify the ticket for an existing session."); - // Increase the total number of decrypted tickets. - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_stat); - - if (i != 0) { // The number of tickets decrypted with "older" keys. - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_old_key_stat); - } - - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - netvc->setSSLSessionCacheHit(true); - // When we decrypt with an "older" key, encrypt the ticket again with the most recent key. - return (i == 0) ? 1 : 2; - } - } - - Debug("ssl", "keyname is not consistent."); - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_not_found_stat); - return 0; - } - - return -1; -} -#endif /* HAVE_OPENSSL_SESSION_TICKETS */ - // Release SSL_CTX and the associated data. This works for both // client and server contexts and gracefully accepts nullptr. void From aecfd526d784d3469cdb0437067ba0e09a5a26d0 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 20 Feb 2019 10:33:03 +0900 Subject: [PATCH 318/526] Cleanup: Loading certs by SSLMultiCertConfigLoader --- iocore/net/P_SSLCertLookup.h | 3 +- iocore/net/P_SSLConfig.h | 10 +- iocore/net/P_SSLUtils.h | 75 ++++-- iocore/net/SSLConfig.cc | 3 +- iocore/net/SSLStats.cc | 4 +- iocore/net/SSLUtils.cc | 428 +++++++++++++++++++---------------- 6 files changed, 301 insertions(+), 222 deletions(-) diff --git a/iocore/net/P_SSLCertLookup.h b/iocore/net/P_SSLCertLookup.h index 65d278abe74..475d05de0cf 100644 --- a/iocore/net/P_SSLCertLookup.h +++ b/iocore/net/P_SSLCertLookup.h @@ -23,8 +23,9 @@ #pragma once +#include + #include "ProxyConfig.h" -#include "P_SSLUtils.h" struct SSLConfigParams; struct SSLContextStorage; diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 14b7b9ff1f1..39058b621b5 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -30,13 +30,15 @@ ****************************************************************************/ #pragma once +#include + +#include "tscore/ink_inet.h" +#include "tscore/IpMap.h" + #include "ProxyConfig.h" + #include "SSLSessionCache.h" -#include "tscore/ink_inet.h" -#include -#include "P_SSLCertLookup.h" #include "YamlSNIConfig.h" -#include struct SSLCertLookup; struct ssl_ticket_key_block; diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index a61d8bcf435..3e3de60ca8f 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -21,10 +21,6 @@ #pragma once -#include "tscore/ink_config.h" -#include "tscore/Diags.h" -#include "P_SSLClientUtils.h" - #define OPENSSL_THREAD_DEFINES // BoringSSL does not have this include file @@ -33,37 +29,88 @@ #endif #include +#include "tscore/ink_config.h" +#include "tscore/Diags.h" + +#include "P_SSLClientUtils.h" +#include "P_SSLCertLookup.h" + struct SSLConfigParams; -struct SSLCertLookup; class SSLNetVConnection; typedef int ssl_error_t; -// Create a default SSL server context. -SSL_CTX *SSLDefaultServerContext(); +/** + @brief Gather user provided settings from ssl_multicert.config in to this single struct + */ +struct SSLMultiCertConfigParams { + SSLMultiCertConfigParams() : opt(SSLCertContext::OPT_NONE) + { + REC_ReadConfigInt32(session_ticket_enabled, "proxy.config.ssl.server.session_ticket.enable"); + } + + int session_ticket_enabled; ///< session ticket enabled + ats_scoped_str addr; ///< IPv[64] address to match + ats_scoped_str cert; ///< certificate + ats_scoped_str first_cert; ///< the first certificate name when multiple cert files are in 'ssl_cert_name' + ats_scoped_str ca; ///< CA public certificate + ats_scoped_str key; ///< Private key + ats_scoped_str dialog; ///< Private key dialog + ats_scoped_str servername; ///< Destination server + SSLCertContext::Option opt; ///< SSLCertContext special handling option +}; + +/** + @brief Load SSL certificates from ssl_multicert.config and setup SSLCertLookup for SSLCertificateConfig + */ +class SSLMultiCertConfigLoader +{ +public: + SSLMultiCertConfigLoader(const SSLConfigParams *p) : _params(p) {} + + bool load(SSLCertLookup *lookup); + + virtual SSL_CTX *default_server_ssl_ctx(); + virtual SSL_CTX *init_server_ssl_ctx(std::vector &certList, const SSLMultiCertConfigParams *sslMultCertSettings); + + static bool load_certs(SSL_CTX *ctx, std::vector &certList, const SSLConfigParams *params, + const SSLMultiCertConfigParams *ssl_multi_cert_params); + static bool set_session_id_context(SSL_CTX *ctx, const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings); + + static bool index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cert, const char *certname); + static int check_server_cert_now(X509 *cert, const char *certname); + static void clear_pw_references(SSL_CTX *ssl_ctx); + +protected: + const SSLConfigParams *_params; + +private: + virtual SSL_CTX *_store_ssl_ctx(SSLCertLookup *lookup, const SSLMultiCertConfigParams *ssl_multi_cert_params); + virtual void _set_handshake_callbacks(SSL_CTX *ctx); +}; // Create a new SSL server context fully configured. +// Used by TS API (TSSslServerContextCreate) SSL_CTX *SSLCreateServerContext(const SSLConfigParams *params); +// Release SSL_CTX and the associated data. This works for both +// client and server contexts and gracefully accepts nullptr. +// Used by TS API (TSSslContextDestroy) +void SSLReleaseContext(SSL_CTX *ctx); + // Initialize the SSL library. void SSLInitializeLibrary(); // Initialize SSL library based on configuration settings void SSLPostConfigInitialize(); -// Release SSL_CTX and the associated data. This works for both -// client and server contexts and gracefully accepts nullptr. -void SSLReleaseContext(SSL_CTX *ctx); - // Wrapper functions to SSL I/O routines ssl_error_t SSLWriteBuffer(SSL *ssl, const void *buf, int64_t nbytes, int64_t &nwritten); ssl_error_t SSLReadBuffer(SSL *ssl, void *buf, int64_t nbytes, int64_t &nread); ssl_error_t SSLAccept(SSL *ssl); ssl_error_t SSLConnect(SSL *ssl); -// Load the SSL certificate configuration. -bool SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *lookup); - // Attach a SSL NetVC back pointer to a SSL session. void SSLNetVCAttach(SSL *ssl, SSLNetVConnection *vc); diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index d822ce41b4e..51855b45fa3 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -518,7 +518,8 @@ SSLCertificateConfig::reconfigure() ink_hrtime_sleep(HRTIME_SECONDS(secs)); } - SSLParseCertificateConfiguration(params, lookup); + SSLMultiCertConfigLoader loader(params); + loader.load(lookup); if (!lookup->is_valid) { retStatus = false; diff --git a/iocore/net/SSLStats.cc b/iocore/net/SSLStats.cc index c748b94820f..b15e40d088c 100644 --- a/iocore/net/SSLStats.cc +++ b/iocore/net/SSLStats.cc @@ -26,6 +26,7 @@ #include #include "P_SSLConfig.h" +#include "P_SSLUtils.h" RecRawStatBlock *ssl_rsb = nullptr; std::unordered_map cipher_map; @@ -206,7 +207,8 @@ SSLInitializeStatistics() // filtered by proxy.config.ssl.server.cipher_suite. This keeps the set of cipher suites stable across // configuration reloads and works for the case where we honor the client cipher preference. - ctx = SSLDefaultServerContext(); + SSLMultiCertConfigLoader loader(nullptr); + ctx = loader.default_server_ssl_ctx(); ssl = SSL_new(ctx); ciphers = SSL_get_ciphers(ssl); diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 1e48bf29e11..20bd0eac199 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -95,36 +95,6 @@ static constexpr char SSL_CERT_SEPARATE_DELIM = ','; #endif #endif -/* - * struct ssl_user_config: gather user provided settings from ssl_multicert.config in to this single struct - * ssl_ticket_enabled - session ticket enabled - * ssl_cert_name - certificate - * dest_ip - IPv[64] address to match - * ssl_cert_name - certificate - * first_cert - the first certificate name when multiple cert files are in 'ssl_cert_name' - * ssl_ca_name - CA public certificate - * ssl_key_name - Private key - * ticket_key_name - session key file. [key_name (16Byte) + HMAC_secret (16Byte) + AES_key (16Byte)] - * ssl_key_dialog - Private key dialog - * servername - Destination server - */ -struct ssl_user_config { - ssl_user_config() : opt(SSLCertContext::OPT_NONE) - { - REC_ReadConfigInt32(session_ticket_enabled, "proxy.config.ssl.server.session_ticket.enable"); - } - - int session_ticket_enabled; - ats_scoped_str addr; - ats_scoped_str cert; - ats_scoped_str first_cert; - ats_scoped_str ca; - ats_scoped_str key; - ats_scoped_str dialog; - ats_scoped_str servername; - SSLCertContext::Option opt; -}; - SSLSessionCache *session_cache; // declared extern in P_SSLConfig.h #if TS_HAVE_OPENSSL_SESSION_TICKETS @@ -1022,7 +992,7 @@ SSLInitializeLibrary() } SSL_CTX * -SSLDefaultServerContext() +SSLMultiCertConfigLoader::default_server_ssl_ctx() { return SSL_CTX_new(SSLv23_server_method()); } @@ -1064,19 +1034,22 @@ SSLPrivateKeyHandler(SSL_CTX *ctx, const SSLConfigParams *params, const std::str return true; } -static int -SSLCheckServerCertNow(X509 *cert, const char *certname) +/** + returns 0 on OK or negative value on failure and update log as appropriate. + + Will check: + - if file exists, and has read permissions + - for truncation or other PEM read fail + - current time is between notBefore and notAfter dates of certificate + if anything is not kosher, a negative value is returned and appropriate error logged. + + @static + */ +int +SSLMultiCertConfigLoader::check_server_cert_now(X509 *cert, const char *certname) { int timeCmpValue; - // SSLCheckServerCertNow() - returns 0 on OK or negative value on failure - // and update log as appropriate. - // Will check: - // - if file exists, and has read permissions - // - for truncation or other PEM read fail - // - current time is between notBefore and notAfter dates of certificate - // if anything is not kosher, a negative value is returned and appropriate error logged. - if (!cert) { // a truncated certificate would fall into here Error("invalid certificate %s: file is truncated or corrupted", certname); @@ -1122,11 +1095,14 @@ asn1_strdup(ASN1_STRING *s) return ats_strndup((const char *)ASN1_STRING_get0_data(s), ASN1_STRING_length(s)); } -// Given a certificate and it's corresponding SSL_CTX context, insert hash -// table aliases for subject CN and subjectAltNames DNS without wildcard, -// insert trie aliases for those with wildcard. -static bool -ssl_index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cert, const char *certname) +/** + Given a certificate and it's corresponding SSL_CTX context, insert hash + table aliases for subject CN and subjectAltNames DNS without wildcard, + insert trie aliases for those with wildcard. + @static +*/ +bool +SSLMultiCertConfigLoader::index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cert, const char *certname) { X509_NAME *subject = nullptr; bool inserted = false; @@ -1232,8 +1208,8 @@ ssl_callback_info(const SSL *ssl, int where, int ret) } } -static void -ssl_set_handshake_callbacks(SSL_CTX *ctx) +void +SSLMultiCertConfigLoader::_set_handshake_callbacks(SSL_CTX *ctx) { // Make sure the callbacks are set #if TS_USE_CERT_CB @@ -1275,16 +1251,17 @@ setClientCertLevel(SSL *ssl, uint8_t certLevel) SSL_set_verify_depth(ssl, params->verify_depth); // might want to make configurable at some point. } +/** + Initialize SSL_CTX for server + This is public function because of used by SSLCreateServerContext. + */ SSL_CTX * -SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMultCertSettings, std::vector &cert_list) +SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, const SSLMultiCertConfigParams *sslMultCertSettings) { + const SSLConfigParams *params = this->_params; + int server_verify_client; - SSL_CTX *ctx = SSLDefaultServerContext(); - EVP_MD_CTX *digest = EVP_MD_CTX_new(); - STACK_OF(X509_NAME) *ca_list = nullptr; - unsigned char hash_buf[EVP_MAX_MD_SIZE]; - unsigned int hash_len = 0; - const char *setting_cert = sslMultCertSettings ? sslMultCertSettings->cert.get() : nullptr; + SSL_CTX *ctx = this->default_server_ssl_ctx(); // disable selected protocols SSL_CTX_set_options(ctx, params->ssl_ctx_options); @@ -1366,84 +1343,9 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } if (sslMultCertSettings->cert) { - SimpleTokenizer cert_tok((const char *)sslMultCertSettings->cert, SSL_CERT_SEPARATE_DELIM); - SimpleTokenizer key_tok((sslMultCertSettings->key ? (const char *)sslMultCertSettings->key : ""), SSL_CERT_SEPARATE_DELIM); - - if (sslMultCertSettings->key && cert_tok.getNumTokensRemaining() != key_tok.getNumTokensRemaining()) { - Error("the number of certificates in ssl_cert_name and ssl_key_name doesn't match"); + if (!SSLMultiCertConfigLoader::load_certs(ctx, cert_list, params, sslMultCertSettings)) { goto fail; } - SimpleTokenizer ca_tok("", SSL_CERT_SEPARATE_DELIM); - if (sslMultCertSettings->ca) { - ca_tok.setString(sslMultCertSettings->ca); - if (cert_tok.getNumTokensRemaining() != ca_tok.getNumTokensRemaining()) { - Error("the number of certificates in ssl_cert_name and ssl_ca_name doesn't match"); - goto fail; - } - } - - for (const char *certname = cert_tok.getNext(); certname; certname = cert_tok.getNext()) { - std::string completeServerCertPath = Layout::relative_to(params->serverCertPathOnly, certname); - scoped_BIO bio(BIO_new_file(completeServerCertPath.c_str(), "r")); - X509 *cert = nullptr; - if (bio) { - cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr); - } - if (!bio || !cert) { - SSLError("failed to load certificate chain from %s", completeServerCertPath.c_str()); - goto fail; - } - if (!SSL_CTX_use_certificate(ctx, cert)) { - SSLError("Failed to assign cert from %s to SSL_CTX", completeServerCertPath.c_str()); - X509_free(cert); - goto fail; - } - - // Load up any additional chain certificates - SSL_CTX_add_extra_chain_cert_bio(ctx, bio); - - const char *keyPath = key_tok.getNext(); - if (!SSLPrivateKeyHandler(ctx, params, completeServerCertPath, keyPath)) { - goto fail; - } - - cert_list.push_back(cert); - if (SSLConfigParams::load_ssl_file_cb) { - SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str(), CONFIG_FLAG_UNVERSIONED); - } - - // Must load all the intermediate certificates before starting the next chain - - // First, load any CA chains from the global chain file. This should probably - // eventually be a comma separated list too. For now we will load it in all chains even - // though it only makes sense in one chain - if (params->serverCertChainFilename) { - ats_scoped_str completeServerCertChainPath( - Layout::relative_to(params->serverCertPathOnly, params->serverCertChainFilename)); - if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { - SSLError("failed to load global certificate chain from %s", (const char *)completeServerCertChainPath); - goto fail; - } - if (SSLConfigParams::load_ssl_file_cb) { - SSLConfigParams::load_ssl_file_cb(completeServerCertChainPath, CONFIG_FLAG_UNVERSIONED); - } - } - - // Now, load any additional certificate chains specified in this entry. - if (sslMultCertSettings->ca) { - const char *ca_name = ca_tok.getNext(); - if (ca_name != nullptr) { - ats_scoped_str completeServerCertChainPath(Layout::relative_to(params->serverCertPathOnly, ca_name)); - if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { - SSLError("failed to load certificate chain from %s", (const char *)completeServerCertChainPath); - goto fail; - } - if (SSLConfigParams::load_ssl_file_cb) { - SSLConfigParams::load_ssl_file_cb(completeServerCertChainPath, CONFIG_FLAG_UNVERSIONED); - } - } - } - } } // SSL_CTX_load_verify_locations() builds the cert chain from the @@ -1497,50 +1399,7 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu SSL_CTX_set_verify_depth(ctx, params->verify_depth); // might want to make configurable at some point. } - // Set the list of CA's to send to client if we ask for a client - // certificate - if (params->serverCACertFilename) { - ca_list = SSL_load_client_CA_file(params->serverCACertFilename); - if (ca_list) { - SSL_CTX_set_client_CA_list(ctx, ca_list); - } - } - - if (EVP_DigestInit_ex(digest, evp_md_func, nullptr) == 0) { - SSLError("EVP_DigestInit_ex failed"); - goto fail; - } - - if (nullptr != setting_cert) { - Debug("ssl", "Using '%s' in hash for session id context", sslMultCertSettings->cert.get()); - if (EVP_DigestUpdate(digest, sslMultCertSettings->cert, strlen(setting_cert)) == 0) { - SSLError("EVP_DigestUpdate failed"); - goto fail; - } - } - - if (ca_list != nullptr) { - size_t num_certs = sk_X509_NAME_num(ca_list); - - for (size_t i = 0; i < num_certs; i++) { - X509_NAME *name = sk_X509_NAME_value(ca_list, i); - if (X509_NAME_digest(name, evp_md_func, hash_buf /* borrow our final hash buffer. */, &hash_len) == 0 || - EVP_DigestUpdate(digest, hash_buf, hash_len) == 0) { - SSLError("Adding X509 name to digest failed"); - goto fail; - } - } - } - - if (EVP_DigestFinal_ex(digest, hash_buf, &hash_len) == 0) { - SSLError("EVP_DigestFinal_ex failed"); - goto fail; - } - EVP_MD_CTX_free(digest); - digest = nullptr; - - if (SSL_CTX_set_session_id_context(ctx, hash_buf, hash_len) == 0) { - SSLError("SSL_CTX_set_session_id_context failed"); + if (!SSLMultiCertConfigLoader::set_session_id_context(ctx, params, sslMultCertSettings)) { goto fail; } @@ -1574,13 +1433,9 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } ssl_context_enable_ecdh(ctx); -#define SSL_CLEAR_PW_REFERENCES(CTX) \ - { \ - SSL_CTX_set_default_passwd_cb(CTX, nullptr); \ - SSL_CTX_set_default_passwd_cb_userdata(CTX, nullptr); \ - } + if (sslMultCertSettings && sslMultCertSettings->dialog) { - SSL_CLEAR_PW_REFERENCES(ctx); + SSLMultiCertConfigLoader::clear_pw_references(ctx); } SSL_CTX_set_info_callback(ctx, ssl_callback_info); @@ -1596,10 +1451,11 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu if (SSLConfigParams::ssl_ocsp_enabled) { Debug("ssl", "SSL OCSP Stapling is enabled"); SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); + const char *cert_name = sslMultCertSettings ? sslMultCertSettings->cert.get() : nullptr; for (auto cert : cert_list) { - if (!ssl_stapling_init_cert(ctx, cert, setting_cert)) { - Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", setting_cert); + if (!ssl_stapling_init_cert(ctx, cert, cert_name)) { + Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", cert_name); } } } else { @@ -1614,13 +1470,11 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu if (SSLConfigParams::init_ssl_ctx_cb) { SSLConfigParams::init_ssl_ctx_cb(ctx, true); } + return ctx; fail: - if (digest) { - EVP_MD_CTX_free(digest); - } - SSL_CLEAR_PW_REFERENCES(ctx) + SSLMultiCertConfigLoader::clear_pw_references(ctx); SSLReleaseContext(ctx); for (auto cert : cert_list) { X509_free(cert); @@ -1632,21 +1486,24 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu SSL_CTX * SSLCreateServerContext(const SSLConfigParams *params) { + SSLMultiCertConfigLoader loader(params); std::vector cert_list; - SSL_CTX *ctx = SSLInitServerContext(params, nullptr, cert_list); + + SSL_CTX *ctx = loader.init_server_ssl_ctx(cert_list, nullptr); ink_assert(cert_list.empty()); + return ctx; } /** Insert SSLCertContext (SSL_CTX ans options) into SSLCertLookup with key. - Do NOT call SSL_CTX_set_* functions from here. SSL_CTX should be set up by SSLInitServerContext(). + Do NOT call SSL_CTX_set_* functions from here. SSL_CTX should be set up by SSLMultiCertConfigLoader::init_server_ssl_ctx(). */ -static SSL_CTX * -ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, const ssl_user_config *sslMultCertSettings) +SSL_CTX * +SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const SSLMultiCertConfigParams *sslMultCertSettings) { std::vector cert_list; - SSL_CTX *ctx = SSLInitServerContext(params, sslMultCertSettings, cert_list); + SSL_CTX *ctx = this->init_server_ssl_ctx(cert_list, sslMultCertSettings); ssl_ticket_key_block *keyblock = nullptr; bool inserted = false; @@ -1657,7 +1514,7 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons const char *certname = sslMultCertSettings->cert.get(); for (auto cert : cert_list) { - if (0 > SSLCheckServerCertNow(cert, certname)) { + if (0 > SSLMultiCertConfigLoader::check_server_cert_now(cert, certname)) { /* At this point, we know cert is bad, and we've already printed a descriptive reason as to why cert is bad to the log file */ Debug("ssl", "Marking certificate as NOT VALID: %s", certname); @@ -1676,7 +1533,7 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons if (lookup->insert(sslMultCertSettings->addr, SSLCertContext(ctx, sslMultCertSettings->opt, keyblock)) >= 0) { inserted = true; lookup->ssl_default = ctx; - ssl_set_handshake_callbacks(ctx); + this->_set_handshake_callbacks(ctx); } } else { IpEndpoint ep; @@ -1705,7 +1562,7 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons // refcounting or alternate way of avoiding double frees. Debug("ssl", "importing SNI names from %s", (const char *)certname); for (auto cert : cert_list) { - if (ssl_index_certificate(lookup, SSLCertContext(ctx, sslMultCertSettings->opt), cert, certname)) { + if (SSLMultiCertConfigLoader::index_certificate(lookup, SSLCertContext(ctx, sslMultCertSettings->opt), cert, certname)) { inserted = true; } } @@ -1729,7 +1586,7 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons } static bool -ssl_extract_certificate(const matcher_line *line_info, ssl_user_config &sslMultCertSettings) +ssl_extract_certificate(const matcher_line *line_info, SSLMultiCertConfigParams &sslMultCertSettings) { for (int i = 0; i < MATCHER_MAX_TOKENS; ++i) { const char *label; @@ -1792,8 +1649,10 @@ ssl_extract_certificate(const matcher_line *line_info, ssl_user_config &sslMultC } bool -SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *lookup) +SSLMultiCertConfigLoader::load(SSLCertLookup *lookup) { + const SSLConfigParams *params = this->_params; + char *tok_state = nullptr; char *line = nullptr; ats_scoped_str file_buf; @@ -1829,7 +1688,7 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l } if (*line != '\0' && *line != '#') { - ssl_user_config sslMultiCertSettings; + SSLMultiCertConfigParams sslMultiCertSettings; const char *errPtr; errPtr = parseConfigLine(line, &line_info, &sslCertTags); @@ -1841,7 +1700,7 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l if (ssl_extract_certificate(&line_info, sslMultiCertSettings)) { // There must be a certificate specified unless the tunnel action is set if (sslMultiCertSettings.cert || sslMultiCertSettings.opt != SSLCertContext::OPT_TUNNEL) { - ssl_store_ssl_context(params, lookup, &sslMultiCertSettings); + this->_store_ssl_ctx(lookup, &sslMultiCertSettings); } else { Warning("No ssl_cert_name specified and no tunnel action set"); } @@ -1856,9 +1715,9 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l // bootstrap the SSL handshake so that we can subsequently do the SNI lookup to switch to the real // context. if (lookup->ssl_default == nullptr) { - ssl_user_config sslMultiCertSettings; + SSLMultiCertConfigParams sslMultiCertSettings; sslMultiCertSettings.addr = ats_strdup("*"); - if (ssl_store_ssl_context(params, lookup, &sslMultiCertSettings) == nullptr) { + if (this->_store_ssl_ctx(lookup, &sslMultiCertSettings) == nullptr) { Error("failed set default context"); return false; } @@ -1988,3 +1847,170 @@ SSLConnect(SSL *ssl) return ssl_error; } + +/** + Load certificats to SSL_CTX + @static + */ +bool +SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_list, const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings) +{ + SimpleTokenizer cert_tok((const char *)sslMultCertSettings->cert, SSL_CERT_SEPARATE_DELIM); + SimpleTokenizer key_tok((sslMultCertSettings->key ? (const char *)sslMultCertSettings->key : ""), SSL_CERT_SEPARATE_DELIM); + + if (sslMultCertSettings->key && cert_tok.getNumTokensRemaining() != key_tok.getNumTokensRemaining()) { + Error("the number of certificates in ssl_cert_name and ssl_key_name doesn't match"); + return false; + } + SimpleTokenizer ca_tok("", SSL_CERT_SEPARATE_DELIM); + if (sslMultCertSettings->ca) { + ca_tok.setString(sslMultCertSettings->ca); + if (cert_tok.getNumTokensRemaining() != ca_tok.getNumTokensRemaining()) { + Error("the number of certificates in ssl_cert_name and ssl_ca_name doesn't match"); + return false; + } + } + + for (const char *certname = cert_tok.getNext(); certname; certname = cert_tok.getNext()) { + std::string completeServerCertPath = Layout::relative_to(params->serverCertPathOnly, certname); + scoped_BIO bio(BIO_new_file(completeServerCertPath.c_str(), "r")); + X509 *cert = nullptr; + if (bio) { + cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr); + } + if (!bio || !cert) { + SSLError("failed to load certificate chain from %s", completeServerCertPath.c_str()); + return false; + } + if (!SSL_CTX_use_certificate(ctx, cert)) { + SSLError("Failed to assign cert from %s to SSL_CTX", completeServerCertPath.c_str()); + X509_free(cert); + return false; + } + + // Load up any additional chain certificates + SSL_CTX_add_extra_chain_cert_bio(ctx, bio); + + const char *keyPath = key_tok.getNext(); + if (!SSLPrivateKeyHandler(ctx, params, completeServerCertPath, keyPath)) { + return false; + } + + cert_list.push_back(cert); + if (SSLConfigParams::load_ssl_file_cb) { + SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str(), CONFIG_FLAG_UNVERSIONED); + } + + // Must load all the intermediate certificates before starting the next chain + + // First, load any CA chains from the global chain file. This should probably + // eventually be a comma separated list too. For now we will load it in all chains even + // though it only makes sense in one chain + if (params->serverCertChainFilename) { + ats_scoped_str completeServerCertChainPath(Layout::relative_to(params->serverCertPathOnly, params->serverCertChainFilename)); + if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { + SSLError("failed to load global certificate chain from %s", (const char *)completeServerCertChainPath); + return false; + } + if (SSLConfigParams::load_ssl_file_cb) { + SSLConfigParams::load_ssl_file_cb(completeServerCertChainPath, CONFIG_FLAG_UNVERSIONED); + } + } + + // Now, load any additional certificate chains specified in this entry. + if (sslMultCertSettings->ca) { + const char *ca_name = ca_tok.getNext(); + if (ca_name != nullptr) { + ats_scoped_str completeServerCertChainPath(Layout::relative_to(params->serverCertPathOnly, ca_name)); + if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { + SSLError("failed to load certificate chain from %s", (const char *)completeServerCertChainPath); + return false; + } + if (SSLConfigParams::load_ssl_file_cb) { + SSLConfigParams::load_ssl_file_cb(completeServerCertChainPath, CONFIG_FLAG_UNVERSIONED); + } + } + } + } + + return true; +} + +/** + Set session_id context for session reuse + @static + */ +bool +SSLMultiCertConfigLoader::set_session_id_context(SSL_CTX *ctx, const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings) +{ + EVP_MD_CTX *digest = EVP_MD_CTX_new(); + STACK_OF(X509_NAME) *ca_list = nullptr; + unsigned char hash_buf[EVP_MAX_MD_SIZE]; + unsigned int hash_len = 0; + const char *setting_cert = sslMultCertSettings ? sslMultCertSettings->cert.get() : nullptr; + bool result = false; + + // Set the list of CA's to send to client if we ask for a client certificate + if (params->serverCACertFilename) { + ca_list = SSL_load_client_CA_file(params->serverCACertFilename); + if (ca_list) { + SSL_CTX_set_client_CA_list(ctx, ca_list); + } + } + + if (EVP_DigestInit_ex(digest, evp_md_func, nullptr) == 0) { + SSLError("EVP_DigestInit_ex failed"); + goto fail; + } + + if (nullptr != setting_cert) { + Debug("ssl", "Using '%s' in hash for session id context", sslMultCertSettings->cert.get()); + if (EVP_DigestUpdate(digest, sslMultCertSettings->cert, strlen(setting_cert)) == 0) { + SSLError("EVP_DigestUpdate failed"); + goto fail; + } + } + + if (ca_list != nullptr) { + size_t num_certs = sk_X509_NAME_num(ca_list); + + for (size_t i = 0; i < num_certs; i++) { + X509_NAME *name = sk_X509_NAME_value(ca_list, i); + if (X509_NAME_digest(name, evp_md_func, hash_buf /* borrow our final hash buffer. */, &hash_len) == 0 || + EVP_DigestUpdate(digest, hash_buf, hash_len) == 0) { + SSLError("Adding X509 name to digest failed"); + goto fail; + } + } + } + + if (EVP_DigestFinal_ex(digest, hash_buf, &hash_len) == 0) { + SSLError("EVP_DigestFinal_ex failed"); + goto fail; + } + + if (SSL_CTX_set_session_id_context(ctx, hash_buf, hash_len) == 0) { + SSLError("SSL_CTX_set_session_id_context failed"); + goto fail; + } + + result = true; + +fail: + EVP_MD_CTX_free(digest); + + return result; +} + +/** + Clear password in SSL_CTX + @static + */ +void +SSLMultiCertConfigLoader::clear_pw_references(SSL_CTX *ssl_ctx) +{ + SSL_CTX_set_default_passwd_cb(ssl_ctx, nullptr); + SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, nullptr); +} From 58dc8b5cb7885a945eadb370e091f828bd199b56 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 27 Feb 2019 08:51:57 +0900 Subject: [PATCH 319/526] Fix directives for checking TS_USE_TLS_OCSP --- iocore/net/OCSPStapling.cc | 2 +- iocore/net/SSLNetProcessor.cc | 4 ++-- iocore/net/SSLUtils.cc | 4 ++-- src/traffic_server/InkAPI.cc | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index 26316ce9cf5..2ebae671673 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -20,7 +20,7 @@ */ #include "P_OCSPStapling.h" -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP #include #include diff --git a/iocore/net/SSLNetProcessor.cc b/iocore/net/SSLNetProcessor.cc index 90c7a4e622c..bec9a3d0725 100644 --- a/iocore/net/SSLNetProcessor.cc +++ b/iocore/net/SSLNetProcessor.cc @@ -37,7 +37,7 @@ SSLNetProcessor ssl_NetProcessor; NetProcessor &sslNetProcessor = ssl_NetProcessor; SNIActionPerformer sni_action_performer; -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP struct OCSPContinuation : public Continuation { int mainEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) @@ -76,7 +76,7 @@ SSLNetProcessor::start(int, size_t stacksize) // Initialize SSL statistics. This depends on an initial set of certificates being loaded above. SSLInitializeStatistics(); -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP if (SSLConfigParams::ssl_ocsp_enabled) { // Call the update initially to get things populated ocsp_update(); diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 20bd0eac199..145a6bb004f 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -980,7 +980,7 @@ SSLInitializeLibrary() } #endif -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP ssl_stapling_ex_init(); #endif /* TS_USE_TLS_OCSP */ @@ -1447,7 +1447,7 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co SSL_CTX_set_alpn_select_cb(ctx, SSLNetVConnection::select_next_protocol, nullptr); #endif /* TS_USE_TLS_ALPN */ -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP if (SSLConfigParams::ssl_ocsp_enabled) { Debug("ssl", "SSL OCSP Stapling is enabled"); SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index e7d108093ec..e214548a081 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -8984,7 +8984,7 @@ TSSslServerContextCreate(TSSslX509 cert, const char *certname) SSLConfigParams *config = SSLConfig::acquire(); if (config != nullptr) { ret = reinterpret_cast(SSLCreateServerContext(config)); -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP if (ret && SSLConfigParams::ssl_ocsp_enabled && cert && certname) { if (SSL_CTX_set_tlsext_status_cb(reinterpret_cast(ret), ssl_callback_ocsp_stapling)) { if (!ssl_stapling_init_cert(reinterpret_cast(ret), reinterpret_cast(cert), certname)) { From d388e7ca820c00665cf38795fa4977a38d49f52c Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 27 Feb 2019 09:38:17 +0900 Subject: [PATCH 320/526] Fix directives for checking TS_HAS_TESTS --- iocore/dns/DNS.cc | 2 +- iocore/hostdb/HostDB.cc | 2 +- iocore/net/SSLCertLookup.cc | 64 ++++++++++++++++++------------------- proxy/Transform.cc | 2 +- proxy/Transform.h | 2 +- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 6deb3dc1a79..aae4c6bf100 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -1825,7 +1825,7 @@ ink_dns_init(ts::ModuleVersion v) RecRawStatSyncSum); } -#ifdef TS_HAS_TESTS +#if TS_HAS_TESTS struct DNSRegressionContinuation; using DNSRegContHandler = int (DNSRegressionContinuation::*)(int, void *); diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 5b81c858277..73c80141c8d 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -2286,7 +2286,7 @@ ParseHostFile(const char *path, unsigned int hostdb_hostfile_check_interval) // Regression tests // // Take a started hostDB and fill it up and make sure it doesn't explode -#ifdef TS_HAS_TESTS +#if TS_HAS_TESTS struct HostDBRegressionContinuation; struct HostDBRegressionContinuation : public Continuation { diff --git a/iocore/net/SSLCertLookup.cc b/iocore/net/SSLCertLookup.cc index 2b6e91dd976..bf6c1a02bdb 100644 --- a/iocore/net/SSLCertLookup.cc +++ b/iocore/net/SSLCertLookup.cc @@ -311,38 +311,6 @@ make_to_lower_case(const char *name, char *lower_case_name, int buf_len) lower_case_name[i] = '\0'; } -static char * -reverse_dns_name(const char *hostname, char (&reversed)[TS_MAX_HOST_NAME_LEN + 1]) -{ - char *ptr = reversed + sizeof(reversed); - const char *part = hostname; - - *(--ptr) = '\0'; // NUL-terminate - - while (*part) { - ssize_t len = strcspn(part, "."); - ssize_t remain = ptr - reversed; - - if (remain < (len + 1)) { - return nullptr; - } - - ptr -= len; - memcpy(ptr, part, len); - - // Skip to the next domain component. This will take us to either a '.' or a NUL. - // If it's a '.' we need to skip over it. - part += len; - if (*part == '.') { - ++part; - *(--ptr) = '.'; - } - } - make_to_lower_case(ptr, ptr, strlen(ptr) + 1); - - return ptr; -} - SSLContextStorage::SSLContextStorage() {} bool @@ -457,6 +425,38 @@ SSLContextStorage::lookup(const char *name) #if TS_HAS_TESTS +static char * +reverse_dns_name(const char *hostname, char (&reversed)[TS_MAX_HOST_NAME_LEN + 1]) +{ + char *ptr = reversed + sizeof(reversed); + const char *part = hostname; + + *(--ptr) = '\0'; // NUL-terminate + + while (*part) { + ssize_t len = strcspn(part, "."); + ssize_t remain = ptr - reversed; + + if (remain < (len + 1)) { + return nullptr; + } + + ptr -= len; + memcpy(ptr, part, len); + + // Skip to the next domain component. This will take us to either a '.' or a NUL. + // If it's a '.' we need to skip over it. + part += len; + if (*part == '.') { + ++part; + *(--ptr) = '.'; + } + } + make_to_lower_case(ptr, ptr, strlen(ptr) + 1); + + return ptr; +} + REGRESSION_TEST(SSLWildcardMatch)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) { TestBox box(t, pstatus); diff --git a/proxy/Transform.cc b/proxy/Transform.cc index 37129d023e8..d15f18ad139 100644 --- a/proxy/Transform.cc +++ b/proxy/Transform.cc @@ -727,7 +727,7 @@ NullTransform::handle_event(int event, void *edata) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -#ifdef TS_HAS_TESTS +#if TS_HAS_TESTS void TransformTest::run() { diff --git a/proxy/Transform.h b/proxy/Transform.h index 5789b98a7dc..5d2e3ed057c 100644 --- a/proxy/Transform.h +++ b/proxy/Transform.h @@ -48,7 +48,7 @@ class TransformProcessor int content_type_len, int64_t content_length); }; -#ifdef TS_HAS_TESTS +#if TS_HAS_TESTS class TransformTest { public: From 45b43fbdf8107f5a413ff7cb86c218fb4bcc188f Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Mon, 15 Oct 2018 15:39:26 +0900 Subject: [PATCH 321/526] Split current client transactions metrics into HTTP/1.1 and HTTP/2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `proxy.process.http.current_client_transactions` has number of transactions regardless protocols. It’s useful that each protocol has its own metrics. // Before - `proxy.process.http.current_client_transactions` for HTTP/1.1 & HTTP/2 - `proxy.process.http2.current_client_streams` for HTTP/2 // After - `proxy.process.http.current_client_transactions` for HTTP/1.1 - `proxy.process.http2.current_client_streams` for HTTP/2 --- .../statistics/core/http-connection.en.rst | 7 +++++++ proxy/ProxyTransaction.cc | 3 +++ proxy/ProxyTransaction.h | 3 +++ proxy/http/Http1Transaction.cc | 12 ++++++++++++ proxy/http/Http1Transaction.h | 3 +++ proxy/http/HttpSM.cc | 2 -- proxy/http2/Http2Stream.cc | 14 +++++++++++++- proxy/http2/Http2Stream.h | 5 +++-- 8 files changed, 44 insertions(+), 5 deletions(-) diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index b2e26fa4d96..dc55cc3ae04 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -58,6 +58,8 @@ HTTP Connection .. ts:stat:: global proxy.process.http.current_client_transactions integer :type: gauge + Represents the current number of HTTP/1.0 and HTTP/1.1 transactions from client to the |TS|. + .. ts:stat:: global proxy.process.http.current_server_connections integer :type: gauge @@ -140,3 +142,8 @@ HTTP Connection :type: gauge Represents the current number of HTTP/2 connections from client to the |TS|. + +.. ts:stat:: global proxy.process.http2.current_client_streams integer + :type: gauge + + Represents the current number of HTTP/2 streams from client to the |TS|. diff --git a/proxy/ProxyTransaction.cc b/proxy/ProxyTransaction.cc index 35c1c94520f..31947cafc42 100644 --- a/proxy/ProxyTransaction.cc +++ b/proxy/ProxyTransaction.cc @@ -57,6 +57,7 @@ ProxyTransaction::new_transaction() current_reader->plugin_id = pi->getPluginId(); } + this->increment_client_transactions_stat(); current_reader->attach_client_session(this, sm_reader); } @@ -66,6 +67,8 @@ ProxyTransaction::release(IOBufferReader *r) HttpTxnDebug("[%" PRId64 "] session released by sm [%" PRId64 "]", parent ? parent->connection_id() : 0, current_reader ? current_reader->sm_id : 0); + this->decrement_client_transactions_stat(); + // Pass along the release to the session if (parent) { parent->release(this); diff --git a/proxy/ProxyTransaction.h b/proxy/ProxyTransaction.h index 07a3c4288b9..c9653242113 100644 --- a/proxy/ProxyTransaction.h +++ b/proxy/ProxyTransaction.h @@ -264,6 +264,9 @@ class ProxyTransaction : public VConnection void set_rx_error_code(ProxyError e); void set_tx_error_code(ProxyError e); + virtual void increment_client_transactions_stat() = 0; + virtual void decrement_client_transactions_stat() = 0; + protected: ProxySession *parent; HttpSM *current_reader; diff --git a/proxy/http/Http1Transaction.cc b/proxy/http/Http1Transaction.cc index 09d5f0eae9e..a9ee93e4bae 100644 --- a/proxy/http/Http1Transaction.cc +++ b/proxy/http/Http1Transaction.cc @@ -78,3 +78,15 @@ Http1Transaction::allow_half_open() const } return false; } + +void +Http1Transaction::increment_client_transactions_stat() +{ + HTTP_INCREMENT_DYN_STAT(http_current_client_transactions_stat); +} + +void +Http1Transaction::decrement_client_transactions_stat() +{ + HTTP_DECREMENT_DYN_STAT(http_current_client_transactions_stat); +} diff --git a/proxy/http/Http1Transaction.h b/proxy/http/Http1Transaction.h index 4175b32a43d..bd8040f676f 100644 --- a/proxy/http/Http1Transaction.h +++ b/proxy/http/Http1Transaction.h @@ -127,6 +127,9 @@ class Http1Transaction : public ProxyTransaction return get_transact_count(); } + void increment_client_transactions_stat() override; + void decrement_client_transactions_stat() override; + protected: bool outbound_transparent{false}; }; diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 1707beb7058..17bc572980a 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -486,7 +486,6 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc, IOBufferReader *buffe ink_release_assert(ua_txn->get_half_close_flag() == false); mutex = client_vc->mutex; - HTTP_INCREMENT_DYN_STAT(http_current_client_transactions_stat); if (ua_txn->debug()) { debug_on = true; } @@ -6918,7 +6917,6 @@ HttpSM::kill_this() SMDebug("http", "[%" PRId64 "] deallocating sm", sm_id); // authAdapter.destroyState(); - HTTP_DECREMENT_DYN_STAT(http_current_client_transactions_stat); destroy(); } } diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 83da4e1375a..c6a83ed7c74 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -738,7 +738,6 @@ Http2Stream::destroy() // Clean up the write VIO in case of inactivity timeout this->do_io_write(nullptr, 0, nullptr); - HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); ink_hrtime end_time = Thread::get_hrtime(); HTTP2_SUM_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_TRANSACTIONS_TIME, _thread, end_time - _start_time); _req_header.destroy(); @@ -893,6 +892,19 @@ Http2Stream::release(IOBufferReader *r) this->do_io_close(); } +void +Http2Stream::increment_client_transactions_stat() +{ + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); +} + +void +Http2Stream::decrement_client_transactions_stat() +{ + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); +} + bool Http2Stream::_switch_thread_if_not_on_right_thread(int event, void *edata) { diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index 22030f4bff6..cb9ca8ffc11 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -51,8 +51,6 @@ class Http2Stream : public ProxyTransaction _start_time = Thread::get_hrtime(); _thread = this_ethread(); this->client_rwnd = initial_rwnd; - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); sm_reader = request_reader = request_buffer.alloc_reader(); http_parser_init(&http_parser); // FIXME: Are you sure? every "stream" needs request_header? @@ -222,6 +220,9 @@ class Http2Stream : public ProxyTransaction return is_first_transaction_flag; } + void increment_client_transactions_stat() override; + void decrement_client_transactions_stat() override; + private: void response_initialize_data_handling(bool &is_done); void response_process_data(bool &is_done); From 5e2d175071190331ec7e13f05938fa0de936f6d9 Mon Sep 17 00:00:00 2001 From: scw00 Date: Thu, 21 Feb 2019 14:38:21 +0800 Subject: [PATCH 322/526] Fixed memory leaking introduced by 4873 --- proxy/http/HttpSM.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 17bc572980a..05fa35c3742 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -3288,7 +3288,6 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) vc_table.remove_entry(this->ua_entry); ua_txn->do_io_close(); - ua_txn = nullptr; } else { ink_assert(ua_buffer_reader != nullptr); ua_txn->release(ua_buffer_reader); From 69dd33dd7f4004105b15178c4cecdd2d2aecdccb Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Thu, 28 Feb 2019 10:08:48 +0900 Subject: [PATCH 323/526] Revert "Rename Session and Transaction classes" This reverts commit dfd5d8f4533fadf1ca15834e0c2cdde06f96453e. Conflicts: proxy/ProxyClientSession.cc proxy/http/HttpSM.h --- .../client-session-architecture.en.rst | 32 +++++------ include/tscore/IntrusiveHashMap.h | 8 +-- iocore/eventsystem/I_Thread.h | 2 +- proxy/Makefile.am | 8 +-- ...{ProxySession.cc => ProxyClientSession.cc} | 26 ++++----- .../{ProxySession.h => ProxyClientSession.h} | 21 ++++--- ...ansaction.cc => ProxyClientTransaction.cc} | 20 +++---- ...Transaction.h => ProxyClientTransaction.h} | 22 ++++---- proxy/http/Http1ClientSession.cc | 12 ++-- proxy/http/Http1ClientSession.h | 23 ++++---- ...ansaction.cc => Http1ClientTransaction.cc} | 16 +++--- ...Transaction.h => Http1ClientTransaction.h} | 16 +++--- proxy/http/HttpSM.cc | 22 ++++---- proxy/http/HttpSM.h | 18 +++--- ...1ServerSession.cc => HttpServerSession.cc} | 30 +++++----- ...tp1ServerSession.h => HttpServerSession.h} | 52 +++++++++-------- proxy/http/HttpSessionManager.cc | 20 +++---- proxy/http/HttpSessionManager.h | 19 ++++--- proxy/http/HttpTransact.h | 2 +- proxy/http/HttpTunnel.cc | 2 +- proxy/http/Makefile.am | 8 +-- proxy/http2/Http2ClientSession.cc | 2 +- proxy/http2/Http2ClientSession.h | 10 ++-- proxy/http2/Http2Stream.h | 6 +- src/traffic_server/InkAPI.cc | 56 +++++++++---------- src/traffic_server/traffic_server.cc | 2 +- 26 files changed, 225 insertions(+), 230 deletions(-) rename proxy/{ProxySession.cc => ProxyClientSession.cc} (88%) rename proxy/{ProxySession.h => ProxyClientSession.h} (92%) rename proxy/{ProxyTransaction.cc => ProxyClientTransaction.cc} (83%) rename proxy/{ProxyTransaction.h => ProxyClientTransaction.h} (92%) rename proxy/http/{Http1Transaction.cc => Http1ClientTransaction.cc} (85%) rename proxy/http/{Http1Transaction.h => Http1ClientTransaction.h} (89%) rename proxy/http/{Http1ServerSession.cc => HttpServerSession.cc} (86%) rename proxy/http/{Http1ServerSession.h => HttpServerSession.h} (80%) diff --git a/doc/developer-guide/client-session-architecture.en.rst b/doc/developer-guide/client-session-architecture.en.rst index 6f1f36f82ce..7df36aa9b68 100644 --- a/doc/developer-guide/client-session-architecture.en.rst +++ b/doc/developer-guide/client-session-architecture.en.rst @@ -27,34 +27,34 @@ The User Agent interacts with ATS by creating a session with the ATS server and submitting sequences of requests over the session. ATS supports several session protocols including HTTP/1.x and HTTP/2. A HTTP State Machine is created for each request to process the request. -ATS uses the generic classes ProxySession and ProxyTransaction to hide the details of +ATS uses the generic classes ProxyClientSession and ProxyClientTransaction to hide the details of the underlaying protocols from the HTTP State Machine. Classes ======= -ProxySession +ProxyClientSession ------------------ .. figure:: /static/images/sessions/session_hierarchy.png :align: center - :alt: ProxySession hierarchy + :alt: ProxyClientSession hierarchy -The ProxySession class abstracts the key features of a client session. It contains zero or more ProxyTransaction objects. It also has a reference to the associated NetVC (either UnixNetVConnection or SSLNetVConnection). The session class is responsible for interfacing with the user agent protocol. +The ProxyClientSession class abstracts the key features of a client session. It contains zero or more ProxyClientTransaction objects. It also has a reference to the associated NetVC (either UnixNetVConnection or SSLNetVConnection). The session class is responsible for interfacing with the user agent protocol. At this point there are two concrete subclasses: Http1ClientSession and Http2ClientSession. The Http1ClientSession only has at most one transaction active at a time. The HTTP/2 protocol allows for multiple simultaneous active transactions -ProxyTransaction +ProxyClientTransaction ---------------------- .. figure:: /static/images/sessions/transaction_hierarchy.png :align: center - :alt: ProxyTransaction hierarchy + :alt: ProxyClientTransaction hierarchy -The ProxyTransaction class abstracts the key features of a client transaction. It has a reference to its -paraent ProxySession. One HttpSM is created for each ProxyTransaction. +The ProxyClientTransaction class abstracts the key features of a client transaction. It has a reference to its +paraent ProxyClientSession. One HttpSM is created for each ProxyClientTransaction. There are two concrete subclasses: Http1ClientTransaction and Http2Stream. @@ -70,18 +70,18 @@ HTTP/1.x Objects This diagram shows the relationships between objects created as part of a HTTP/1.x session. A NetVC object performs the basic network level protocols. The Http1ClientSession object has a reference to the -associated NetVC object. The NetVC object is available via the :code:`ProxySession::get_netvc()` method. +associated NetVC object. The NetVC object is available via the :code:`ProxyClientSession::get_netvc()` method. The Http1ClientSession object contains a Http1ClientTransaction objet. For each HTTP request, it calls -the :code:`ProxySession::new_transaction()` method to instantiate the Http1ClientTransaction object. With the HTTP/1.x +the :code:`ProxyClientSession::new_transaction()` method to instantiate the Http1ClientTransaction object. With the HTTP/1.x protocol at most one transaction can be active at a time. -When the Http1ClientTransaction object is instantiated via :code:`ProxyTransaction::new_transaction()` it allocates a +When the Http1ClientTransaction object is instantiated via :code:`ProxyClientTransaction::new_transaction()` it allocates a new HttpSM object, initializes it, and calls :code:`HttpSM::attach_client_session()` to associate the Http1ClientTransaction object with the new HttpSM. -The ProxyTransaction object refers to the HttpSM via the current_reader member variable. The HttpSM object -refers to ProxyTransaction via the ua_session member variable (session in the member name is +The ProxyClientTransaction object refers to the HttpSM via the current_reader member variable. The HttpSM object +refers to ProxyClientTransaction via the ua_session member variable (session in the member name is historical because the HttpSM used to refer directly to the ClientSession object). HTTP/2 Objects @@ -93,11 +93,11 @@ HTTP/2 Objects This diagram shows the relationships between objects created as part of a HTTP/2 session. It is very similar to the HTTP/1.x case. The Http2ClientSession object interacts with the NetVC. The Http2Stream object creates -a HttpSM object object when :code:`ProxySession::new_transaction()` is called. +a HttpSM object object when :code:`ProxyClient::new_transaction()` is called. One difference is that the Http/2 protocol allows for multiple simultaneous transactions, so the Http2ClientSession object must be able to manage multiple streams. From the HttpSM perspective it is interacting with a -ProxyTransaction object, and there is no difference between working with a Http2Stream and a Http1ClientTransaction. +ProxyClientTransaction object, and there is no difference between working with a Http2Stream and a Http1ClientTransaction. Transaction and Session Shutdown ================================ @@ -114,5 +114,5 @@ cause use-after-free and other related memory corruption errors. To ensure that sessions and transactions are correctly shutdown the following assertions are maintained. * The Session object will not call :code:`::destroy()` on itself until all child transaction objects are fully shutdown (i.e. TXN_CLOSE hooks are called and the transaction objects have been freed). -* The Transaction object will not call :code:`::destroy()` on itself until the associated HttpSM has been shutdown. In :code:`HttpSM::kill_this()`, the HttpSM will call :code:`ProxyTransaction::transaction_done()` on the ua_session object. If the user agent initiates the termination, the ProxyTransaction object will send a WRITE_COMPLETE, EOS, or ERROR event on the open VIO object. This should signal to the HttpSM object to shut itself down. +* The Transaction object will not call :code:`::destroy()` on itself until the associated HttpSM has been shutdown. In :code:`HttpSM::kill_this()`, the HttpSM will call :code:`ProxyClientTransaction::transaction_done()` on the ua_session object. If the user agent initiates the termination, the ProxyClientTransaction object will send a WRITE_COMPLETE, EOS, or ERROR event on the open VIO object. This should signal to the HttpSM object to shut itself down. diff --git a/include/tscore/IntrusiveHashMap.h b/include/tscore/IntrusiveHashMap.h index 8a44738a74a..c787e4b358b 100644 --- a/include/tscore/IntrusiveHashMap.h +++ b/include/tscore/IntrusiveHashMap.h @@ -67,15 +67,15 @@ @a ID The numeric type that is the hash value for an instance of @a Key. - Example for @c Http1ServerSession keyed by the origin server IP address. + Example for @c HttpServerSession keyed by the origin server IP address. @code struct Descriptor { - static sockaddr const* key_of(Http1ServerSession const* value) { return &value->ip.sa } + static sockaddr const* key_of(HttpServerSession const* value) { return &value->ip.sa } static bool equal(sockaddr const* lhs, sockaddr const* rhs) { return ats_ip_eq(lhs, rhs); } static uint32_t hash_of(sockaddr const* key) { return ats_ip_hash(key); } - static Http1ServerSession *& next_ptr(Http1ServerSession * ssn) { return ssn->_next; } - static Http1ServerSession *& prev_ptr(Http1ServerSession * ssn) { return ssn->_prev; } + static HttpServerSession *& next_ptr(HttpServerSession * ssn) { return ssn->_next; } + static HttpServerSession *& prev_ptr(HttpServerSession * ssn) { return ssn->_prev; } }; using Table = IntrusiveHashMap; @endcode diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index ac8d661cb4b..0545d6cde9a 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -120,7 +120,7 @@ class Thread ProxyAllocator http1ClientSessionAllocator; ProxyAllocator http2ClientSessionAllocator; ProxyAllocator http2StreamAllocator; - ProxyAllocator http1ServerSessionAllocator; + ProxyAllocator httpServerSessionAllocator; ProxyAllocator hdrHeapAllocator; ProxyAllocator strHeapAllocator; ProxyAllocator cacheVConnectionAllocator; diff --git a/proxy/Makefile.am b/proxy/Makefile.am index 78f31972f3c..0bd93d99822 100644 --- a/proxy/Makefile.am +++ b/proxy/Makefile.am @@ -59,10 +59,10 @@ libproxy_a_SOURCES = \ PluginVC.h \ ProtocolProbeSessionAccept.cc \ ProtocolProbeSessionAccept.h \ - ProxySession.cc \ - ProxySession.h \ - ProxyTransaction.cc \ - ProxyTransaction.h \ + ProxyClientSession.cc \ + ProxyClientSession.h \ + ProxyClientTransaction.cc \ + ProxyClientTransaction \ ReverseProxy.cc \ ReverseProxy.h \ StatPages.cc \ diff --git a/proxy/ProxySession.cc b/proxy/ProxyClientSession.cc similarity index 88% rename from proxy/ProxySession.cc rename to proxy/ProxyClientSession.cc index 6afb04f65ea..214295a7d9c 100644 --- a/proxy/ProxySession.cc +++ b/proxy/ProxyClientSession.cc @@ -1,6 +1,6 @@ /** @file - ProxySession - Base class for protocol client & server sessions. + ProxyClientSession - Base class for protocol client sessions. @section license License @@ -23,17 +23,17 @@ #include "HttpConfig.h" #include "HttpDebugNames.h" -#include "ProxySession.h" +#include "ProxyClientSession.h" static int64_t next_cs_id = 0; -ProxySession::ProxySession() : VConnection(nullptr) +ProxyClientSession::ProxyClientSession() : VConnection(nullptr) { ink_zero(this->user_args); } void -ProxySession::set_session_active() +ProxyClientSession::set_session_active() { if (!m_active) { m_active = true; @@ -42,7 +42,7 @@ ProxySession::set_session_active() } void -ProxySession::clear_session_active() +ProxyClientSession::clear_session_active() { if (m_active) { m_active = false; @@ -51,7 +51,7 @@ ProxySession::clear_session_active() } int64_t -ProxySession::next_connection_id() +ProxyClientSession::next_connection_id() { return ink_atomic_increment(&next_cs_id, 1); } @@ -84,7 +84,7 @@ is_valid_hook(TSHttpHookID hookid) } void -ProxySession::free() +ProxyClientSession::free() { if (schedule_event) { schedule_event->cancel(); @@ -96,7 +96,7 @@ ProxySession::free() } int -ProxySession::state_api_callout(int event, void *data) +ProxyClientSession::state_api_callout(int event, void *data) { Event *e = static_cast(data); if (e == schedule_event) { @@ -124,7 +124,7 @@ ProxySession::state_api_callout(int event, void *data) MUTEX_TRY_LOCK(lock, hook->m_cont->mutex, mutex->thread_holding); // Have a mutex but did't get the lock, reschedule if (!lock.is_locked()) { - SET_HANDLER(&ProxySession::state_api_callout); + SET_HANDLER(&ProxyClientSession::state_api_callout); if (!schedule_event) { // Don't bother to schedule is there is already one out. schedule_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); } @@ -153,7 +153,7 @@ ProxySession::state_api_callout(int event, void *data) } void -ProxySession::do_api_callout(TSHttpHookID id) +ProxyClientSession::do_api_callout(TSHttpHookID id) { ink_assert(id == TS_HTTP_SSN_START_HOOK || id == TS_HTTP_SSN_CLOSE_HOOK); @@ -162,7 +162,7 @@ ProxySession::do_api_callout(TSHttpHookID id) this->api_current = nullptr; if (this->has_hooks()) { - SET_HANDLER(&ProxySession::state_api_callout); + SET_HANDLER(&ProxyClientSession::state_api_callout); this->state_api_callout(EVENT_NONE, nullptr); } else { this->handle_api_return(TS_EVENT_HTTP_CONTINUE); @@ -170,11 +170,11 @@ ProxySession::do_api_callout(TSHttpHookID id) } void -ProxySession::handle_api_return(int event) +ProxyClientSession::handle_api_return(int event) { TSHttpHookID hookid = this->api_hookid; - SET_HANDLER(&ProxySession::state_api_callout); + SET_HANDLER(&ProxyClientSession::state_api_callout); this->api_hookid = TS_HTTP_LAST_HOOK; this->api_scope = API_HOOK_SCOPE_NONE; diff --git a/proxy/ProxySession.h b/proxy/ProxyClientSession.h similarity index 92% rename from proxy/ProxySession.h rename to proxy/ProxyClientSession.h index 609af3e708a..241da9b205f 100644 --- a/proxy/ProxySession.h +++ b/proxy/ProxyClientSession.h @@ -1,6 +1,6 @@ /** @file - ProxySession - Base class for protocol client & server sessions. + ProxyClientSession - Base class for protocol client sessions. @section license License @@ -28,7 +28,7 @@ #include #include "P_Net.h" #include "InkAPIInternal.h" -#include "http/Http1ServerSession.h" +#include "http/HttpServerSession.h" #include "IPAllow.h" // Emit a debug message conditional on whether this particular client session @@ -36,7 +36,7 @@ // member function. #define SsnDebug(ssn, tag, ...) SpecificDebug((ssn)->debug(), tag, __VA_ARGS__) -class ProxyTransaction; +class ProxyClientTransaction; enum class ProxyErrorClass { NONE, @@ -72,11 +72,10 @@ struct ProxyError { // A little ugly, but this global is tracked by traffic_server. extern bool ts_is_draining; -/// Abstract class for HttpSM to interface with any session -class ProxySession : public VConnection +class ProxyClientSession : public VConnection { public: - ProxySession(); + ProxyClientSession(); virtual void destroy() = 0; virtual void free(); @@ -182,7 +181,7 @@ class ProxySession : public VConnection } // Indicate we are done with a transaction. - virtual void release(ProxyTransaction *trans) = 0; + virtual void release(ProxyClientTransaction *trans) = 0; virtual in_port_t get_outbound_port() const @@ -209,11 +208,11 @@ class ProxySession : public VConnection } virtual void - attach_server_session(Http1ServerSession *ssession, bool transaction_done = true) + attach_server_session(HttpServerSession *ssession, bool transaction_done = true) { } - virtual Http1ServerSession * + virtual HttpServerSession * get_server_session() const { return nullptr; @@ -297,8 +296,8 @@ class ProxySession : public VConnection ink_hrtime ssn_last_txn_time = 0; // noncopyable - ProxySession(ProxySession &) = delete; - ProxySession &operator=(const ProxySession &) = delete; + ProxyClientSession(ProxyClientSession &) = delete; + ProxyClientSession &operator=(const ProxyClientSession &) = delete; protected: // XXX Consider using a bitwise flags variable for the following flags, so diff --git a/proxy/ProxyTransaction.cc b/proxy/ProxyClientTransaction.cc similarity index 83% rename from proxy/ProxyTransaction.cc rename to proxy/ProxyClientTransaction.cc index 31947cafc42..c00de079a35 100644 --- a/proxy/ProxyTransaction.cc +++ b/proxy/ProxyClientTransaction.cc @@ -1,6 +1,6 @@ /** @file - ProxyTransaction - Base class for protocol client transactions. + ProxyClientTransaction - Base class for protocol client transactions. @section license License @@ -22,12 +22,12 @@ */ #include "http/HttpSM.h" -#include "http/Http1ServerSession.h" +#include "http/HttpServerSession.h" #include "Plugin.h" #define HttpTxnDebug(fmt, ...) SsnDebug(this, "http_txn", fmt, __VA_ARGS__) -ProxyTransaction::ProxyTransaction() +ProxyClientTransaction::ProxyClientTransaction() : VConnection(nullptr), parent(nullptr), current_reader(nullptr), @@ -38,7 +38,7 @@ ProxyTransaction::ProxyTransaction() } void -ProxyTransaction::new_transaction() +ProxyClientTransaction::new_transaction() { ink_assert(current_reader == nullptr); @@ -62,7 +62,7 @@ ProxyTransaction::new_transaction() } void -ProxyTransaction::release(IOBufferReader *r) +ProxyClientTransaction::release(IOBufferReader *r) { HttpTxnDebug("[%" PRId64 "] session released by sm [%" PRId64 "]", parent ? parent->connection_id() : 0, current_reader ? current_reader->sm_id : 0); @@ -76,20 +76,20 @@ ProxyTransaction::release(IOBufferReader *r) } void -ProxyTransaction::attach_server_session(Http1ServerSession *ssession, bool transaction_done) +ProxyClientTransaction::attach_server_session(HttpServerSession *ssession, bool transaction_done) { parent->attach_server_session(ssession, transaction_done); } void -ProxyTransaction::destroy() +ProxyClientTransaction::destroy() { current_reader = nullptr; this->mutex.clear(); } Action * -ProxyTransaction::adjust_thread(Continuation *cont, int event, void *data) +ProxyClientTransaction::adjust_thread(Continuation *cont, int event, void *data) { NetVConnection *vc = this->get_netvc(); EThread *this_thread = this_ethread(); @@ -104,7 +104,7 @@ ProxyTransaction::adjust_thread(Continuation *cont, int event, void *data) } void -ProxyTransaction::set_rx_error_code(ProxyError e) +ProxyClientTransaction::set_rx_error_code(ProxyError e) { if (this->current_reader) { this->current_reader->t_state.client_info.rx_error_code = e; @@ -112,7 +112,7 @@ ProxyTransaction::set_rx_error_code(ProxyError e) } void -ProxyTransaction::set_tx_error_code(ProxyError e) +ProxyClientTransaction::set_tx_error_code(ProxyError e) { if (this->current_reader) { this->current_reader->t_state.client_info.tx_error_code = e; diff --git a/proxy/ProxyTransaction.h b/proxy/ProxyClientTransaction.h similarity index 92% rename from proxy/ProxyTransaction.h rename to proxy/ProxyClientTransaction.h index c9653242113..ce5d009be9c 100644 --- a/proxy/ProxyTransaction.h +++ b/proxy/ProxyClientTransaction.h @@ -1,6 +1,6 @@ /** @file - ProxyTransaction - Base class for protocol client/server transactions. + ProxyClientTransaction - Base class for protocol client transactions. @section license License @@ -23,17 +23,15 @@ #pragma once -#include "ProxySession.h" +#include "ProxyClientSession.h" #include class HttpSM; -class Http1ServerSession; - -// Abstract Class for any transaction with-in the HttpSM -class ProxyTransaction : public VConnection +class HttpServerSession; +class ProxyClientTransaction : public VConnection { public: - ProxyTransaction(); + ProxyClientTransaction(); // do_io methods implemented by subclasses @@ -49,7 +47,7 @@ class ProxyTransaction : public VConnection virtual void set_inactivity_timeout(ink_hrtime timeout_in) = 0; virtual void cancel_inactivity_timeout() = 0; - virtual void attach_server_session(Http1ServerSession *ssession, bool transaction_done = true); + virtual void attach_server_session(HttpServerSession *ssession, bool transaction_done = true); // See if we need to schedule on the primary thread for the transaction or change the thread that is associated with the VC. // If we reschedule, the scheduled action is returned. Otherwise, NULL is returned @@ -197,14 +195,14 @@ class ProxyTransaction : public VConnection virtual void transaction_done() = 0; - ProxySession * + ProxyClientSession * get_parent() { return parent; } virtual void - set_parent(ProxySession *new_parent) + set_parent(ProxyClientSession *new_parent) { parent = new_parent; host_res_style = parent->host_res_style; @@ -214,7 +212,7 @@ class ProxyTransaction : public VConnection { } - Http1ServerSession * + HttpServerSession * get_server_session() const { return parent ? parent->get_server_session() : nullptr; @@ -268,7 +266,7 @@ class ProxyTransaction : public VConnection virtual void decrement_client_transactions_stat() = 0; protected: - ProxySession *parent; + ProxyClientSession *parent; HttpSM *current_reader; IOBufferReader *sm_reader; diff --git a/proxy/http/Http1ClientSession.cc b/proxy/http/Http1ClientSession.cc index 388e8a4f049..9f1a00224e2 100644 --- a/proxy/http/Http1ClientSession.cc +++ b/proxy/http/Http1ClientSession.cc @@ -32,10 +32,10 @@ #include "tscore/ink_resolver.h" #include "Http1ClientSession.h" -#include "Http1Transaction.h" +#include "Http1ClientTransaction.h" #include "HttpSM.h" #include "HttpDebugNames.h" -#include "Http1ServerSession.h" +#include "HttpServerSession.h" #include "Plugin.h" #define HttpSsnDebug(fmt, ...) SsnDebug(this, "http_cs", fmt, __VA_ARGS__) @@ -160,7 +160,7 @@ Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB ink_assert(lock.is_locked()); // Unique client session identifier. - con_id = ProxySession::next_connection_id(); + con_id = ProxyClientSession::next_connection_id(); schedule_event = nullptr; @@ -448,9 +448,9 @@ Http1ClientSession::reenable(VIO *vio) client_vc->reenable(vio); } -// Called from the Http1Transaction::release +// Called from the Http1ClientTransaction::release void -Http1ClientSession::release(ProxyTransaction *trans) +Http1ClientSession::release(ProxyClientTransaction *trans) { ink_assert(read_state == HCS_ACTIVE_READER || read_state == HCS_INIT); @@ -506,7 +506,7 @@ Http1ClientSession::new_transaction() } void -Http1ClientSession::attach_server_session(Http1ServerSession *ssession, bool transaction_done) +Http1ClientSession::attach_server_session(HttpServerSession *ssession, bool transaction_done) { if (ssession) { ink_assert(bound_ss == nullptr); diff --git a/proxy/http/Http1ClientSession.h b/proxy/http/Http1ClientSession.h index f2581e075d1..ce8346665ce 100644 --- a/proxy/http/Http1ClientSession.h +++ b/proxy/http/Http1ClientSession.h @@ -37,24 +37,23 @@ #include "HTTP.h" #include "HttpConfig.h" #include "IPAllow.h" -#include "ProxySession.h" -#include "Http1Transaction.h" +#include "ProxyClientSession.h" +#include "Http1ClientTransaction.h" #ifdef USE_HTTP_DEBUG_LISTS extern ink_mutex debug_cs_list_mutex; #endif class HttpSM; -class Http1ServerSession; +class HttpServerSession; -/// Class to manage a Http v1 session to a client -class Http1ClientSession : public ProxySession +class Http1ClientSession : public ProxyClientSession { public: - typedef ProxySession super; ///< Parent type. + typedef ProxyClientSession super; ///< Parent type. Http1ClientSession(); - // Implement ProxySession interface. + // Implement ProxyClientSession interface. void destroy() override; void free() override; void release_transaction(); @@ -121,11 +120,11 @@ class Http1ClientSession : public ProxySession } // Indicate we are done with a transaction - void release(ProxyTransaction *trans) override; + void release(ProxyClientTransaction *trans) override; - void attach_server_session(Http1ServerSession *ssession, bool transaction_done = true) override; + void attach_server_session(HttpServerSession *ssession, bool transaction_done = true) override; - Http1ServerSession * + HttpServerSession * get_server_session() const override { return bound_ss; @@ -200,7 +199,7 @@ class Http1ClientSession : public ProxySession VIO *ka_vio; VIO *slave_ka_vio; - Http1ServerSession *bound_ss; + HttpServerSession *bound_ss; int released_transactions; @@ -213,7 +212,7 @@ class Http1ClientSession : public ProxySession /// Transparently pass-through non-HTTP traffic. bool f_transparent_passthrough; - Http1Transaction trans; + Http1ClientTransaction trans; }; extern ClassAllocator http1ClientSessionAllocator; diff --git a/proxy/http/Http1Transaction.cc b/proxy/http/Http1ClientTransaction.cc similarity index 85% rename from proxy/http/Http1Transaction.cc rename to proxy/http/Http1ClientTransaction.cc index a9ee93e4bae..983ee029c99 100644 --- a/proxy/http/Http1Transaction.cc +++ b/proxy/http/Http1ClientTransaction.cc @@ -1,6 +1,6 @@ /** @file - Http1Transaction.cc - The Transaction class for Http1* + Http1ClientTransaction.cc - The Transaction class for Http1* @section license License @@ -21,12 +21,12 @@ limitations under the License. */ -#include "Http1Transaction.h" +#include "Http1ClientTransaction.h" #include "Http1ClientSession.h" #include "HttpSM.h" void -Http1Transaction::release(IOBufferReader *r) +Http1ClientTransaction::release(IOBufferReader *r) { // Must set this inactivity count here rather than in the session because the state machine // is not availble then @@ -47,7 +47,7 @@ Http1Transaction::release(IOBufferReader *r) } void -Http1Transaction::set_parent(ProxySession *new_parent) +Http1ClientTransaction::set_parent(ProxyClientSession *new_parent) { parent = new_parent; Http1ClientSession *http1_parent = dynamic_cast(new_parent); @@ -61,7 +61,7 @@ Http1Transaction::set_parent(ProxySession *new_parent) } void -Http1Transaction::transaction_done() +Http1ClientTransaction::transaction_done() { if (parent) { static_cast(parent)->release_transaction(); @@ -69,7 +69,7 @@ Http1Transaction::transaction_done() } bool -Http1Transaction::allow_half_open() const +Http1ClientTransaction::allow_half_open() const { bool config_allows_it = (current_reader) ? current_reader->t_state.txn_conf->allow_half_open > 0 : true; if (config_allows_it) { @@ -80,13 +80,13 @@ Http1Transaction::allow_half_open() const } void -Http1Transaction::increment_client_transactions_stat() +Http1ClientTransaction::increment_client_transactions_stat() { HTTP_INCREMENT_DYN_STAT(http_current_client_transactions_stat); } void -Http1Transaction::decrement_client_transactions_stat() +Http1ClientTransaction::decrement_client_transactions_stat() { HTTP_DECREMENT_DYN_STAT(http_current_client_transactions_stat); } diff --git a/proxy/http/Http1Transaction.h b/proxy/http/Http1ClientTransaction.h similarity index 89% rename from proxy/http/Http1Transaction.h rename to proxy/http/Http1ClientTransaction.h index bd8040f676f..c0dbab48ef4 100644 --- a/proxy/http/Http1Transaction.h +++ b/proxy/http/Http1ClientTransaction.h @@ -1,6 +1,6 @@ /** @file - Http1Transaction.h - The Transaction class for Http1* + Http1ClientTransaction.h - The Transaction class for Http1* @section license License @@ -23,16 +23,16 @@ #pragma once -#include "../ProxyTransaction.h" +#include "../ProxyClientTransaction.h" class Continuation; -/// Concrete class for any Http1 Transaction -class Http1Transaction : public ProxyTransaction + +class Http1ClientTransaction : public ProxyClientTransaction { public: - using super_type = ProxyTransaction; + using super_type = ProxyClientTransaction; - Http1Transaction() {} + Http1ClientTransaction() {} // Implement VConnection interface. VIO * do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override @@ -53,7 +53,7 @@ class Http1Transaction : public ProxyTransaction } // Don't destroy your elements. Rely on the Http1ClientSession to clean up the - // Http1Transaction class as necessary. The super::destroy() clears the + // Http1ClientTransaction class as necessary. The super::destroy() clears the // mutex, which Http1ClientSession owns. void destroy() override @@ -83,7 +83,7 @@ class Http1Transaction : public ProxyTransaction bool allow_half_open() const override; - void set_parent(ProxySession *new_parent) override; + void set_parent(ProxyClientSession *new_parent) override; bool is_outbound_transparent() const override diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 05fa35c3742..5b0fa9afd88 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -22,12 +22,12 @@ */ -#include "../ProxyTransaction.h" +#include "../ProxyClientTransaction.h" #include "HttpSM.h" #include "HttpTransact.h" #include "HttpTransactHeaders.h" #include "ProxyConfig.h" -#include "Http1ServerSession.h" +#include "HttpServerSession.h" #include "HttpDebugNames.h" #include "HttpSessionManager.h" #include "P_Cache.h" @@ -443,7 +443,7 @@ HttpSM::start_sub_sm() } void -HttpSM::attach_client_session(ProxyTransaction *client_vc, IOBufferReader *buffer_reader) +HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader *buffer_reader) { milestones[TS_MILESTONE_UA_BEGIN] = Thread::get_hrtime(); ink_assert(client_vc != nullptr); @@ -638,7 +638,7 @@ HttpSM::state_read_client_request_header(int event, void *data) // Reset the inactivity timeout if this is the first // time we've been called. The timeout had been set to - // the accept timeout by the ProxyTransaction + // the accept timeout by the ProxyClientTransaction // if ((ua_buffer_reader->read_avail() > 0) && (client_request_hdr_bytes == 0)) { milestones[TS_MILESTONE_UA_FIRST_READ] = Thread::get_hrtime(); @@ -1713,14 +1713,14 @@ HttpSM::state_http_server_open(int event, void *data) pending_action = nullptr; } milestones[TS_MILESTONE_SERVER_CONNECT_END] = Thread::get_hrtime(); - Http1ServerSession *session; + HttpServerSession *session; NetVConnection *netvc = nullptr; switch (event) { case NET_EVENT_OPEN: { session = (TS_SERVER_SESSION_SHARING_POOL_THREAD == t_state.http_config_param->server_session_sharing_pool) ? - THREAD_ALLOC_INIT(http1ServerSessionAllocator, mutex->thread_holding) : - http1ServerSessionAllocator.alloc(); + THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : + httpServerSessionAllocator.alloc(); session->sharing_pool = static_cast(t_state.http_config_param->server_session_sharing_pool); session->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); @@ -4882,7 +4882,7 @@ HttpSM::do_http_server_open(bool raw) // session when we already have an attached server session. else if ((TS_SERVER_SESSION_SHARING_MATCH_NONE == t_state.txn_conf->server_session_sharing_match || is_private()) && (ua_txn != nullptr)) { - Http1ServerSession *existing_ss = ua_txn->get_server_session(); + HttpServerSession *existing_ss = ua_txn->get_server_session(); if (existing_ss) { // [amc] Not sure if this is the best option, but we don't get here unless session sharing is disabled @@ -4909,7 +4909,7 @@ HttpSM::do_http_server_open(bool raw) // to get a new one. // ua_txn is null when t_state.req_flavor == REQ_FLAVOR_SCHEDULED_UPDATE else if (ua_txn != nullptr) { - Http1ServerSession *existing_ss = ua_txn->get_server_session(); + HttpServerSession *existing_ss = ua_txn->get_server_session(); if (existing_ss) { existing_ss->get_netvc()->set_inactivity_timeout(HRTIME_SECONDS(t_state.txn_conf->keep_alive_no_activity_timeout_out)); existing_ss->release(); @@ -5879,7 +5879,7 @@ HttpSM::write_header_into_buffer(HTTPHdr *h, MIOBuffer *b) } void -HttpSM::attach_server_session(Http1ServerSession *s) +HttpSM::attach_server_session(HttpServerSession *s) { hsm_release_assert(server_session == nullptr); hsm_release_assert(server_entry == nullptr); @@ -7947,7 +7947,7 @@ HttpSM::is_private() if (server_session) { res = server_session->private_session; } else if (ua_txn) { - Http1ServerSession *ss = ua_txn->get_server_session(); + HttpServerSession *ss = ua_txn->get_server_session(); if (ss) { res = ss->private_session; } else if (will_be_private_ss) { diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index 6b55386e006..21f707170d8 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -39,7 +39,7 @@ #include "UrlRewrite.h" #include "HttpTunnel.h" #include "InkAPIInternal.h" -#include "../ProxyTransaction.h" +#include "../ProxyClientTransaction.h" #include "HdrUtils.h" #include #include "tscore/History.h" @@ -59,7 +59,7 @@ static size_t const HTTP_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ // the larger buffer size static size_t const HTTP_SERVER_RESP_HDR_BUFFER_INDEX = BUFFER_SIZE_INDEX_8K; -class Http1ServerSession; +class HttpServerSession; class AuthHttpAdapter; class HttpSM; @@ -215,22 +215,22 @@ class HttpSM : public Continuation void init(); - void attach_client_session(ProxyTransaction *client_vc_arg, IOBufferReader *buffer_reader); + void attach_client_session(ProxyClientTransaction *client_vc_arg, IOBufferReader *buffer_reader); // Called by httpSessionManager so that we can reset // the session timeouts and initiate a read while // holding the lock for the server session - void attach_server_session(Http1ServerSession *s); + void attach_server_session(HttpServerSession *s); // Used to read attributes of // the current active server session - Http1ServerSession * + HttpServerSession * get_server_session() { return server_session; } - ProxyTransaction * + ProxyClientTransaction * get_ua_txn() { return ua_txn; @@ -350,7 +350,7 @@ class HttpSM : public Continuation void remove_ua_entry(); public: - ProxyTransaction *ua_txn = nullptr; + ProxyClientTransaction *ua_txn = nullptr; BackgroundFill_t background_fill = BACKGROUND_FILL_NONE; // AuthHttpAdapter authAdapter; void set_http_schedule(Continuation *); @@ -362,8 +362,8 @@ class HttpSM : public Continuation IOBufferReader *ua_buffer_reader = nullptr; IOBufferReader *ua_raw_buffer_reader = nullptr; - HttpVCTableEntry *server_entry = nullptr; - Http1ServerSession *server_session = nullptr; + HttpVCTableEntry *server_entry = nullptr; + HttpServerSession *server_session = nullptr; /* Because we don't want to take a session from a shared pool if we know that it will be private, * but we cannot set it to private until we have an attached server session. diff --git a/proxy/http/Http1ServerSession.cc b/proxy/http/HttpServerSession.cc similarity index 86% rename from proxy/http/Http1ServerSession.cc rename to proxy/http/HttpServerSession.cc index 3c2361a0441..9c77ac3eceb 100644 --- a/proxy/http/Http1ServerSession.cc +++ b/proxy/http/HttpServerSession.cc @@ -23,7 +23,7 @@ /**************************************************************************** - Http1ServerSession.cc + HttpServerSession.cc Description: @@ -32,15 +32,15 @@ #include "tscore/BufferWriter.h" #include "tscore/bwf_std_format.h" #include "tscore/Allocator.h" -#include "Http1ServerSession.h" +#include "HttpServerSession.h" #include "HttpSessionManager.h" #include "HttpSM.h" static int64_t next_ss_id = (int64_t)0; -ClassAllocator http1ServerSessionAllocator("http1ServerSessionAllocator"); +ClassAllocator httpServerSessionAllocator("httpServerSessionAllocator"); void -Http1ServerSession::destroy() +HttpServerSession::destroy() { ink_release_assert(server_vc == nullptr); ink_assert(read_buffer); @@ -53,14 +53,14 @@ Http1ServerSession::destroy() mutex.clear(); if (TS_SERVER_SESSION_SHARING_POOL_THREAD == sharing_pool) { - THREAD_FREE(this, http1ServerSessionAllocator, this_thread()); + THREAD_FREE(this, httpServerSessionAllocator, this_thread()); } else { - http1ServerSessionAllocator.free(this); + httpServerSessionAllocator.free(this); } } void -Http1ServerSession::new_connection(NetVConnection *new_vc) +HttpServerSession::new_connection(NetVConnection *new_vc) { ink_assert(new_vc != nullptr); server_vc = new_vc; @@ -85,7 +85,7 @@ Http1ServerSession::new_connection(NetVConnection *new_vc) } void -Http1ServerSession::enable_outbound_connection_tracking(OutboundConnTrack::Group *group) +HttpServerSession::enable_outbound_connection_tracking(OutboundConnTrack::Group *group) { ink_assert(nullptr == conn_track_group); conn_track_group = group; @@ -97,25 +97,25 @@ Http1ServerSession::enable_outbound_connection_tracking(OutboundConnTrack::Group } VIO * -Http1ServerSession::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) +HttpServerSession::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) { return server_vc ? server_vc->do_io_read(c, nbytes, buf) : nullptr; } VIO * -Http1ServerSession::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) +HttpServerSession::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) { return server_vc ? server_vc->do_io_write(c, nbytes, buf, owner) : nullptr; } void -Http1ServerSession::do_io_shutdown(ShutdownHowTo_t howto) +HttpServerSession::do_io_shutdown(ShutdownHowTo_t howto) { server_vc->do_io_shutdown(howto); } void -Http1ServerSession::do_io_close(int alerrno) +HttpServerSession::do_io_close(int alerrno) { ts::LocalBufferWriter<256> w; bool debug_p = is_debug_tag_set("http_ss"); @@ -160,17 +160,17 @@ Http1ServerSession::do_io_close(int alerrno) } void -Http1ServerSession::reenable(VIO *vio) +HttpServerSession::reenable(VIO *vio) { server_vc->reenable(vio); } -// void Http1ServerSession::release() +// void HttpServerSession::release() // // Releases the session for K-A reuse // void -Http1ServerSession::release() +HttpServerSession::release() { Debug("http_ss", "Releasing session, private_session=%d, sharing_match=%d", private_session, sharing_match); // Set our state to KA for stat issues diff --git a/proxy/http/Http1ServerSession.h b/proxy/http/HttpServerSession.h similarity index 80% rename from proxy/http/Http1ServerSession.h rename to proxy/http/HttpServerSession.h index fd8e25511f1..03ff0150031 100644 --- a/proxy/http/Http1ServerSession.h +++ b/proxy/http/HttpServerSession.h @@ -23,7 +23,7 @@ /**************************************************************************** - Http1ServerSession.h + HttpServerSession.h Description: @@ -53,16 +53,14 @@ enum { HTTP_SS_MAGIC_DEAD = 0xDEADFEED, }; -/// Class to manage a Http v1 session to a server -// TODO: inherit from ProxySession -class Http1ServerSession : public VConnection +class HttpServerSession : public VConnection { - using self_type = Http1ServerSession; + using self_type = HttpServerSession; using super_type = VConnection; public: - Http1ServerSession() : super_type(nullptr) {} - Http1ServerSession(self_type const &) = delete; + HttpServerSession() : super_type(nullptr) {} + HttpServerSession(self_type const &) = delete; self_type &operator=(self_type const &) = delete; void destroy(); @@ -151,8 +149,8 @@ class Http1ServerSession : public VConnection static sockaddr const *key_of(self_type const *ssn); static bool equal(sockaddr const *lhs, sockaddr const *rhs); // Add a couple overloads for internal convenience. - static bool equal(sockaddr const *lhs, Http1ServerSession const *rhs); - static bool equal(Http1ServerSession const *lhs, sockaddr const *rhs); + static bool equal(sockaddr const *lhs, HttpServerSession const *rhs); + static bool equal(HttpServerSession const *lhs, sockaddr const *rhs); } _ip_link; /// Hash map descriptor class for FQDN map. @@ -201,86 +199,86 @@ class Http1ServerSession : public VConnection IOBufferReader *buf_reader = nullptr; }; -extern ClassAllocator http1ServerSessionAllocator; +extern ClassAllocator httpServerSessionAllocator; // --- Implementation --- inline void -Http1ServerSession::attach_hostname(const char *hostname) +HttpServerSession::attach_hostname(const char *hostname) { if (CRYPTO_HASH_ZERO == hostname_hash) { CryptoContext().hash_immediate(hostname_hash, (unsigned char *)hostname, strlen(hostname)); } } -inline Http1ServerSession *& -Http1ServerSession::IPLinkage::next_ptr(self_type *ssn) +inline HttpServerSession *& +HttpServerSession::IPLinkage::next_ptr(self_type *ssn) { return ssn->_ip_link._next; } -inline Http1ServerSession *& -Http1ServerSession::IPLinkage::prev_ptr(self_type *ssn) +inline HttpServerSession *& +HttpServerSession::IPLinkage::prev_ptr(self_type *ssn) { return ssn->_ip_link._prev; } inline uint32_t -Http1ServerSession::IPLinkage::hash_of(sockaddr const *key) +HttpServerSession::IPLinkage::hash_of(sockaddr const *key) { return ats_ip_hash(key); } inline sockaddr const * -Http1ServerSession::IPLinkage::key_of(self_type const *ssn) +HttpServerSession::IPLinkage::key_of(self_type const *ssn) { return &ssn->get_server_ip().sa; } inline bool -Http1ServerSession::IPLinkage::equal(sockaddr const *lhs, sockaddr const *rhs) +HttpServerSession::IPLinkage::equal(sockaddr const *lhs, sockaddr const *rhs) { return ats_ip_addr_port_eq(lhs, rhs); } inline bool -Http1ServerSession::IPLinkage::equal(sockaddr const *lhs, Http1ServerSession const *rhs) +HttpServerSession::IPLinkage::equal(sockaddr const *lhs, HttpServerSession const *rhs) { return ats_ip_addr_port_eq(lhs, key_of(rhs)); } inline bool -Http1ServerSession::IPLinkage::equal(Http1ServerSession const *lhs, sockaddr const *rhs) +HttpServerSession::IPLinkage::equal(HttpServerSession const *lhs, sockaddr const *rhs) { return ats_ip_addr_port_eq(key_of(lhs), rhs); } -inline Http1ServerSession *& -Http1ServerSession::FQDNLinkage::next_ptr(self_type *ssn) +inline HttpServerSession *& +HttpServerSession::FQDNLinkage::next_ptr(self_type *ssn) { return ssn->_fqdn_link._next; } -inline Http1ServerSession *& -Http1ServerSession::FQDNLinkage::prev_ptr(self_type *ssn) +inline HttpServerSession *& +HttpServerSession::FQDNLinkage::prev_ptr(self_type *ssn) { return ssn->_fqdn_link._prev; } inline uint64_t -Http1ServerSession::FQDNLinkage::hash_of(CryptoHash const &key) +HttpServerSession::FQDNLinkage::hash_of(CryptoHash const &key) { return key.fold(); } inline CryptoHash const & -Http1ServerSession::FQDNLinkage::key_of(self_type *ssn) +HttpServerSession::FQDNLinkage::key_of(self_type *ssn) { return ssn->hostname_hash; } inline bool -Http1ServerSession::FQDNLinkage::equal(CryptoHash const &lhs, CryptoHash const &rhs) +HttpServerSession::FQDNLinkage::equal(CryptoHash const &lhs, CryptoHash const &rhs) { return lhs == rhs; } diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index 33033b2d48d..d95a79decdc 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -31,8 +31,8 @@ ****************************************************************************/ #include "HttpSessionManager.h" -#include "../ProxySession.h" -#include "Http1ServerSession.h" +#include "../ProxyClientSession.h" +#include "HttpServerSession.h" #include "HttpSM.h" #include "HttpDebugNames.h" @@ -57,13 +57,13 @@ ServerSessionPool::purge() { // @c do_io_close can free the instance which clears the intrusive links and breaks the iterator. // Therefore @c do_io_close is called on a post-incremented iterator. - m_ip_pool.apply([](Http1ServerSession *ssn) -> void { ssn->do_io_close(); }); + m_ip_pool.apply([](HttpServerSession *ssn) -> void { ssn->do_io_close(); }); m_ip_pool.clear(); m_fqdn_pool.clear(); } bool -ServerSessionPool::match(Http1ServerSession *ss, sockaddr const *addr, CryptoHash const &hostname_hash, +ServerSessionPool::match(HttpServerSession *ss, sockaddr const *addr, CryptoHash const &hostname_hash, TSServerSessionSharingMatchType match_style) { return TS_SERVER_SESSION_SHARING_MATCH_NONE != @@ -93,7 +93,7 @@ ServerSessionPool::validate_sni(HttpSM *sm, NetVConnection *netvc) HSMresult_t ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostname_hash, - TSServerSessionSharingMatchType match_style, HttpSM *sm, Http1ServerSession *&to_return) + TSServerSessionSharingMatchType match_style, HttpSM *sm, HttpServerSession *&to_return) { HSMresult_t zret = HSM_NOT_FOUND; to_return = nullptr; @@ -144,7 +144,7 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna } void -ServerSessionPool::releaseSession(Http1ServerSession *ss) +ServerSessionPool::releaseSession(HttpServerSession *ss) { ss->state = HSS_KA_SHARED; // Now we need to issue a read on the connection to detect @@ -176,7 +176,7 @@ int ServerSessionPool::eventHandler(int event, void *data) { NetVConnection *net_vc = nullptr; - Http1ServerSession *s = nullptr; + HttpServerSession *s = nullptr; switch (event) { case VC_EVENT_READ_READY: @@ -275,9 +275,9 @@ HttpSessionManager::purge_keepalives() HSMresult_t HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockaddr const *ip, const char *hostname, - ProxyTransaction *ua_txn, HttpSM *sm) + ProxyClientTransaction *ua_txn, HttpSM *sm) { - Http1ServerSession *to_return = nullptr; + HttpServerSession *to_return = nullptr; TSServerSessionSharingMatchType match_style = static_cast(sm->t_state.txn_conf->server_session_sharing_match); CryptoHash hostname_hash; @@ -371,7 +371,7 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad } HSMresult_t -HttpSessionManager::release_session(Http1ServerSession *to_release) +HttpSessionManager::release_session(HttpServerSession *to_release) { EThread *ethread = this_ethread(); ServerSessionPool *pool = diff --git a/proxy/http/HttpSessionManager.h b/proxy/http/HttpSessionManager.h index 10930d58cc0..f83196aad0e 100644 --- a/proxy/http/HttpSessionManager.h +++ b/proxy/http/HttpSessionManager.h @@ -33,10 +33,10 @@ #pragma once #include "P_EventSystem.h" -#include "Http1ServerSession.h" +#include "HttpServerSession.h" #include "tscore/IntrusiveHashMap.h" -class ProxyTransaction; +class ProxyClientTransaction; class HttpSM; void initialize_thread_for_http_sessions(EThread *thread, int thread_index); @@ -67,13 +67,13 @@ class ServerSessionPool : public Continuation static bool validate_sni(HttpSM *sm, NetVConnection *netvc); protected: - using IPTable = IntrusiveHashMap; - using FQDNTable = IntrusiveHashMap; + using IPTable = IntrusiveHashMap; + using FQDNTable = IntrusiveHashMap; public: /** Check if a session matches address and host name. */ - static bool match(Http1ServerSession *ss, sockaddr const *addr, CryptoHash const &host_hash, + static bool match(HttpServerSession *ss, sockaddr const *addr, CryptoHash const &host_hash, TSServerSessionSharingMatchType match_style); /** Get a session from the pool. @@ -84,10 +84,10 @@ class ServerSessionPool : public Continuation @return A pointer to the session or @c NULL if not matching session was found. */ HSMresult_t acquireSession(sockaddr const *addr, CryptoHash const &host_hash, TSServerSessionSharingMatchType match_style, - HttpSM *sm, Http1ServerSession *&server_session); + HttpSM *sm, HttpServerSession *&server_session); /** Release a session to to pool. */ - void releaseSession(Http1ServerSession *ss); + void releaseSession(HttpServerSession *ss); /// Close all sessions and then clear the table. void purge(); @@ -103,8 +103,9 @@ class HttpSessionManager public: HttpSessionManager() : m_g_pool(nullptr) {} ~HttpSessionManager() {} - HSMresult_t acquire_session(Continuation *cont, sockaddr const *addr, const char *hostname, ProxyTransaction *ua_txn, HttpSM *sm); - HSMresult_t release_session(Http1ServerSession *to_release); + HSMresult_t acquire_session(Continuation *cont, sockaddr const *addr, const char *hostname, ProxyClientTransaction *ua_txn, + HttpSM *sm); + HSMresult_t release_session(HttpServerSession *to_release); void purge_keepalives(); void init(); int main_handler(int event, void *data); diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 73d5e123889..1f5375dc37f 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -39,7 +39,7 @@ #include "RemapPluginInfo.h" #include "UrlMapping.h" #include "records/I_RecHttp.h" -#include "ProxySession.h" +#include "ProxyClientSession.h" #define HTTP_RELEASE_ASSERT(X) ink_release_assert(X) diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 9e0d05202ba..22199d17082 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -932,7 +932,7 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) // the amount to read since we know it. We will forward the FIN // to the server on VC_EVENT_WRITE_COMPLETE. if (p->vc_type == HT_HTTP_CLIENT) { - ProxyTransaction *ua_vc = static_cast(p->vc); + ProxyClientTransaction *ua_vc = static_cast(p->vc); if (ua_vc->get_half_close_flag()) { c_write = c->buffer_reader->read_avail(); p->alive = false; diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am index 7b5abd7707d..6f5733229b0 100644 --- a/proxy/http/Makefile.am +++ b/proxy/http/Makefile.am @@ -46,8 +46,8 @@ libhttp_a_SOURCES = \ HttpCacheSM.h \ Http1ClientSession.cc \ Http1ClientSession.h \ - Http1Transaction.cc \ - Http1Transaction.h \ + Http1ClientTransaction.cc \ + Http1ClientTransaction.h \ HttpConfig.cc \ HttpConfig.h \ HttpConnectionCount.cc \ @@ -60,8 +60,8 @@ libhttp_a_SOURCES = \ HttpProxyServerMain.h \ HttpSM.cc \ HttpSM.h \ - Http1ServerSession.cc \ - Http1ServerSession.h \ + HttpServerSession.cc \ + HttpServerSession.h \ HttpSessionManager.cc \ HttpSessionManager.h \ HttpTransact.cc \ diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index dafa43ee97c..a48ad3d260c 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -179,7 +179,7 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT, new_vc->mutex->thread_holding); // Unique client session identifier. - this->con_id = ProxySession::next_connection_id(); + this->con_id = ProxyClientSession::next_connection_id(); this->client_vc = new_vc; client_vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); this->schedule_event = nullptr; diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 06e62622bb8..5f22c7bfe48 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -25,7 +25,7 @@ #include "HTTP2.h" #include "Plugin.h" -#include "ProxySession.h" +#include "ProxyClientSession.h" #include "Http2ConnectionState.h" #include #include "tscore/ink_inet.h" @@ -158,15 +158,15 @@ class Http2Frame IOBufferReader *ioreader; }; -class Http2ClientSession : public ProxySession +class Http2ClientSession : public ProxyClientSession { public: - typedef ProxySession super; ///< Parent type. + typedef ProxyClientSession super; ///< Parent type. typedef int (Http2ClientSession::*SessionHandler)(int, void *); Http2ClientSession(); - // Implement ProxySession interface. + // Implement ProxyClientSession interface. void start() override; void destroy() override; void free() override; @@ -225,7 +225,7 @@ class Http2ClientSession : public ProxySession } void - release(ProxyTransaction *trans) override + release(ProxyClientTransaction *trans) override { } diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index cb9ca8ffc11..a753d0db9dc 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -24,7 +24,7 @@ #pragma once #include "HTTP2.h" -#include "../ProxyTransaction.h" +#include "../ProxyClientTransaction.h" #include "Http2DebugNames.h" #include "../http/HttpTunnel.h" // To get ChunkedHandler #include "Http2DependencyTree.h" @@ -35,10 +35,10 @@ class Http2ConnectionState; typedef Http2DependencyTree::Tree DependencyTree; -class Http2Stream : public ProxyTransaction +class Http2Stream : public ProxyClientTransaction { public: - typedef ProxyTransaction super; ///< Parent type. + typedef ProxyClientTransaction super; ///< Parent type. Http2Stream(Http2StreamId sid = 0, ssize_t initial_rwnd = Http2::initial_window_size) : client_rwnd(initial_rwnd), _id(sid) { SET_HANDLER(&Http2Stream::main_event_handler); diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index e214548a081..2b348cb7755 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -37,9 +37,9 @@ #include "URL.h" #include "MIME.h" #include "HTTP.h" -#include "ProxySession.h" +#include "ProxyClientSession.h" #include "Http2ClientSession.h" -#include "Http1ServerSession.h" +#include "HttpServerSession.h" #include "HttpSM.h" #include "HttpConfig.h" #include "P_Net.h" @@ -4721,7 +4721,7 @@ TSHttpSsnHookAdd(TSHttpSsn ssnp, TSHttpHookID id, TSCont contp) sdk_assert(sdk_sanity_check_continuation(contp) == TS_SUCCESS); sdk_assert(sdk_sanity_check_hook_id(id) == TS_SUCCESS); - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); cs->ssn_hook_append(id, (INKContInternal *)contp); } @@ -4730,28 +4730,28 @@ TSHttpSsnTransactionCount(TSHttpSsn ssnp) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); return cs->get_transact_count(); } TSVConn TSHttpSsnClientVConnGet(TSHttpSsn ssnp) { - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); return reinterpret_cast(cs->get_netvc()); } TSVConn TSHttpSsnServerVConnGet(TSHttpSsn ssnp) { - Http1ServerSession *ss = reinterpret_cast(ssnp); + HttpServerSession *ss = reinterpret_cast(ssnp); return reinterpret_cast(ss->get_netvc()); } class TSHttpSsnCallback : public Continuation { public: - TSHttpSsnCallback(ProxySession *cs, TSEvent event) : Continuation(cs->mutex), m_cs(cs), m_event(event) + TSHttpSsnCallback(ProxyClientSession *cs, TSEvent event) : Continuation(cs->mutex), m_cs(cs), m_event(event) { SET_HANDLER(&TSHttpSsnCallback::event_handler); } @@ -4765,7 +4765,7 @@ class TSHttpSsnCallback : public Continuation } private: - ProxySession *m_cs; + ProxyClientSession *m_cs; TSEvent m_event; }; @@ -4774,8 +4774,8 @@ TSHttpSsnReenable(TSHttpSsn ssnp, TSEvent event) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - ProxySession *cs = reinterpret_cast(ssnp); - EThread *eth = this_ethread(); + ProxyClientSession *cs = reinterpret_cast(ssnp); + EThread *eth = this_ethread(); // If this function is being executed on a thread created by the API // which is DEDICATED, the continuation needs to be called back on a @@ -5475,7 +5475,7 @@ TSHttpTxnTransformRespGet(TSHttpTxn txnp, TSMBuffer *bufp, TSMLoc *obj) sockaddr const * TSHttpSsnClientAddrGet(TSHttpSsn ssnp) { - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); if (cs == nullptr) { return nullptr; @@ -5494,7 +5494,7 @@ TSHttpTxnClientAddrGet(TSHttpTxn txnp) sockaddr const * TSHttpSsnIncomingAddrGet(TSHttpSsn ssnp) { - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); if (cs == nullptr) { return nullptr; @@ -5517,7 +5517,7 @@ TSHttpTxnOutgoingAddrGet(TSHttpTxn txnp) HttpSM *sm = reinterpret_cast(txnp); - Http1ServerSession *ssn = sm->get_server_session(); + HttpServerSession *ssn = sm->get_server_session(); if (ssn == nullptr) { return nullptr; } @@ -5635,7 +5635,7 @@ TSHttpTxnServerPacketMarkSet(TSHttpTxn txnp, int mark) // change the mark on an active server session if (nullptr != sm->ua_txn) { - Http1ServerSession *ssn = sm->ua_txn->get_server_session(); + HttpServerSession *ssn = sm->ua_txn->get_server_session(); if (nullptr != ssn) { NetVConnection *vc = ssn->get_netvc(); if (vc != nullptr) { @@ -5677,7 +5677,7 @@ TSHttpTxnServerPacketTosSet(TSHttpTxn txnp, int tos) // change the tos on an active server session if (nullptr != sm->ua_txn) { - Http1ServerSession *ssn = sm->ua_txn->get_server_session(); + HttpServerSession *ssn = sm->ua_txn->get_server_session(); if (nullptr != ssn) { NetVConnection *vc = ssn->get_netvc(); if (vc != nullptr) { @@ -5719,7 +5719,7 @@ TSHttpTxnServerPacketDscpSet(TSHttpTxn txnp, int dscp) // change the tos on an active server session if (nullptr != sm->ua_txn) { - Http1ServerSession *ssn = sm->ua_txn->get_server_session(); + HttpServerSession *ssn = sm->ua_txn->get_server_session(); if (nullptr != ssn) { NetVConnection *vc = ssn->get_netvc(); if (vc != nullptr) { @@ -6073,7 +6073,7 @@ TSHttpSsnArgSet(TSHttpSsn ssnp, int arg_idx, void *arg) sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); sdk_assert(arg_idx >= 0 && arg_idx < TS_HTTP_MAX_USER_ARG); - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); cs->set_user_arg(arg_idx, arg); } @@ -6084,7 +6084,7 @@ TSHttpSsnArgGet(TSHttpSsn ssnp, int arg_idx) sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); sdk_assert(arg_idx >= 0 && arg_idx < TS_HTTP_MAX_USER_ARG); - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); return cs->get_user_arg(arg_idx); } @@ -6219,14 +6219,14 @@ void TSHttpSsnDebugSet(TSHttpSsn ssnp, int on) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - (reinterpret_cast(ssnp))->set_debug(0 != on); + (reinterpret_cast(ssnp))->set_debug(0 != on); } int TSHttpSsnDebugGet(TSHttpSsn ssnp) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - return (reinterpret_cast(ssnp))->debug(); + return (reinterpret_cast(ssnp))->debug(); } int @@ -7387,8 +7387,8 @@ TSHttpSsnClientFdGet(TSHttpSsn ssnp, int *fdp) { sdk_assert(sdk_sanity_check_null_ptr((void *)fdp) == TS_SUCCESS); - VConnection *basecs = reinterpret_cast(ssnp); - ProxySession *cs = dynamic_cast(basecs); + VConnection *basecs = reinterpret_cast(ssnp); + ProxyClientSession *cs = dynamic_cast(basecs); if (cs == nullptr) { return TS_ERROR; @@ -7421,7 +7421,7 @@ TSHttpTxnServerFdGet(TSHttpTxn txnp, int *fdp) HttpSM *sm = reinterpret_cast(txnp); *fdp = -1; - Http1ServerSession *ss = sm->get_server_session(); + HttpServerSession *ss = sm->get_server_session(); if (ss == nullptr) { return TS_ERROR; } @@ -7789,7 +7789,7 @@ TSFetchRespHdrMLocGet(TSFetchSM fetch_sm) int TSHttpSsnIsInternal(TSHttpSsn ssnp) { - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); if (!cs) { return 0; @@ -9277,7 +9277,7 @@ int64_t TSHttpSsnIdGet(TSHttpSsn ssnp) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); return cs->connection_id(); } @@ -9307,8 +9307,8 @@ TSHttpSsnClientProtocolStackGet(TSHttpSsn ssnp, int n, const char **result, int { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); sdk_assert(n == 0 || result != nullptr); - ProxySession *cs = reinterpret_cast(ssnp); - int count = 0; + ProxyClientSession *cs = reinterpret_cast(ssnp); + int count = 0; if (cs && n > 0) { auto mem = static_cast(alloca(sizeof(std::string_view) * n)); count = cs->populate_protocol(mem, n); @@ -9340,7 +9340,7 @@ const char * TSHttpSsnClientProtocolStackContains(TSHttpSsn ssnp, const char *tag) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - ProxySession *cs = reinterpret_cast(ssnp); + ProxyClientSession *cs = reinterpret_cast(ssnp); return cs->protocol_contains(std::string_view{tag}); } diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 2c09bf18451..7aba78d17ca 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -76,7 +76,7 @@ extern "C" int plock(int); #include "ProxyConfig.h" #include "HttpProxyServerMain.h" #include "HttpBodyFactory.h" -#include "ProxySession.h" +#include "ProxyClientSession.h" #include "logging/Log.h" #include "CacheControl.h" #include "IPAllow.h" From 4fdc01c5c8e595c521e983eacdfa9d744f15b895 Mon Sep 17 00:00:00 2001 From: dyrock Date: Wed, 27 Feb 2019 16:46:09 -0600 Subject: [PATCH 324/526] Remove extra vars to bwprint in SSLConfigParams::getCTX --- iocore/net/SSLConfig.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 51855b45fa3..dd4edcfcb93 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -668,7 +668,7 @@ SSLConfigParams::getCTX(const char *client_cert, const char *key_file, const cha CTX_MAP *ctx_map = nullptr; std::string top_level_key, ctx_key; ts::bwprint(top_level_key, "{}:{}", ca_bundle_file, ca_bundle_path); - ts::bwprint(ctx_key, "{}:{}", client_cert, key_file, ca_bundle_file, ca_bundle_path); + ts::bwprint(ctx_key, "{}:{}", client_cert, key_file); ink_mutex_acquire(&ctxMapLock); // Do first level searching and create new CTX_MAP as second level if not exists. From 9c4bfa34d69d1017c47dad3712be7136162f1f24 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Thu, 21 Feb 2019 17:07:07 +0000 Subject: [PATCH 325/526] Merge url_scheme_get into only calling class method --- proxy/hdrs/URL.cc | 18 ------------------ proxy/hdrs/URL.h | 20 +++++++++++++++++--- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 8f94329796d..dccc527408c 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -684,24 +684,6 @@ url_string_get_buf(URLImpl *url, char *dstbuf, int dstbuf_size, int *length) return buf; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -const char * -url_scheme_get(URLImpl *url, int *length) -{ - const char *str; - - if (url->m_scheme_wks_idx >= 0) { - str = hdrtoken_index_to_wks(url->m_scheme_wks_idx); - *length = hdrtoken_index_to_length(url->m_scheme_wks_idx); - } else { - str = url->m_ptr_scheme; - *length = url->m_len_scheme; - } - return str; -} - /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/hdrs/URL.h b/proxy/hdrs/URL.h index 31abaa1b40e..1524a5a332c 100644 --- a/proxy/hdrs/URL.h +++ b/proxy/hdrs/URL.h @@ -173,7 +173,6 @@ char *url_string_get_ref(HdrHeap *heap, URLImpl *url, int *length); void url_called_set(URLImpl *url); char *url_string_get_buf(URLImpl *url, char *dstbuf, int dstbuf_size, int *length); -const char *url_scheme_get(URLImpl *url, int *length); void url_CryptoHash_get(const URLImpl *url, CryptoHash *hash, cache_generation_t generation = -1); void url_host_CryptoHash_get(URLImpl *url, CryptoHash *hash); const char *url_scheme_set(HdrHeap *heap, URLImpl *url, const char *value, int value_wks_idx, int length, bool copy_string); @@ -249,6 +248,7 @@ class URL : public HdrHeapSDKHandle void host_hash_get(CryptoHash *hash); const char *scheme_get(int *length); + const std::string_view scheme_get(); int scheme_get_wksidx(); void scheme_set(const char *value, int length); @@ -444,11 +444,25 @@ URL::host_hash_get(CryptoHash *hash) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +inline const std::string_view +URL::scheme_get() +{ + ink_assert(valid()); + + if (m_url_impl->m_scheme_wks_idx >= 0) { + return std::string_view{hdrtoken_index_to_wks(m_url_impl->m_scheme_wks_idx), + static_cast(hdrtoken_index_to_length(m_url_impl->m_scheme_wks_idx))}; + } else { + return std::string_view{m_url_impl->m_ptr_scheme, m_url_impl->m_len_scheme}; + } +} + inline const char * URL::scheme_get(int *length) { - ink_assert(valid()); - return (url_scheme_get(m_url_impl, length)); + std::string_view ret = this->scheme_get(); + *length = ret.size(); + return ret.data(); } inline int From f78dcdc78ff8d7f4cae7b588c743c2a11ce2176f Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 28 Feb 2019 10:56:44 -0600 Subject: [PATCH 326/526] MIME: Fix line_is_real false positive. --- proxy/hdrs/MIME.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/proxy/hdrs/MIME.cc b/proxy/hdrs/MIME.cc index d130b202feb..8d223242ca1 100644 --- a/proxy/hdrs/MIME.cc +++ b/proxy/hdrs/MIME.cc @@ -2442,14 +2442,14 @@ MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, b } // adjust out arguments. + output_shares_input = true; if (PARSE_RESULT_CONT != zret) { if (!m_line.empty()) { output = m_line; m_line.resize(0); // depending resize(0) not deallocating internal string memory. output_shares_input = false; } else { - output = parsed_text; - output_shares_input = true; + output = parsed_text; } } @@ -2569,8 +2569,8 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char ////////////////////////////////////////////////////////////////////// if (must_copy_strings || (!line_is_real)) { - char *dup = heap->duplicate_str(parsed.data(), parsed.size()); - intptr_t delta = dup - parsed.data(); + char *dup = heap->duplicate_str(parsed.data(), parsed.size()); + ptrdiff_t delta = dup - parsed.data(); field_name.assign(field_name.data() + delta, field_name.size()); field_value.assign(field_value.data() + delta, field_value.size()); } From 33400052698c0c8bd1787654171173bf8bbcfcd6 Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Thu, 28 Feb 2019 15:24:56 -0600 Subject: [PATCH 327/526] set thread affinity to current thread if the current thread type is the same as the target thread --- iocore/eventsystem/P_UnixEventProcessor.h | 10 ++++++++-- .../cont_schedule/gold/schedule_on_pool.gold | 3 +-- tests/tools/plugins/cont_schedule.cc | 8 ++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/iocore/eventsystem/P_UnixEventProcessor.h b/iocore/eventsystem/P_UnixEventProcessor.h index 003d425e37e..b4dea2e9340 100644 --- a/iocore/eventsystem/P_UnixEventProcessor.h +++ b/iocore/eventsystem/P_UnixEventProcessor.h @@ -70,8 +70,14 @@ EventProcessor::schedule(Event *e, EventType etype, bool fast_signal) if (ethread != nullptr && ethread->is_event_type(etype)) { e->ethread = ethread; } else { - e->ethread = assign_thread(etype); - if (ethread == nullptr) { + ethread = this_ethread(); + // Is the current thread eligible? + if (ethread != nullptr && ethread->is_event_type(etype)) { + e->ethread = ethread; + } else { + e->ethread = assign_thread(etype); + } + if (e->continuation->getThreadAffinity() == nullptr) { e->continuation->setThreadAffinity(e->ethread); } } diff --git a/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold b/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold index 42328e0dcd2..df9cf85eaca 100644 --- a/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold +++ b/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold @@ -3,7 +3,6 @@ ``ET_NET``TSContScheduleOnPool handler 1`` ``(TSContSchedule_test.check) pass [should not be the same thread] ``ET_TASK``TSContScheduleOnPool handler 2`` -``(TSContSchedule_test.check) pass [should not be the same thread] ``ET_TASK``TSContScheduleOnPool handler 2`` -``(TSContSchedule_test.check) pass [should not be the same thread] +``(TSContSchedule_test.check) pass [should be the same thread] `` diff --git a/tests/tools/plugins/cont_schedule.cc b/tests/tools/plugins/cont_schedule.cc index abc8066de5c..e678029f659 100644 --- a/tests/tools/plugins/cont_schedule.cc +++ b/tests/tools/plugins/cont_schedule.cc @@ -103,7 +103,7 @@ TSContScheduleOnPool_handler_1(TSCont contp, TSEvent event, void *edata) } else { TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); } - check_thread = TSThreadSelf(); + check_thread = nullptr; } return 0; } @@ -115,10 +115,10 @@ TSContScheduleOnPool_handler_2(TSCont contp, TSEvent event, void *edata) if (check_thread == nullptr) { check_thread = TSThreadSelf(); } else { - if (check_thread != TSThreadSelf()) { - TSDebug(DEBUG_TAG_CHK, "pass [should not be the same thread]"); + if (check_thread == TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should be the same thread]"); } else { - TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); + TSDebug(DEBUG_TAG_CHK, "fail [not the same thread]"); } check_thread = TSThreadSelf(); } From a105fd3d177c7c4223609b09014bff372f0c7d0e Mon Sep 17 00:00:00 2001 From: Evan Zelkowitz Date: Tue, 26 Feb 2019 19:47:07 -0700 Subject: [PATCH 328/526] Add support for the old lua formatted ease of use conventions and the numeric log rolling values Addresses issue #5080 --- proxy/logging/YamlLogConfig.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc index 948c5a630cd..8988458f805 100644 --- a/proxy/logging/YamlLogConfig.cc +++ b/proxy/logging/YamlLogConfig.cc @@ -99,7 +99,9 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) return true; } -TsEnumDescriptor ROLLING_MODE = {{{"none", 0}, {"time", 1}, {"size", 2}, {"both", 3}, {"any", 4}}}; +TsEnumDescriptor ROLLING_MODE_TEXT = {{{"none", 0}, {"time", 1}, {"size", 2}, {"both", 3}, {"any", 4}}}; +TsEnumDescriptor ROLLING_MODE_LUA = { + {{"log.roll.none", 0}, {"log.roll.time", 1}, {"log.roll.size", 2}, {"log.roll.both", 3}, {"log.roll.any", 4}}}; std::set valid_log_object_keys = { "filename", "format", "mode", "header", "rolling_enabled", "rolling_interval_sec", @@ -153,9 +155,15 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) if (node["rolling_enabled"]) { auto value = node["rolling_enabled"].as(); - obj_rolling_enabled = ROLLING_MODE.get(value); + obj_rolling_enabled = ROLLING_MODE_TEXT.get(value); if (obj_rolling_enabled < 0) { - throw YAML::ParserException(node["rolling_enabled"].Mark(), "unknown value " + value); + obj_rolling_enabled = ROLLING_MODE_LUA.get(value); + if (obj_rolling_enabled < 0) { + obj_rolling_enabled = node["rolling_enabled"].as(); + if (obj_rolling_enabled < Log::NO_ROLLING || obj_rolling_enabled > Log::ROLL_ON_TIME_AND_SIZE) { + throw YAML::ParserException(node["rolling_enabled"].Mark(), "unknown value " + value); + } + } } } if (node["rolling_interval_sec"]) { From 9a93757ad4eafa51c93143afe331a4840c14167b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstro=CC=88m?= Date: Fri, 1 Mar 2019 20:05:17 -0300 Subject: [PATCH 329/526] tscore/eventnotify: change fcntl include path The default include path for `fnctl.h` changed some time in the 90's. --- src/tscore/EventNotify.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tscore/EventNotify.cc b/src/tscore/EventNotify.cc index dc1e1521573..a18decc3021 100644 --- a/src/tscore/EventNotify.cc +++ b/src/tscore/EventNotify.cc @@ -33,7 +33,7 @@ #ifdef HAVE_EVENTFD #include -#include +#include #include #endif From e0c6dc12eb77d32e8e7e40a0d34588250f9499eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstro=CC=88m?= Date: Fri, 1 Mar 2019 19:59:36 -0300 Subject: [PATCH 330/526] tscore/ink_hrtime: add missing cstdint import On alpine linux 3.9 (and edge) the compilation was failing because of a missing import. Refs: https://github.com/apache/trafficserver/issues/5103 --- include/tscore/ink_hrtime.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/tscore/ink_hrtime.h b/include/tscore/ink_hrtime.h index a1405a98c74..f321476913a 100644 --- a/include/tscore/ink_hrtime.h +++ b/include/tscore/ink_hrtime.h @@ -33,6 +33,7 @@ #include "tscore/ink_config.h" #include "tscore/ink_assert.h" #include +#include #include #include typedef int64_t ink_hrtime; From 656f25a10b6aea68f4bb141c8e4cbbe2c73fd9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstro=CC=88m?= Date: Fri, 1 Mar 2019 20:13:51 -0300 Subject: [PATCH 331/526] traffic_layout/engine: add missing stat import We were using `mode_t` without it being available on some architectures. --- src/traffic_layout/engine.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/traffic_layout/engine.h b/src/traffic_layout/engine.h index 769b6e71ca9..c79a186d648 100644 --- a/src/traffic_layout/engine.h +++ b/src/traffic_layout/engine.h @@ -23,6 +23,7 @@ #pragma once +#include #include "tscore/ArgParser.h" #include From 2a334d83af31a93fb75d1eae50288cf73d44ba29 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 5 Mar 2019 20:11:35 +0000 Subject: [PATCH 332/526] Remove duplicate calls to TSHttpTxnReenable in xdebug --- plugins/xdebug/xdebug.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/xdebug/xdebug.cc b/plugins/xdebug/xdebug.cc index 29751e6bd36..c6eefd033db 100644 --- a/plugins/xdebug/xdebug.cc +++ b/plugins/xdebug/xdebug.cc @@ -540,7 +540,6 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) } } } - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); return TS_EVENT_NONE; }; TSHttpTxnHookAdd(txn, TS_HTTP_SEND_REQUEST_HDR_HOOK, TSContCreate(send_req_dump, nullptr)); @@ -553,7 +552,6 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) if (TSHttpTxnServerRespGet(txn, &buffer, &hdr) == TS_SUCCESS) { log_headers(txn, buffer, hdr, "ServerResponse"); } - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); return TS_EVENT_NONE; }; TSHttpTxnHookAdd(txn, TS_HTTP_READ_RESPONSE_HDR_HOOK, TSContCreate(read_resp_dump, nullptr)); From f202f8497f4a854b17a9711ad51559c5c89642ea Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 5 Mar 2019 17:12:00 +0000 Subject: [PATCH 333/526] Fixing a previous fix to fully allocate strings when heap mismatch detected. --- proxy/hdrs/HTTP.cc | 2 +- proxy/hdrs/HdrHeap.h | 20 ++++++++++++++++++++ proxy/hdrs/URL.cc | 15 +++++++++++++++ proxy/hdrs/URL.h | 1 + 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index eebf4ad8741..74e79df1e6b 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -708,7 +708,7 @@ http_hdr_url_set(HdrHeap *heap, HTTPHdrImpl *hh, URLImpl *url) int url_string_length = url->strings_length(); heap->m_read_write_heap = new_HdrStrHeap(url_string_length); } - hh->u.req.m_url_impl->move_strings(heap->m_read_write_heap.get()); + hh->u.req.m_url_impl->rehome_strings(heap); } else { hh->u.req.m_url_impl = url; } diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index 98113626bc4..1435f4b391a 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -235,6 +235,26 @@ class HdrHeap } } + // Working function to copy strings into a new heap + // Unlike the HDR_MOVE_STR macro, this function will call + // allocate_str which will update the new_heap to create more space + // if there is not originally sufficient space + inline std::string_view + localize(const std::string_view &string) + { + auto length = string.length(); + if (length > 0) { + char *new_str = this->allocate_str(length); + if (new_str) { + memcpy(new_str, string.data(), length); + } else { + length = 0; + } + return {new_str, length}; + } + return {nullptr, 0}; + } + // Sanity Check Functions void sanity_check_strs(); bool check_marshalled(uint32_t buf_length); diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index dccc527408c..6137add2df6 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -348,6 +348,21 @@ URLImpl::unmarshal(intptr_t offset) // HDR_UNMARSHAL_STR(m_ptr_printed_string, offset); } +void +URLImpl::rehome_strings(HdrHeap *new_heap) +{ + m_ptr_scheme = new_heap->localize({m_ptr_scheme, m_len_scheme}).data(); + m_ptr_user = new_heap->localize({m_ptr_user, m_len_user}).data(); + m_ptr_password = new_heap->localize({m_ptr_password, m_len_password}).data(); + m_ptr_host = new_heap->localize({m_ptr_host, m_len_host}).data(); + m_ptr_port = new_heap->localize({m_ptr_port, m_len_port}).data(); + m_ptr_path = new_heap->localize({m_ptr_path, m_len_path}).data(); + m_ptr_params = new_heap->localize({m_ptr_params, m_len_params}).data(); + m_ptr_query = new_heap->localize({m_ptr_query, m_len_query}).data(); + m_ptr_fragment = new_heap->localize({m_ptr_fragment, m_len_fragment}).data(); + m_ptr_printed_string = new_heap->localize({m_ptr_printed_string, m_len_printed_string}).data(); +} + void URLImpl::move_strings(HdrStrHeap *new_heap) { diff --git a/proxy/hdrs/URL.h b/proxy/hdrs/URL.h index 1524a5a332c..a98d24c1d84 100644 --- a/proxy/hdrs/URL.h +++ b/proxy/hdrs/URL.h @@ -80,6 +80,7 @@ struct URLImpl : public HdrHeapObjImpl { int marshal(MarshalXlate *str_xlate, int num_xlate); void unmarshal(intptr_t offset); void move_strings(HdrStrHeap *new_heap); + void rehome_strings(HdrHeap *new_heap); size_t strings_length(); // Sanity Check Functions From ca609e5931703fb6664422b145a4cc1e7dfbaf87 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Thu, 28 Feb 2019 16:28:55 -0800 Subject: [PATCH 334/526] Override delete in Extendible --- include/tscore/Extendible.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/tscore/Extendible.h b/include/tscore/Extendible.h index 16c9c874493..c6a1920d835 100644 --- a/include/tscore/Extendible.h +++ b/include/tscore/Extendible.h @@ -105,6 +105,8 @@ template struct Extendible { Extendible(Extendible &) = delete; /** allocate a new object with additional field data */ void *operator new(size_t size); + /** free the object */ + void operator delete(void *ptr); /** construct all fields */ Extendible() { schema.call_construct(this_as_char_ptr()); } /** destruct all fields */ @@ -550,6 +552,15 @@ Extendible::operator new(size_t size) return ptr; } +/// free the object +template +void +Extendible::operator delete(void *ptr) +{ + ats_free(ptr); + ink_release_assert(ptr != nullptr); +} + // private template char * From 773108b0469770691a750e4ed42b7b66cd41678a Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 4 Mar 2019 15:37:43 -0600 Subject: [PATCH 335/526] Fix #5094: Fix use after free in test_IntrusiveHashMap.cc --- src/tscore/unit_tests/test_IntrusiveHashMap.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tscore/unit_tests/test_IntrusiveHashMap.cc b/src/tscore/unit_tests/test_IntrusiveHashMap.cc index a0521a27a21..5e615c0e8d1 100644 --- a/src/tscore/unit_tests/test_IntrusiveHashMap.cc +++ b/src/tscore/unit_tests/test_IntrusiveHashMap.cc @@ -90,9 +90,10 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") map.insert(new Thing("dave")); map.insert(new Thing("persia")); REQUIRE(map.count() == 3); - for (auto &thing : map) { - delete &thing; - } + // Need to be bit careful cleaning up, since the link pointers are in the objects and deleting + // the object makes it unsafe to use an iterator referencing that object. For a full cleanup, + // the best option is to first delete everything, then clean up the map. + map.apply([](Thing *thing) { delete thing; }); map.clear(); REQUIRE(map.count() == 0); From 82e1497cc8bf8d8648d303881c60378369043134 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 4 Mar 2019 15:45:11 -0600 Subject: [PATCH 336/526] Fix #5093: new/delete mismatch in test_IntrusivePtr.cc. --- src/tscore/unit_tests/test_IntrusivePtr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tscore/unit_tests/test_IntrusivePtr.cc b/src/tscore/unit_tests/test_IntrusivePtr.cc index c2fcf7e8190..ffacbc6a615 100644 --- a/src/tscore/unit_tests/test_IntrusivePtr.cc +++ b/src/tscore/unit_tests/test_IntrusivePtr.cc @@ -28,7 +28,7 @@ struct Thing : public ts::IntrusivePtrCounter { Thing() { ++_count; } - ~Thing() { --_count; } + virtual ~Thing() { --_count; } std::string _name; static int _count; // instance count. }; From 012d437f54daedaf0cc6d67d2d15f836c38d0bf6 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Fri, 15 Feb 2019 22:45:25 +0000 Subject: [PATCH 337/526] Implement aud claim in Uri Signing Plugin The Aud claim is implemented as per the RFC version 16 that can be found here:https://tools.ietf.org/html/draft-ietf-cdni-uri-signing-16 As per the specification, the aud claim can be either a JSON array or a string. The aud claim is stored as raw json in the jwt class in this implementation. It is converted either to an array or a string at validation time. This commit also expands the unit tests quite a bit. Test configs can be provided in the unit_tests directory and parsed in the test framework. JWS validation is also testable now. This commit also fixes two memory leaks 1. Issuers were never being freed on configuration cleanup. 2. Token renewal allocates a tmp json_object without freeing. --- plugins/experimental/uri_signing/config.c | 25 +++ plugins/experimental/uri_signing/config.h | 1 + plugins/experimental/uri_signing/jwt.c | 57 +++++- plugins/experimental/uri_signing/jwt.h | 3 +- plugins/experimental/uri_signing/parse.c | 5 + .../uri_signing/unit_tests/testConfig.config | 102 ++++++++++ .../unit_tests/uri_signing_test.cc | 179 ++++++++++++++++++ 7 files changed, 364 insertions(+), 8 deletions(-) create mode 100644 plugins/experimental/uri_signing/unit_tests/testConfig.config diff --git a/plugins/experimental/uri_signing/config.c b/plugins/experimental/uri_signing/config.c index b52b94470b3..96429148e90 100644 --- a/plugins/experimental/uri_signing/config.c +++ b/plugins/experimental/uri_signing/config.c @@ -45,6 +45,7 @@ struct config { char **issuer_names; struct signer signer; struct auth_directive *auth_directives; + char *id; }; cjose_jwk_t ** @@ -80,6 +81,12 @@ find_key_by_kid(struct config *cfg, const char *issuer, const char *kid) return NULL; } +const char * +config_get_id(struct config *cfg) +{ + return cfg->id; +} + struct config * config_new(size_t n) { @@ -105,6 +112,7 @@ config_new(size_t n) cfg->signer.alg = NULL; cfg->auth_directives = NULL; + cfg->id = NULL; PluginDebug("New config object created at %p", cfg); return cfg; @@ -117,6 +125,7 @@ config_delete(struct config *cfg) return; } hdestroy_r(cfg->issuers); + free(cfg->issuers); for (cjose_jwk_t ***jwkis = cfg->jwkis; *jwkis; ++jwkis) { for (cjose_jwk_t **jwks = *jwkis; *jwks; ++jwks) { @@ -126,6 +135,10 @@ config_delete(struct config *cfg) } free(cfg->jwkis); + if (cfg->id) { + free(cfg->id); + } + for (char **name = cfg->issuer_names; *name; ++name) { free(*name); } @@ -259,6 +272,18 @@ read_config(const char *path) renewal_kid = json_string_value(renewal_kid_json); } + json_t *id_json = json_object_get(jwks, "id"); + const char *id; + if (id_json) { + id = json_string_value(id_json); + if (id) { + cfg->id = malloc(strlen(id) + 1); + strcpy(cfg->id, id); + PluginDebug("Found Id in the config: %s", cfg->id); + } + } + json_decref(id_json); + size_t jwks_ct = json_array_size(key_ary); cjose_jwk_t **jwks = (*jwkis++ = malloc((jwks_ct + 1) * sizeof *jwks)); PluginDebug("Created table with size %d", cfg->issuers->size); diff --git a/plugins/experimental/uri_signing/config.h b/plugins/experimental/uri_signing/config.h index 75a82f24d88..a22ec5d273d 100644 --- a/plugins/experimental/uri_signing/config.h +++ b/plugins/experimental/uri_signing/config.h @@ -33,3 +33,4 @@ struct signer *config_signer(struct config *); struct _cjose_jwk_int **find_keys(struct config *cfg, const char *issuer); struct _cjose_jwk_int *find_key_by_kid(struct config *cfg, const char *issuer, const char *kid); bool uri_matches_auth_directive(struct config *cfg, const char *uri, size_t uri_ct); +const char *config_get_id(struct config *cfg); diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index 69a07e3714c..aeaa218424f 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -56,7 +56,7 @@ parse_jwt(json_t *raw) jwt->raw = raw; jwt->iss = json_string_value(json_object_get(raw, "iss")); jwt->sub = json_string_value(json_object_get(raw, "sub")); - jwt->aud = json_string_value(json_object_get(raw, "aud")); + jwt->aud = json_object_get(raw, "aud"); jwt->exp = parse_number(json_object_get(raw, "exp")); jwt->nbf = parse_number(json_object_get(raw, "nbf")); jwt->iat = parse_number(json_object_get(raw, "iat")); @@ -77,6 +77,8 @@ jwt_delete(struct jwt *jwt) if (!jwt) { return; } + + json_decref(jwt->aud); json_decref(jwt->raw); free(jwt); } @@ -110,11 +112,6 @@ jwt_validate(struct jwt *jwt) return false; } - if (!unsupported_string_claim(jwt->aud)) { - PluginDebug("Initial JWT Failure: aud unsupported"); - return false; - } - if (now() > jwt->exp) { PluginDebug("Initial JWT Failure: expired token"); return false; @@ -153,6 +150,43 @@ jwt_validate(struct jwt *jwt) return true; } +bool +jwt_check_aud(json_t *aud, const char *id) +{ + if (!aud) { + return true; + } + if (!id) { + return false; + } + /* If aud is a string, do a simple string comparison */ + const char *aud_str = json_string_value(aud); + if (aud_str) { + PluginDebug("Checking aud %s agaisnt token aud string \"%s\"", id, aud_str); + /* Both strings will be null terminated per jansson docs */ + if (strcmp(aud_str, id) == 0) { + return true; + } + return false; + } + PluginDebug("Checking aud %s agaisnt token aud array", id); + /* If aud is an array, check all items */ + if (json_is_array(aud)) { + size_t index; + json_t *aud_item; + json_array_foreach(aud, index, aud_item) + { + aud_str = json_string_value(aud_item); + if (aud_str) { + if (strcmp(aud_str, id) == 0) { + return true; + } + } + } + } + return false; +} + bool jwt_check_uri(const char *cdniuc, const char *uri) { @@ -213,6 +247,14 @@ renew_copy_string(json_t *new_json, const char *name, const char *old) } } +void +renew_copy_raw(json_t *new_json, const char *name, json_t *old_json) +{ + if (old_json) { + json_object_set_new(new_json, name, old_json); + } +} + void renew_copy_real(json_t *new_json, const char *name, double old) { @@ -245,7 +287,7 @@ renew(struct jwt *jwt, const char *iss, cjose_jwk_t *jwk, const char *alg, const json_t *new_json = json_object(); renew_copy_string(new_json, "iss", iss); /* use issuer of new signing key */ renew_copy_string(new_json, "sub", jwt->sub); - renew_copy_string(new_json, "aud", jwt->aud); + renew_copy_raw(new_json, "aud", jwt->aud); renew_copy_real(new_json, "exp", now() + jwt->cdniets); /* expire ets seconds hence */ renew_copy_real(new_json, "nbf", jwt->nbf); renew_copy_real(new_json, "iat", now()); /* issued now */ @@ -255,6 +297,7 @@ renew(struct jwt *jwt, const char *iss, cjose_jwk_t *jwk, const char *alg, const renew_copy_integer(new_json, "cdnistt", jwt->cdnistt); char *pt = json_dumps(new_json, JSON_COMPACT); + json_decref(new_json); cjose_header_t *hdr = cjose_header_new(NULL); if (!hdr) { diff --git a/plugins/experimental/uri_signing/jwt.h b/plugins/experimental/uri_signing/jwt.h index 5e09f028e7c..95efbbc1da0 100644 --- a/plugins/experimental/uri_signing/jwt.h +++ b/plugins/experimental/uri_signing/jwt.h @@ -23,7 +23,7 @@ struct jwt { json_t *raw; const char *iss; const char *sub; - const char *aud; + json_t *aud; double exp; double nbf; double iat; @@ -39,6 +39,7 @@ struct jwt { struct jwt *parse_jwt(json_t *raw); void jwt_delete(struct jwt *jwt); bool jwt_validate(struct jwt *jwt); +bool jwt_check_aud(json_t *aud, const char *id); bool jwt_check_uri(const char *cdniuc, const char *uri); struct _cjose_jwk_int; diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index 099acdb315f..43667659922 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -229,6 +229,11 @@ validate_jws(cjose_jws_t *jws, struct config *cfg, const char *uri, size_t uri_c } } + if (!jwt_check_aud(jwt->aud, config_get_id(cfg))) { + PluginDebug("Valid key for %16p that does not match aud.", jws); + goto jwt_fail; + } + if (!jwt_check_uri(jwt->cdniuc, uri)) { PluginDebug("Valid key for %16p that does not match uri.", jws); goto jwt_fail; diff --git a/plugins/experimental/uri_signing/unit_tests/testConfig.config b/plugins/experimental/uri_signing/unit_tests/testConfig.config new file mode 100644 index 00000000000..aeecf363434 --- /dev/null +++ b/plugins/experimental/uri_signing/unit_tests/testConfig.config @@ -0,0 +1,102 @@ +{ + "Master Issuer": { + "renewal_kid": "6", + "id": "tester", + "auth_directives": [ + { + "auth": "allow", + "uri": "regex:invalid" + } + ], + "keys": [ + { + "alg": "HS256", + "k": "nxb7fyO5Z2hGz9E3oKm1357ptvC2su5QwQUb4YaIaIc", + "kid": "0", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "cXKukBqFvQ0n3WAuRnWfExC14dmHdGoJULoZjGu9tJC", + "kid": "1", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "38pJlSXfX87jWL0a03luml9QzUmM4qts1nmfIHA3B7r", + "kid": "2", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "zNQPphknDGvzR5kA7IonXIDWKMyB1b8NpGmmDNlpgtM", + "kid": "3", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "iB2ogCmQRt7r5hW7pgyP5FqiFcCl53MPQvfXv8wrZAn", + "kid": "4", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "GJMCTyZhNoSOZvUOKmmY9MtGSLaONNLHqtKwsC3MWKo", + "kid": "5", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "u2LziZKJFBnOfjUQUmvot7C9t91jj7ocJPIU9aDdbUl", + "kid": "6", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "DRBKrBh87NYkH3UzfW1tWbiXCYXiYGZUE9w1orZngL0", + "kid": "7", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "KNNKFbun8lEs7GbiKlo9mYGNdvpt33tdFzHbNnasDyP", + "kid": "8", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "yb6kOddMUdupPRSkWMUdE6jrWT4MqUnVyTjpeJBYIqp", + "kid": "9", + "kty": "oct" + } + ] + }, + "Second Issuer": { + "keys": [ + { + "alg": "HS256", + "k": "testkey1", + "kid": "one", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "testkey2", + "kid": "two", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "testkey3", + "kid": "three", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "testkey4", + "kid": "four", + "kty": "oct" + } + ] + } +} diff --git a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc index f39758e097c..fe938160ccf 100644 --- a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc +++ b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc @@ -30,6 +30,7 @@ extern "C" { #include "../normalize.h" #include "../parse.h" #include "../match.h" +#include "../config.h" } bool @@ -475,4 +476,182 @@ TEST_CASE("5", "[RegexTests]") } SECTION("Alternation") { REQUIRE(match_regex("cat|dog", "dog")); } + fprintf(stderr, "\n"); +} + +TEST_CASE("6", "[AudTests]") +{ + INFO("TEST 6, Test Aud Matching"); + + json_error_t *err = NULL; + SECTION("Standard aud string match") + { + json_t *raw = json_loads("{\"aud\": \"tester\"}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Standard aud array match") + { + json_t *raw = json_loads("{\"aud\": [ \"foo\", \"bar\", \"tester\"]}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Standard aud string mismatch") + { + json_t *raw = json_loads("{\"aud\": \"foo\"}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Standard aud array mismatch") + { + json_t *raw = json_loads("{\"aud\": [\"foo\", \"bar\", \"foobar\"]}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Integer trying to pass as an aud") + { + json_t *raw = json_loads("{\"aud\": 1}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Integer mixed into a passing aud array") + { + json_t *raw = json_loads("{\"aud\": [1, \"foo\", \"bar\", \"tester\"]}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Case sensitive test for single string") + { + json_t *raw = json_loads("{\"aud\": \"TESTer\"}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Case sensitive test for array") + { + json_t *raw = json_loads("{\"aud\": [1, \"foo\", \"bar\", \"Tester\"]}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + fprintf(stderr, "\n"); +} + +TEST_CASE("7", "[TestsConfig]") +{ + INFO("TEST 7, Config Loading and Config Functions"); + + SECTION("Config Loading ID Field") + { + struct config *cfg = read_config("experimental/uri_signing/unit_tests/testConfig.config"); + REQUIRE(cfg != NULL); + REQUIRE(strcmp(config_get_id(cfg), "tester") == 0); + config_delete(cfg); + } + fprintf(stderr, "\n"); +} + +bool +jws_validation_helper(const char *url, const char *package, struct config *cfg) +{ + size_t url_ct = strlen(url); + size_t strip_ct = 0; + char uri_strip[url_ct + 1]; + memset(uri_strip, 0, sizeof uri_strip); + cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, uri_strip, url_ct, &strip_ct); + if (!jws) { + return false; + } + struct jwt *jwt = validate_jws(jws, cfg, uri_strip, strip_ct); + if (jwt) { + jwt_delete(jwt); + cjose_jws_release(jws); + return true; + } + cjose_jws_release(jws); + return false; +} + +TEST_CASE("8", "[TestsWithConfig]") +{ + INFO("TEST 8, Tests Involving Validation with Config"); + struct config *cfg = read_config("experimental/uri_signing/unit_tests/testConfig.config"); + + SECTION("Validation of Valid Aud String in JWS") + { + REQUIRE(jws_validation_helper("http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6InRlc3RlciIsImNkbml1YyI6" + "InJlZ2V4Omh0dHA6Ly93d3cuZm9vYmFyLmNvbS8qIn0.InBxVm6OOAglNqc-U5wAZaRQVebJ9PK7Y9i7VFHWYHU", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + SECTION("Validation of Invalid Aud String in JWS") + { + REQUIRE(!jws_validation_helper("http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6ImJhZCIsImNkbml1YyI6InJ" + "lZ2V4Omh0dHA6Ly93d3cuZm9vYmFyLmNvbS8qIn0.aCOo8gOBa5G1RKkkzgWYwc79dPRw_fQUC0k1sWcjkyM", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + SECTION("Validation of Valid Aud Array in JWS") + { + REQUIRE(jws_validation_helper( + "http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6WyJiYWQiLCJpbnZhbGlkIiwidGVzdGVyIl0sImNkbml1YyI6InJl" + "Z2V4Omh0dHA6Ly93d3cuZm9vYmFyLmNvbS8qIn0.7lyepZMzc_odieKvOTN2U-k1gLwRKS8KJIvDFQXDqGs", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + SECTION("Validation of Invalid Aud Array in JWS") + { + REQUIRE(!jws_validation_helper( + "http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6WyJiYWQiLCJpbnZhbGlkIiwiZm9vYmFyIl0sImNkbml1YyI6InJl" + "Z2V4Omh0dHA6Ly93d3cuZm9vYmFyLmNvbS8qIn0.CU3WMJAPs0uRC7NKXvatVG9uU9SANdZzqO0GdQUatxk", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + SECTION("Validation of Valid Aud Array Mixed types in JWS") + { + REQUIRE(jws_validation_helper( + "http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6WyJiYWQiLDEsImZvb2JhciIsInRlc3RlciJdLCJjZG5pdWMiOiJy" + "ZWdleDpodHRwOi8vd3d3LmZvb2Jhci5jb20vKiJ9._vlXsA3r7RPje2ZdMnpaGTwIsdNMjuQWPEHRkGKTVL8", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + config_delete(cfg); + fprintf(stderr, "\n"); } From 1514babe826f2f44608c5cca7b150900e4592150 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 6 Mar 2019 14:51:52 +0900 Subject: [PATCH 338/526] Fix SessionProtocolNameRegistry lookup Prior this change, SessionProtocolNameRegistry::indexFor() always returns 0. Because `spot` never reach to `m_names.end()`. This is introduced by 5ad8eec303b5f9c38da0de3775e0aadb7186fc38. --- lib/records/RecHttp.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index d20ab3aacd3..0ad3378f62e 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -751,8 +751,9 @@ SessionProtocolNameRegistry::toIndexConst(TextView name) int SessionProtocolNameRegistry::indexFor(TextView name) const { - auto spot = std::find(m_names.begin(), m_names.begin() + m_n, name); - if (spot != m_names.end()) { + const ts::TextView *end = m_names.begin() + m_n; + auto spot = std::find(m_names.begin(), end, name); + if (spot != end) { return static_cast(spot - m_names.begin()); } return INVALID; From 814ccc5ea60942eb0a5f0bf71f6a777f0535df12 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 26 Feb 2019 12:37:26 +0900 Subject: [PATCH 339/526] Move minimum OpenSSL version to 1.0.2 Remove following macros by assuming OpenSSL 1.0.2 API and headers - HAVE_OPENSSL_EC_H - HAVE_OPENSSL_EVP_H - HAVE_EC_KEY_NEW_BY_CURVE_NAME - HAVE_SSL_SELECT_NEXT_PROTO - TS_USE_TLS_NPN - TS_USE_TLS_ALPN - TS_USE_CERT_CB - TS_USE_TLS_ECKEY --- build/crypto.m4 | 120 +++++----------- configure.ac | 20 +-- include/tscore/ink_config.h.in | 4 - iocore/net/SSLClientUtils.cc | 10 +- iocore/net/SSLConfig.cc | 3 +- iocore/net/SSLNetVConnection.cc | 9 -- iocore/net/SSLUtils.cc | 133 +++--------------- src/traffic_layout/info.cc | 4 - src/traffic_server/InkAPI.cc | 7 - tests/README.md | 10 +- tests/gold_tests/headers/forwarded.test.py | 1 - tests/gold_tests/headers/via.test.py | 1 - tests/gold_tests/logging/ccid_ctid.test.py | 1 - .../pluginTest/sslheaders/sslheaders.test.py | 1 - .../pluginTest/test_hooks/test_hooks.test.py | 1 - .../gold_tests/pluginTest/tsapi/tsapi.test.py | 1 - .../pluginTest/url_sig/url_sig.test.py | 3 - 17 files changed, 64 insertions(+), 265 deletions(-) diff --git a/build/crypto.m4 b/build/crypto.m4 index 248be9eefbe..09d40866440 100644 --- a/build/crypto.m4 +++ b/build/crypto.m4 @@ -39,38 +39,28 @@ AC_DEFUN([TS_CHECK_CRYPTO], [ dnl add checks for other varieties of ssl here ]) -dnl - -AC_DEFUN([TS_CHECK_CRYPTO_EC_KEYS], [ - _eckeys_saved_LIBS=$LIBS - - TS_ADDTO(LIBS, [$OPENSSL_LIBS]) - AC_CHECK_HEADERS(openssl/ec.h) - AC_CHECK_FUNCS(EC_KEY_new_by_curve_name, [enable_tls_eckey=yes], [enable_tls_eckey=no]) - LIBS=$_eckeys_saved_LIBS - AC_MSG_CHECKING(whether EC keys are supported) - AC_MSG_RESULT([$enable_tls_eckey]) - TS_ARG_ENABLE_VAR([use], [tls-eckey]) - AC_SUBST(use_tls_eckey) -]) - -AC_DEFUN([TS_CHECK_CRYPTO_NEXTPROTONEG], [ - enable_tls_npn=yes - _npn_saved_LIBS=$LIBS - - TS_ADDTO(LIBS, [$OPENSSL_LIBS]) - AC_CHECK_FUNCS(SSL_CTX_set_next_protos_advertised_cb SSL_CTX_set_next_proto_select_cb SSL_select_next_proto SSL_get0_next_proto_negotiated, - [], [enable_tls_npn=no] - ) - LIBS=$_npn_saved_LIBS - - AC_MSG_CHECKING(whether to enable Next Protocol Negotiation TLS extension support) - AC_MSG_RESULT([$enable_tls_npn]) - TS_ARG_ENABLE_VAR([use], [tls-npn]) - AC_SUBST(use_tls_npn) +dnl +dnl Check OpenSSL Version +dnl +AC_DEFUN([TS_CHECK_CRYPTO_VERSION], [ + AC_MSG_CHECKING([OpenSSL version]) + AC_TRY_RUN([ +#include +int main() { + if (OPENSSL_VERSION_NUMBER < 0x1000200fL) { + return 1; + } + return 0; +} +], + [AC_MSG_RESULT([ok])], + [AC_MSG_FAILURE([requires an OpenSSL version 1.0.2 or greater])]) ]) +dnl +dnl Since OpenSSL 1.1.0 +dnl AC_DEFUN([TS_CHECK_CRYPTO_ASYNC], [ enable_tls_async=yes _async_saved_LIBS=$LIBS @@ -87,63 +77,9 @@ AC_DEFUN([TS_CHECK_CRYPTO_ASYNC], [ AC_SUBST(use_tls_async) ]) -AC_DEFUN([TS_CHECK_CRYPTO_ALPN], [ - enable_tls_alpn=yes - _alpn_saved_LIBS=$LIBS - - TS_ADDTO(LIBS, [$OPENSSL_LIBS]) - AC_CHECK_FUNCS(SSL_CTX_set_alpn_protos SSL_CTX_set_alpn_select_cb SSL_get0_alpn_selected SSL_select_next_proto, - [], [enable_tls_alpn=no] - ) - LIBS=$_alpn_saved_LIBS - - AC_MSG_CHECKING(whether to enable Application Layer Protocol Negotiation TLS extension support) - AC_MSG_RESULT([$enable_tls_alpn]) - TS_ARG_ENABLE_VAR([use], [tls-alpn]) - AC_SUBST(use_tls_alpn) -]) - -AC_DEFUN([TS_CHECK_CRYPTO_CERT_CB], [ - _cert_saved_LIBS=$LIBS - enable_cert_cb=yes - - TS_ADDTO(LIBS, [$OPENSSL_LIBS]) - AC_CHECK_HEADERS(openssl/ssl.h openssl/ts.h) - AC_CHECK_HEADERS(openssl/tls1.h, [], [], -[ #if HAVE_OPENSSL_SSL_H -#include -#include -#endif ]) - - AC_MSG_CHECKING([for SSL_CTX_set_cert_cb]) - AC_LINK_IFELSE( - [ - AC_LANG_PROGRAM([[ -#if HAVE_OPENSSL_SSL_H -#include -#endif -#if HAVE_OPENSSL_TLS1_H -#include -#endif - ]], - [[SSL_CTX_set_cert_cb(NULL, NULL, NULL);]]) - ], - [ - AC_MSG_RESULT([yes]) - ], - [ - AC_MSG_RESULT([no]) - enable_cert_cb=no - ]) - - LIBS=$_cert_saved_LIBS - - AC_MSG_CHECKING(whether to enable TLS certificate callback support) - AC_MSG_RESULT([$enable_cert_cb]) - TS_ARG_ENABLE_VAR([use], [cert-cb]) - AC_SUBST(use_cert_cb) -]) - +dnl +dnl Since OpenSSL 1.1.1 +dnl AC_DEFUN([TS_CHECK_CRYPTO_HELLO_CB], [ _hello_saved_LIBS=$LIBS enable_hello_cb=yes @@ -185,6 +121,9 @@ AC_DEFUN([TS_CHECK_CRYPTO_HELLO_CB], [ AC_SUBST(use_hello_cb) ]) +dnl +dnl Since OpenSSL 1.1.0 +dnl AC_DEFUN([TS_CHECK_CRYPTO_SET_RBIO], [ _rbio_saved_LIBS=$LIBS enable_set_rbio=yes @@ -219,6 +158,9 @@ AC_DEFUN([TS_CHECK_CRYPTO_SET_RBIO], [ AC_SUBST(use_set_rbio) ]) +dnl +dnl Since OpenSSL 1.1.0 +dnl AC_DEFUN([TS_CHECK_CRYPTO_DH_GET_2048_256], [ _dh_saved_LIBS=$LIBS enable_dh_get_2048_256=yes @@ -253,6 +195,9 @@ AC_DEFUN([TS_CHECK_CRYPTO_DH_GET_2048_256], [ AC_SUBST(use_dh_get_2048_256) ]) +dnl +dnl Since OpenSSL 1.1.0 +dnl AC_DEFUN([TS_CHECK_CRYPTO_OCSP], [ _ocsp_saved_LIBS=$LIBS @@ -268,6 +213,9 @@ AC_DEFUN([TS_CHECK_CRYPTO_OCSP], [ AC_SUBST(use_tls_ocsp) ]) +dnl +dnl Since OpenSSL 1.1.1 +dnl AC_DEFUN([TS_CHECK_CRYPTO_SET_CIPHERSUITES], [ _set_ciphersuites_saved_LIBS=$LIBS diff --git a/configure.ac b/configure.ac index 5ecc6b6b59d..7ab2ccdf18a 100644 --- a/configure.ac +++ b/configure.ac @@ -1173,32 +1173,18 @@ TS_ADDTO([LDFLAGS], [$ATOMIC_LIBS]) # # Check for SSL presence and usability +# TS_CHECK_CRYPTO -# -# Check for NextProtocolNegotiation TLS extension support. -TS_CHECK_CRYPTO_NEXTPROTONEG +# Check for OpenSSL Version +TS_CHECK_CRYPTO_VERSION -# -# Check for ALPN TLS extension support. -TS_CHECK_CRYPTO_ALPN - -# # Check for openssl ASYNC jobs TS_CHECK_CRYPTO_ASYNC -# -# Check for EC key support. -TS_CHECK_CRYPTO_EC_KEYS - -# -# Check for the presense of the certificate callback in the ssl library -TS_CHECK_CRYPTO_CERT_CB - # Check for the client hello callback TS_CHECK_CRYPTO_HELLO_CB -# # Check for SSL_set0_rbio call TS_CHECK_CRYPTO_SET_RBIO diff --git a/include/tscore/ink_config.h.in b/include/tscore/ink_config.h.in index 770bd7653ba..4b976debbb8 100644 --- a/include/tscore/ink_config.h.in +++ b/include/tscore/ink_config.h.in @@ -68,14 +68,10 @@ #define TS_HAS_SO_MARK @has_so_mark@ #define TS_HAS_IP_TOS @has_ip_tos@ #define TS_USE_HWLOC @use_hwloc@ -#define TS_USE_TLS_NPN @use_tls_npn@ -#define TS_USE_TLS_ALPN @use_tls_alpn@ #define TS_USE_TLS_ASYNC @use_tls_async@ -#define TS_USE_CERT_CB @use_cert_cb@ #define TS_USE_HELLO_CB @use_hello_cb@ #define TS_USE_SET_RBIO @use_set_rbio@ #define TS_USE_GET_DH_2048_256 @use_dh_get_2048_256@ -#define TS_USE_TLS_ECKEY @use_tls_eckey@ #define TS_USE_TLS_SET_CIPHERSUITES @use_tls_set_ciphersuites@ #define TS_USE_LINUX_NATIVE_AIO @use_linux_native_aio@ #define TS_USE_REMOTE_UNWINDING @use_remote_unwinding@ diff --git a/iocore/net/SSLClientUtils.cc b/iocore/net/SSLClientUtils.cc index dd67e88b051..bb048a3faee 100644 --- a/iocore/net/SSLClientUtils.cc +++ b/iocore/net/SSLClientUtils.cc @@ -32,12 +32,6 @@ #include #include -#if (OPENSSL_VERSION_NUMBER >= 0x10000000L) // openssl returns a const SSL_METHOD -using ink_ssl_method_t = const SSL_METHOD *; -#else -typedef SSL_METHOD *ink_ssl_method_t; -#endif - int verify_callback(int signature_ok, X509_STORE_CTX *ctx) { @@ -143,8 +137,8 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) SSL_CTX * SSLInitClientContext(const SSLConfigParams *params) { - ink_ssl_method_t meth = nullptr; - SSL_CTX *client_ctx = nullptr; + const SSL_METHOD *meth = nullptr; + SSL_CTX *client_ctx = nullptr; // Note that we do not call RAND_seed() explicitly here, we depend on OpenSSL // to do the seeding of the PRNG for us. This is the case for all platforms that diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index dd4edcfcb93..cb477f8f142 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -256,10 +256,9 @@ SSLConfigParams::initialize() #endif #ifdef SSL_OP_NO_COMPRESSION - /* OpenSSL >= 1.0 only */ ssl_ctx_options |= SSL_OP_NO_COMPRESSION; ssl_client_ctx_options |= SSL_OP_NO_COMPRESSION; -#elif OPENSSL_VERSION_NUMBER >= 0x00908000L +#else sk_SSL_COMP_zero(SSL_COMP_get_compression_methods()); #endif diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 2c555715759..a93869b5a75 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -1274,16 +1274,10 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // is preferred since it is the server's preference. The server // preference would not be meaningful if we let the client // preference have priority. - -#if TS_USE_TLS_ALPN SSL_get0_alpn_selected(ssl, &proto, &len); -#endif /* TS_USE_TLS_ALPN */ - -#if TS_USE_TLS_NPN if (len == 0) { SSL_get0_next_proto_negotiated(ssl, &proto, &len); } -#endif /* TS_USE_TLS_NPN */ if (len) { // If there's no NPN set, we should not have done this negotiation. @@ -1517,13 +1511,10 @@ SSLNetVConnection::select_next_protocol(SSL *ssl, const unsigned char **out, uns if (netvc->npnSet && netvc->npnSet->advertiseProtocols(&npn, &npnsz)) { // SSL_select_next_proto chooses the first server-offered protocol that appears in the clients protocol set, ie. the // server selects the protocol. This is a n^2 search, so it's preferable to keep the protocol set short. - -#if HAVE_SSL_SELECT_NEXT_PROTO if (SSL_select_next_proto((unsigned char **)out, outlen, npn, npnsz, in, inlen) == OPENSSL_NPN_NEGOTIATED) { Debug("ssl", "selected ALPN protocol %.*s", (int)(*outlen), *out); return SSL_TLSEXT_ERR_OK; } -#endif /* HAVE_SSL_SELECT_NEXT_PROTO */ } *out = nullptr; diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 145a6bb004f..c83b3143803 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -42,32 +42,27 @@ #include "SSLStats.h" #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include #include -#if HAVE_OPENSSL_EVP_H +#include +#include +#include +#include +#include +#include +#include +#include #include -#endif +#include +#include +#include #if HAVE_OPENSSL_TS_H #include #endif -#if HAVE_OPENSSL_EC_H -#include -#endif - using namespace std::literals; // ssl_multicert.config field names: @@ -82,11 +77,6 @@ static constexpr std::string_view SSL_KEY_DIALOG("ssl_key_dialog"sv); static constexpr std::string_view SSL_SERVERNAME("dest_fqdn"sv); static constexpr char SSL_CERT_SEPARATE_DELIM = ','; -// openssl version must be 0.9.4 or greater -#if (OPENSSL_VERSION_NUMBER < 0x00090400L) -#error Traffic Server requires an OpenSSL library version 0.9.4 or greater -#endif - #ifndef evp_md_func #ifdef OPENSSL_NO_SHA256 #define evp_md_func EVP_sha1() @@ -439,9 +429,6 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) } #endif -// Use the certificate callback for openssl 1.0.2 and greater -// otherwise use the SNI callback -#if TS_USE_CERT_CB /** * Called before either the server or the client certificate is used * Return 1 on success, 0 on error, or -1 to pause @@ -483,7 +470,7 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) * Cannot stop this callback. Always reeneabled */ static int -ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) +ssl_servername_callback(SSL *ssl, int * /* ad */, void * /*arg*/) { SSLNetVConnection *netvc = SSLNetVCAccess(ssl); netvc->callHooks(TS_EVENT_SSL_SERVERNAME); @@ -504,71 +491,6 @@ ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) return SSL_TLSEXT_ERR_OK; } -#else -static int -ssl_servername_and_cert_callback(SSL *ssl, int * /* ad */, void * /*arg*/) -{ - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - bool reenabled; - int retval = 1; - - const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); - if (servername == nullptr) { - servername = ""; - } - Debug("ssl", "Requested servername is %s", servername); - int ret = PerformAction(netvc, servername); - if (ret != SSL_TLSEXT_ERR_OK) { - return SSL_TLSEXT_ERR_ALERT_FATAL; - } - - // If we are in tunnel mode, don't select a cert. Pause! - if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == netvc->attributes) { - return -1; // Pause - } - - // Do the common certificate lookup only once. If we pause - // and restart processing, do not execute the common logic again - if (!netvc->calledHooks(TS_EVENT_SSL_CERT)) { - retval = set_context_cert(ssl); - if (retval != 1) { - goto done; - } - } - - // Call the plugin SNI code - reenabled = netvc->callHooks(TS_EVENT_SSL_CERT); - // If it did not re-enable, return the code to - // stop the accept processing - if (!reenabled) { - retval = -1; - } - -done: - // Map 1 to SSL_TLSEXT_ERR_OK - // Map 0 to SSL_TLSEXT_ERR_ALERT_FATAL - // Map -1 to SSL_TLSEXT_ERR_READ_AGAIN, if present - switch (retval) { - case 1: - retval = SSL_TLSEXT_ERR_OK; - break; - case -1: -#ifdef SSL_TLSEXT_ERR_READ_AGAIN - retval = SSL_TLSEXT_ERR_READ_AGAIN; -#else - Error("Cannot pause SNI processsing with this version of openssl"); - retval = SSL_TLSEXT_ERR_ALERT_FATAL; -#endif - break; - case 0: - default: - retval = SSL_TLSEXT_ERR_ALERT_FATAL; - break; - } - return retval; -} -#endif - #if TS_USE_GET_DH_2048_256 == 0 /* Build 2048-bit MODP Group with 256-bit Prime Order Subgroup from RFC 5114 */ static DH * @@ -651,20 +573,17 @@ ssl_context_enable_ecdh(SSL_CTX *ctx) { #if OPENSSL_VERSION_NUMBER < 0x10100000 -#if TS_USE_TLS_ECKEY - -#if defined(SSL_CTRL_SET_ECDH_AUTO) +#if defined(SSL_CTX_set_ecdh_auto) SSL_CTX_set_ecdh_auto(ctx, 1); -#elif defined(HAVE_EC_KEY_NEW_BY_CURVE_NAME) && defined(NID_X9_62_prime256v1) +#elif defined(NID_X9_62_prime256v1) EC_KEY *ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); if (ecdh) { SSL_CTX_set_tmp_ecdh(ctx, ecdh); EC_KEY_free(ecdh); } -#endif -#endif -#endif +#endif /* SSL_CTRL_SET_ECDH_AUTO */ +#endif /* OPENSSL_VERSION_NUMBER */ return ctx; } @@ -1211,13 +1130,10 @@ ssl_callback_info(const SSL *ssl, int where, int ret) void SSLMultiCertConfigLoader::_set_handshake_callbacks(SSL_CTX *ctx) { -// Make sure the callbacks are set -#if TS_USE_CERT_CB + // Make sure the callbacks are set SSL_CTX_set_cert_cb(ctx, ssl_cert_callback, nullptr); - SSL_CTX_set_tlsext_servername_callback(ctx, ssl_servername_only_callback); -#else - SSL_CTX_set_tlsext_servername_callback(ctx, ssl_servername_and_cert_callback); -#endif + SSL_CTX_set_tlsext_servername_callback(ctx, ssl_servername_callback); + #if TS_USE_HELLO_CB SSL_CTX_set_client_hello_cb(ctx, ssl_client_hello_callback, nullptr); #endif @@ -1305,10 +1221,8 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co } #ifdef SSL_MODE_RELEASE_BUFFERS - if (OPENSSL_VERSION_NUMBER > 0x1000107fL) { - Debug("ssl", "enabling SSL_MODE_RELEASE_BUFFERS"); - SSL_CTX_set_mode(ctx, SSL_MODE_RELEASE_BUFFERS); - } + Debug("ssl", "enabling SSL_MODE_RELEASE_BUFFERS"); + SSL_CTX_set_mode(ctx, SSL_MODE_RELEASE_BUFFERS); #endif #ifdef SSL_OP_SAFARI_ECDHE_ECDSA_BUG @@ -1439,13 +1353,8 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co } SSL_CTX_set_info_callback(ctx, ssl_callback_info); -#if TS_USE_TLS_NPN SSL_CTX_set_next_protos_advertised_cb(ctx, SSLNetVConnection::advertise_next_protocol, nullptr); -#endif /* TS_USE_TLS_NPN */ - -#if TS_USE_TLS_ALPN SSL_CTX_set_alpn_select_cb(ctx, SSLNetVConnection::select_next_protocol, nullptr); -#endif /* TS_USE_TLS_ALPN */ #if TS_USE_TLS_OCSP if (SSLConfigParams::ssl_ocsp_enabled) { diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index f67a737fd20..94060cfbf00 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -89,11 +89,7 @@ produce_features(bool json) print_feature("TS_HAS_SO_MARK", TS_HAS_SO_MARK, json); print_feature("TS_HAS_IP_TOS", TS_HAS_IP_TOS, json); print_feature("TS_USE_HWLOC", TS_USE_HWLOC, json); - print_feature("TS_USE_TLS_NPN", TS_USE_TLS_NPN, json); - print_feature("TS_USE_TLS_ALPN", TS_USE_TLS_ALPN, json); - print_feature("TS_USE_CERT_CB", TS_USE_CERT_CB, json); print_feature("TS_USE_SET_RBIO", TS_USE_SET_RBIO, json); - print_feature("TS_USE_TLS_ECKEY", TS_USE_TLS_ECKEY, json); print_feature("TS_USE_LINUX_NATIVE_AIO", TS_USE_LINUX_NATIVE_AIO, json); print_feature("TS_HAS_SO_PEERCRED", TS_HAS_SO_PEERCRED, json); print_feature("TS_USE_REMOTE_UNWINDING", TS_USE_REMOTE_UNWINDING, json); diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 2b348cb7755..62cda080053 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -6971,7 +6971,6 @@ extern bool ssl_register_protocol(const char *, Continuation *); extern bool ssl_unregister_protocol(const char *, Continuation *); TSReturnCode -#if TS_USE_TLS_NPN TSNetAcceptNamedProtocol(TSCont contp, const char *protocol) { sdk_assert(protocol != nullptr); @@ -6985,12 +6984,6 @@ TSNetAcceptNamedProtocol(TSCont contp, const char *protocol) return TS_SUCCESS; } -#else /* TS_USE_TLS_NPN */ -TSNetAcceptNamedProtocol(TSCont, const char *) -{ - return TS_ERROR; -} -#endif /* TS_USE_TLS_NPN */ /* DNS Lookups */ TSAction diff --git a/tests/README.md b/tests/README.md index 8d0702973d6..f16b2932eb9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,8 +6,8 @@ This directory contains different tests for Apache Trafficserver. It is recommen ## Layout The current layout is: -**gold_tests/** - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest) -**tools/** - contains programs used to help with testing. +**gold_tests/** - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest) +**tools/** - contains programs used to help with testing. **include/** - contains headers used for unit testing. ## Scripts @@ -290,11 +290,7 @@ ts.Disk.remap_config.AddLine( * TS_HAS_SO_MARK * TS_HAS_IP_TOS * TS_USE_HWLOC - * TS_USE_TLS_NPN - * TS_USE_TLS_ALPN - * TS_USE_CERT_CB * TS_USE_SET_RBIO - * TS_USE_TLS_ECKEY * TS_USE_LINUX_NATIVE_AIO * TS_HAS_SO_PEERCRED * TS_USE_REMOTE_UNWINDING @@ -307,7 +303,7 @@ ts.Disk.remap_config.AddLine( ```python #create the origin server process Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), + Condition.HasATSFeature('TS_USE_LINUX_NATIVE_AIO'), ) ``` diff --git a/tests/gold_tests/headers/forwarded.test.py b/tests/gold_tests/headers/forwarded.test.py index e5854f2e752..eb95e382b4b 100644 --- a/tests/gold_tests/headers/forwarded.test.py +++ b/tests/gold_tests/headers/forwarded.test.py @@ -25,7 +25,6 @@ ''' Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2'), Condition.HasCurlFeature('IPv6'), ) diff --git a/tests/gold_tests/headers/via.test.py b/tests/gold_tests/headers/via.test.py index 9746cdec606..1244b489647 100644 --- a/tests/gold_tests/headers/via.test.py +++ b/tests/gold_tests/headers/via.test.py @@ -26,7 +26,6 @@ ''' Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2'), Condition.HasCurlFeature('IPv6') ) diff --git a/tests/gold_tests/logging/ccid_ctid.test.py b/tests/gold_tests/logging/ccid_ctid.test.py index ef3f03122e4..b66a6d185bb 100644 --- a/tests/gold_tests/logging/ccid_ctid.test.py +++ b/tests/gold_tests/logging/ccid_ctid.test.py @@ -27,7 +27,6 @@ Condition.HasProgram( "curl", "Curl need to be installed on system for this test to work"), # Condition.IsPlatform("linux"), Don't see the need for this. - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2') ) diff --git a/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py b/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py index 26c3a432583..c13d0da6117 100644 --- a/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py +++ b/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py @@ -22,7 +22,6 @@ ''' Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2'), ) diff --git a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py index ef7b08688b8..82883036c51 100644 --- a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py +++ b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py @@ -19,7 +19,6 @@ ''' Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2'), ) Test.ContinueOnFail = True diff --git a/tests/gold_tests/pluginTest/tsapi/tsapi.test.py b/tests/gold_tests/pluginTest/tsapi/tsapi.test.py index 6293ae0cb59..91574569a66 100644 --- a/tests/gold_tests/pluginTest/tsapi/tsapi.test.py +++ b/tests/gold_tests/pluginTest/tsapi/tsapi.test.py @@ -19,7 +19,6 @@ ''' Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2'), ) Test.ContinueOnFail = True diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py index 3d0dade02cd..e4a5819a36e 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py @@ -22,9 +22,6 @@ Test url_sig plugin ''' -Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), -) Test.ContinueOnFail = True Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) From d91ca9ee43973fcce9cfccbb146788eb95ae6ffc Mon Sep 17 00:00:00 2001 From: rienzi2012 <51553316@qq.com> Date: Wed, 6 Mar 2019 11:54:33 +0800 Subject: [PATCH 340/526] fix crash in CacheVC::openReadFromWriter --- proxy/hdrs/HTTP.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 74e79df1e6b..8faf57eca0f 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -2035,8 +2035,7 @@ HTTPInfo::marshal(char *buf, int len) buf += m_alt->m_frag_offset_count * sizeof(FragOffset); used += m_alt->m_frag_offset_count * sizeof(FragOffset); } else { - // the data stored in intergral buffer - m_alt->m_frag_offsets = nullptr; + marshal_alt->m_frag_offsets = nullptr; } // The m_{request,response}_hdr->m_heap pointers are converted From d003e8d4b6549589ea83016e54958de0a0daeb55 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Thu, 7 Mar 2019 15:49:07 +0800 Subject: [PATCH 341/526] Fix for() loop, correctly calculate the value of seg_in_use within Vol::dir_check() --- iocore/cache/CacheDir.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/iocore/cache/CacheDir.cc b/iocore/cache/CacheDir.cc index 427ce31ffd6..109538f176f 100644 --- a/iocore/cache/CacheDir.cc +++ b/iocore/cache/CacheDir.cc @@ -1267,10 +1267,6 @@ int Vol::dir_check(bool /* fix ATS_UNUSED */) // TODO: we should eliminate this ++frag_demographics[dir_size(e)][dir_big(e)]; } } - e = next_dir(e, seg); - if (!e) { - break; - } } // Check for duplicates (identical tags in the same bucket). From f35ba19c99478405c894e097780bce52868bba43 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 6 Mar 2019 11:08:54 -0600 Subject: [PATCH 342/526] Change url_mapping::toUrl to url_mapping::toURL for consistency with url_mapping::fromURL. --- proxy/http/remap/RemapConfig.cc | 18 +++++++++--------- proxy/http/remap/UrlMapping.cc | 4 ++-- proxy/http/remap/UrlMapping.h | 4 ++-- proxy/http/remap/UrlRewrite.cc | 10 +++++----- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/proxy/http/remap/RemapConfig.cc b/proxy/http/remap/RemapConfig.cc index 872eb6d11ff..36440e39765 100644 --- a/proxy/http/remap/RemapConfig.cc +++ b/proxy/http/remap/RemapConfig.cc @@ -854,7 +854,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in parv[parc++] = ats_strdup(err); ats_free(err); - if ((err = mp->toUrl.string_get(nullptr)) == nullptr) { + if ((err = mp->toURL.string_get(nullptr)) == nullptr) { snprintf(errbuf, errbufsize, "Can't load toURL from URL class"); return -7; } @@ -960,7 +960,7 @@ process_regex_mapping_config(const char *from_host_lower, url_mapping *new_mappi goto lFail; } - to_host = new_mapping->toUrl.host_get(&to_host_len); + to_host = new_mapping->toURL.host_get(&to_host_len); for (int i = 0; i < (to_host_len - 1); ++i) { if (to_host[i] == '$') { if (substitution_count > UrlRewrite::MAX_REGEX_SUBS) { @@ -981,7 +981,7 @@ process_regex_mapping_config(const char *from_host_lower, url_mapping *new_mappi // so the regex itself is stored in fromURL.host; string to match // will be in the request; string to use for substitutions will be // in this buffer - str = new_mapping->toUrl.host_get(&str_index); // reusing str and str_index + str = new_mapping->toURL.host_get(&str_index); // reusing str and str_index reg_map->to_url_host_template_len = str_index; reg_map->to_url_host_template = static_cast(ats_malloc(str_index)); memcpy(reg_map->to_url_host_template, str, str_index); @@ -1185,8 +1185,8 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) map_to_start = map_to; tmp = map_to; - new_mapping->toUrl.create(nullptr); - rparse = new_mapping->toUrl.parse_no_path_component_breakdown(tmp, length); + new_mapping->toURL.create(nullptr); + rparse = new_mapping->toURL.parse_no_path_component_breakdown(tmp, length); map_to_start[origLength] = '\0'; // Unwhack if (rparse != PARSE_RESULT_DONE) { @@ -1202,7 +1202,7 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) fromScheme = new_mapping->fromURL.scheme_get(&fromSchemeLen); new_mapping->wildcard_from_scheme = true; } - toScheme = new_mapping->toUrl.scheme_get(&toSchemeLen); + toScheme = new_mapping->toURL.scheme_get(&toSchemeLen); // Include support for HTTPS scheme // includes support for FILE scheme @@ -1281,7 +1281,7 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) } } - toHost = new_mapping->toUrl.host_get(&toHostLen); + toHost = new_mapping->toURL.host_get(&toHostLen); if (toHost == nullptr || toHostLen <= 0) { errStr = "The remap destinations require a hostname"; goto MAP_ERROR; @@ -1340,8 +1340,8 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) u_mapping->fromURL.create(nullptr); u_mapping->fromURL.copy(&new_mapping->fromURL); u_mapping->fromURL.host_set(ipb, strlen(ipb)); - u_mapping->toUrl.create(nullptr); - u_mapping->toUrl.copy(&new_mapping->toUrl); + u_mapping->toURL.create(nullptr); + u_mapping->toURL.copy(&new_mapping->toURL); if (bti->paramv[3] != nullptr) { u_mapping->tag = ats_strdup(&(bti->paramv[3][0])); diff --git a/proxy/http/remap/UrlMapping.cc b/proxy/http/remap/UrlMapping.cc index 5979490ec87..c675abd2ca7 100644 --- a/proxy/http/remap/UrlMapping.cc +++ b/proxy/http/remap/UrlMapping.cc @@ -111,7 +111,7 @@ url_mapping::~url_mapping() // Destroy the URLs fromURL.destroy(); - toUrl.destroy(); + toURL.destroy(); } void @@ -120,7 +120,7 @@ url_mapping::Print() char from_url_buf[131072], to_url_buf[131072]; fromURL.string_get_buf(from_url_buf, (int)sizeof(from_url_buf)); - toUrl.string_get_buf(to_url_buf, (int)sizeof(to_url_buf)); + toURL.string_get_buf(to_url_buf, (int)sizeof(to_url_buf)); printf("\t %s %s=> %s %s <%s> [plugins %s enabled; running with %zu plugins]\n", from_url_buf, unique ? "(unique)" : "", to_url_buf, homePageRedirect ? "(R)" : "", tag ? tag : "", plugin_count() > 0 ? "are" : "not", plugin_count()); } diff --git a/proxy/http/remap/UrlMapping.h b/proxy/http/remap/UrlMapping.h index 63f7ca01e7d..96d2109541b 100644 --- a/proxy/http/remap/UrlMapping.h +++ b/proxy/http/remap/UrlMapping.h @@ -94,7 +94,7 @@ class url_mapping int from_path_len = 0; URL fromURL; - URL toUrl; // Default TO-URL (from remap.config) + URL toURL; // Default TO-URL (from remap.config) bool homePageRedirect = false; bool unique = false; // INKqa11970 - unique mapping bool default_redirect_url = false; @@ -159,7 +159,7 @@ class UrlMappingContainer { deleteToURL(); _mapping = m; - _toURLPtr = m ? &(m->toUrl) : nullptr; + _toURLPtr = m ? &(m->toURL) : nullptr; } void diff --git a/proxy/http/remap/UrlRewrite.cc b/proxy/http/remap/UrlRewrite.cc index 13e39808d47..5c154d3a7f2 100644 --- a/proxy/http/remap/UrlRewrite.cc +++ b/proxy/http/remap/UrlRewrite.cc @@ -144,8 +144,8 @@ UrlRewrite::SetupBackdoorMapping() mapping->fromURL.parse(from_url, sizeof(from_url) - 1); mapping->fromURL.scheme_set(URL_SCHEME_HTTP, URL_LEN_HTTP); - mapping->toUrl.create(nullptr); - mapping->toUrl.parse(to_url, sizeof(to_url) - 1); + mapping->toURL.create(nullptr); + mapping->toURL.parse(to_url, sizeof(to_url) - 1); return mapping; } @@ -599,7 +599,7 @@ UrlRewrite::InsertMapping(mapping_type maptype, url_mapping *new_mapping, RegexM success = _addToStore(forward_mappings, new_mapping, reg_map, src_host, is_cur_mapping_regex, num_rules_forward); if (success) { // @todo: is this applicable to regex mapping too? - SetHomePageRedirectFlag(new_mapping, new_mapping->toUrl); + SetHomePageRedirectFlag(new_mapping, new_mapping->toURL); } break; case REVERSE_MAP: @@ -641,7 +641,7 @@ UrlRewrite::InsertForwardMapping(mapping_type maptype, url_mapping *mapping, con case FORWARD_MAP: case FORWARD_MAP_REFERER: case FORWARD_MAP_WITH_RECV_PORT: - SetHomePageRedirectFlag(mapping, mapping->toUrl); + SetHomePageRedirectFlag(mapping, mapping->toURL); break; default: break; @@ -914,7 +914,7 @@ UrlRewrite::_regexMappingLookup(RegexMappingList ®ex_mappings, URL *request_u // Expand substitutions in the host field from the stored template buf_len = _expandSubstitutions(matches_info, list_iter, request_host, buf, sizeof(buf)); URL *expanded_url = mapping_container.createNewToURL(); - expanded_url->copy(&((list_iter->url_map)->toUrl)); + expanded_url->copy(&((list_iter->url_map)->toURL)); expanded_url->host_set(buf, buf_len); Debug("url_rewrite_regex", "Expanded toURL to [%.*s]", expanded_url->length_get(), expanded_url->string_get_ref()); From a181db6f5948d2f35e73a82332695ad35bc2bc33 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Thu, 7 Mar 2019 09:25:29 +0900 Subject: [PATCH 343/526] Do not run clang-format for @default_stack_size@ --- include/tscore/ink_config.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/tscore/ink_config.h.in b/include/tscore/ink_config.h.in index 4b976debbb8..3e3aa288780 100644 --- a/include/tscore/ink_config.h.in +++ b/include/tscore/ink_config.h.in @@ -121,6 +121,6 @@ #define TS_BUILD_CANONICAL_HOST "@host@" #define TS_BUILD_DEFAULT_LOOPBACK_IFACE "@default_loopback_iface@" -/* clang-format on */ static const int DEFAULT_STACKSIZE = @default_stack_size@; +/* clang-format on */ From 6b1c2117d408a96f42dd2e74a7171d214b1a2ce8 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 7 Mar 2019 15:53:13 -0700 Subject: [PATCH 344/526] Fixes some places where refactoring was not complete --- .../ja/LC_MESSAGES/admin-guide/files/records.config.en.po | 2 +- .../api/functions/TSHttpOverridableConfig.en.po | 4 ---- lib/perl/lib/Apache/TS/AdminClient.pm | 2 +- tests/gold_tests/slow_post/slow_post.test.py | 4 ++-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po index 6838781cfa0..baab72cf7df 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po @@ -2335,7 +2335,7 @@ msgid "" "Limits the number of requests to be queued when the :ts:cv:`proxy.config." "http.origin_max_connections` is reached. When disabled (``-1``) requests " "are will wait indefinitely for an available connection. When set to ``0`` " -"all requests past the :ts:cv:`proxy.config.http.origin_max_connections` " +"all requests past the :ts:cv:`proxy.config.http.per_server.connection.max` " "will immediately fail. When set to ``>0`` ATS will queue that many requests " "to go to the origin, any additional requests past the limit will " "immediately fail." diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po index 20e89995138..01ef8bb56b8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po @@ -270,10 +270,6 @@ msgstr ":ts:cv:`proxy.config.http.cache.heuristic_min_lifetime`" msgid ":ts:cv:`proxy.config.websocket.active_timeout`" msgstr ":ts:cv:`proxy.config.http.cache.heuristic_min_lifetime`" -#: ../../../developer-guide/api/functions/TSHttpOverridableConfig.en.rst:112 -msgid ":ts:cv:`proxy.config.http.origin_max_connections`" -msgstr "" - #: ../../../developer-guide/api/functions/TSHttpOverridableConfig.en.rst:113 msgid ":ts:cv:`proxy.config.http.connect_attempts_max_retries`" msgstr "" diff --git a/lib/perl/lib/Apache/TS/AdminClient.pm b/lib/perl/lib/Apache/TS/AdminClient.pm index db7818e0008..5f9aab63717 100644 --- a/lib/perl/lib/Apache/TS/AdminClient.pm +++ b/lib/perl/lib/Apache/TS/AdminClient.pm @@ -482,7 +482,7 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.http.no_origin_server_dns proxy.config.http.normalize_ae_gzip proxy.config.http.number_of_redirections - proxy.config.http.origin_max_connections + proxy.config.http.per_server.connection.max proxy.config.http.origin_min_keep_alive_connections proxy.config.http.parent_proxies proxy.config.http.parent_proxy.connect_attempts_timeout diff --git a/tests/gold_tests/slow_post/slow_post.test.py b/tests/gold_tests/slow_post/slow_post.test.py index 906a87f14fd..c1e92f85374 100644 --- a/tests/gold_tests/slow_post/slow_post.test.py +++ b/tests/gold_tests/slow_post/slow_post.test.py @@ -53,9 +53,9 @@ def setupTS(self): self._ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http', - 'proxy.config.http.origin_max_connections': self._origin_max_connections, + 'proxy.config.http.per_server.connection.max': self._origin_max_connections, # Disable queueing when connection reaches limit - 'proxy.config.http.origin_max_connections_queue': 0, + 'proxy.config.http.per_server.connection.queue_size': 0, }) def run(self): From 4934fb23c18caeec12de7bd9891f92da6cc11b7b Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Thu, 7 Mar 2019 16:53:30 -0800 Subject: [PATCH 345/526] Doc: open_read_retry_time is overridable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit proxy.config.http.cache.open_read_retry_time is overridable ''' $ git grep proxy.config.http.cache.open_read_retry_time | grep InkAPI.cc src/traffic_server/InkAPI.cc:8630:   {"proxy.config.http.cache.open_read_retry_time", {TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME, TS_RECORDDATATYPE_INT}}, ''' (i.e. its overridable). --- doc/admin-guide/files/records.config.en.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 3abea79396d..eb380e44545 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -2307,6 +2307,7 @@ all the different user-agent versions of documents it encounters. .. ts:cv:: CONFIG proxy.config.http.cache.open_read_retry_time INT 10 :reloadable: + :overridable: The number of milliseconds a cacheable request will wait before requesting the object from cache if an equivalent request is in flight. From 4be1c00da03d821808ba08b150db1f367895af18 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 8 Mar 2019 10:17:40 -0600 Subject: [PATCH 346/526] Add virtual destructor to SSLMultiCertConfigLoader. --- iocore/net/P_SSLUtils.h | 1 + 1 file changed, 1 insertion(+) diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index 3e3de60ca8f..648ed8035ff 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -67,6 +67,7 @@ class SSLMultiCertConfigLoader { public: SSLMultiCertConfigLoader(const SSLConfigParams *p) : _params(p) {} + virtual ~SSLMultiCertConfigLoader(){}; bool load(SSLCertLookup *lookup); From b09ac093c7ec27f7a408b3ec2036592521d8da89 Mon Sep 17 00:00:00 2001 From: John Rushford Date: Fri, 8 Mar 2019 19:38:23 +0000 Subject: [PATCH 347/526] Fix a self detection issue where parents are not marked down when proxy.config.http.parent_proxy.self_detect is set to 2 because of multiple calls to creatHostStat() which was marking parents back up. Also added a new HostStatus Reason Code SELF_DETECT used when self detection marks a parent down. --- proxy/HostStatus.h | 9 +++++---- proxy/ParentSelection.cc | 14 ++++++++------ src/traffic_server/HostStatus.cc | 5 ++++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/proxy/HostStatus.h b/proxy/HostStatus.h index 8257913dfde..c9e14431f5f 100644 --- a/proxy/HostStatus.h +++ b/proxy/HostStatus.h @@ -50,11 +50,12 @@ struct HostStatRec_t { }; struct Reasons { - static constexpr const char *ACTIVE = "active"; - static constexpr const char *LOCAL = "local"; - static constexpr const char *MANUAL = "manual"; + static constexpr const char *ACTIVE = "active"; + static constexpr const char *LOCAL = "local"; + static constexpr const char *MANUAL = "manual"; + static constexpr const char *SELF_DETECT = "self_detect"; - static constexpr const char *reasons[3] = {ACTIVE, LOCAL, MANUAL}; + static constexpr const char *reasons[4] = {ACTIVE, LOCAL, MANUAL, SELF_DETECT}; static bool validReason(const char *reason) diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index 6e7e963b579..d8b64d43dc4 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -372,8 +372,7 @@ ParentRecord::PreProcessParents(const char *val, const int line_num, char *buf, continue; } else { Debug("parent_select", "token: %s, matches this machine. Marking down self from parent list at line %d", fqdn, line_num); - hs.createHostStat(fqdn); - hs.setHostStatus(fqdn, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::MANUAL); + hs.setHostStatus(fqdn, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::SELF_DETECT); } } } else { @@ -385,8 +384,7 @@ ParentRecord::PreProcessParents(const char *val, const int line_num, char *buf, } else { Debug("parent_select", "token: %s, matches this machine. Marking down self from parent list at line %d", token, line_num); - hs.createHostStat(token); - hs.setHostStatus(token, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::MANUAL); + hs.setHostStatus(token, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::SELF_DETECT); } } } @@ -517,7 +515,9 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) memcpy(this->parents[i].hash_string, tmp3 + 1, strlen(tmp3)); this->parents[i].name = this->parents[i].hash_string; } - hs.createHostStat(this->parents[i].hostname); + if (hs.getHostStatus(this->parents[i].hostname) == HostStatus_t::HOST_STATUS_INIT) { + hs.setHostStatus(this->parents[i].hostname, HOST_STATUS_UP, 0, Reasons::MANUAL); + } } else { memcpy(this->secondary_parents[i].hostname, current, tmp - current); this->secondary_parents[i].hostname[tmp - current] = '\0'; @@ -533,7 +533,9 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) memcpy(this->secondary_parents[i].hash_string, tmp3 + 1, strlen(tmp3)); this->secondary_parents[i].name = this->secondary_parents[i].hash_string; } - hs.createHostStat(this->secondary_parents[i].hostname); + if (hs.getHostStatus(this->secondary_parents[i].hostname) == HostStatus_t::HOST_STATUS_INIT) { + hs.setHostStatus(this->secondary_parents[i].hostname, HOST_STATUS_UP, 0, Reasons::MANUAL); + } } tmp3 = nullptr; } diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index ae6302d620a..993feb363b3 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -120,6 +120,10 @@ HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned getStatName(reason_stat, name, reason); + if (getHostStatId(reason_stat.c_str()) == -1) { + createHostStat(name); + } + int stat_id = getHostStatId(reason_stat.c_str()); // update the stats @@ -212,7 +216,6 @@ HostStatus::createHostStat(const char *name) } } ink_rwlock_unlock(&host_statids_rwlock); - setHostStatus(name, HostStatus_t::HOST_STATUS_UP, 0, nullptr); } int From 40f94cc062e39c15f21b97e75151b32253a08d97 Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Fri, 8 Mar 2019 13:15:32 -0600 Subject: [PATCH 348/526] pop messages --- plugins/experimental/ssl_session_reuse/src/publish.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/experimental/ssl_session_reuse/src/publish.cc b/plugins/experimental/ssl_session_reuse/src/publish.cc index 872920494d9..c0235b310c8 100644 --- a/plugins/experimental/ssl_session_reuse/src/publish.cc +++ b/plugins/experimental/ssl_session_reuse/src/publish.cc @@ -239,6 +239,9 @@ RedisPublisher::runWorker() } Message ¤t_message(m_messageQueue.front()); + if (!current_message.cleanup) { + m_messageQueue.pop_front(); + } m_messageQueueMutex.unlock(); ::sem_wait(&m_workerSem); @@ -264,10 +267,6 @@ RedisPublisher::runWorker() } } - if (!current_message.cleanup) { - m_messageQueue.pop_front(); - } - clear_reply(current_reply); current_reply = nullptr; } From 97aac1e1a7d328076752dd032bca3967b0d22023 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 18 Feb 2019 14:26:00 -0700 Subject: [PATCH 349/526] Add Perltidy configuration and build target Also does an initial run. --- .perltidyrc | 14 + Makefile.am | 5 +- ci/jenkins/ats_conf.pl | 68 ++-- ci/rat-regex.txt | 1 + lib/perl/examples/forw_proxy_conf.pl | 11 +- lib/perl/lib/Apache/TS/AdminClient.pm | 61 ++- lib/perl/lib/Apache/TS/Config.pm | 6 +- lib/perl/lib/Apache/TS/Config/Records.pm | 66 ++-- plugins/experimental/url_sig/genkeys.pl | 8 +- plugins/experimental/url_sig/sign.pl | 329 +++++++++-------- proxy/http/test_http_client.pl | 390 +++++++++----------- proxy/http/test_proxy.pl | 450 +++++++++++------------ tools/changelog.pl | 223 ++++++----- tools/compare_records.pl | 158 ++++---- tools/compare_servers.pl | 386 +++++++++---------- tools/freelist_diff.pl | 25 +- tools/http_load/merge_stats.pl | 91 ++--- tools/slow_log_report.pl | 104 +++--- tools/traffic_via.pl | 250 ++++++------- 19 files changed, 1304 insertions(+), 1342 deletions(-) create mode 100644 .perltidyrc diff --git a/.perltidyrc b/.perltidyrc new file mode 100644 index 00000000000..86440f5ae93 --- /dev/null +++ b/.perltidyrc @@ -0,0 +1,14 @@ + # This is a simple of a .perltidyrc configuration file + -l=132 # Line length + -i=4 # 4-space indentation + -nlp # Line up params + -ce # cuddle the braces + -tso # Tight secret ops + -nsfs # No space for semicolon + -pt=2 # tight parens + -bt=2 # tight braces + -sbt=2 # tight brackets + -bbt=2 # tight code brackets + -nbbc # No blank lines before comment lines + -otr # No break between a comma and an opening token + -sbl # Empty lane for sub's opening brace diff --git a/Makefile.am b/Makefile.am index 15a411a7709..889be94abc9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -119,7 +119,7 @@ autopep8: # If you make changes to directory structures, you must update this as well. # .PHONY: clang-format-src clang-format-example clang-format-iocore clang-format-lib clang-format-mgmt \ - clang-format-plugins clang-format-proxy clang-format-tools + clang-format-plugins clang-format-proxy clang-format-tools perltidy clang-format: clang-format-src clang-format-example clang-format-iocore clang-format-lib clang-format-mgmt \ clang-format-plugins clang-format-proxy clang-format-tools clang-format-tests @@ -152,6 +152,9 @@ clang-format-tools: clang-format-tests: @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/tests +perltidy: + perltidy -q -b -bext='/' `find . -name \*.pm -o -name \*.pl` + help: @echo 'all default target for building the package' @echo 'asf-dist recreate source package' diff --git a/ci/jenkins/ats_conf.pl b/ci/jenkins/ats_conf.pl index 0fd0b75c056..83d293e3c26 100755 --- a/ci/jenkins/ats_conf.pl +++ b/ci/jenkins/ats_conf.pl @@ -32,69 +32,69 @@ #$recedit->append(line => "CONFIG proxy.config.crash_log_helper STRING /home/admin/bin/invoker_wrap.sh"); # Port setup -$recedit->set(conf => "proxy.config.http.server_ports", val => "80 80:ipv6 443:ssl 443:ipv6:ssl"); -$recedit->set(conf => "proxy.config.admin.autoconf_port", val => "48083"); +$recedit->set(conf => "proxy.config.http.server_ports", val => "80 80:ipv6 443:ssl 443:ipv6:ssl"); +$recedit->set(conf => "proxy.config.admin.autoconf_port", val => "48083"); $recedit->set(conf => "proxy.config.process_manager.mgmt_port", val => "48084"); # Threads $recedit->set(conf => "proxy.config.exec_thread.autoconfig", val => "0"); -$recedit->set(conf => "proxy.config.exec_thread.limit", val => "8"); +$recedit->set(conf => "proxy.config.exec_thread.limit", val => "8"); $recedit->set(conf => "proxy.config.cache.threads_per_disk", val => "8"); -$recedit->set(conf => "proxy.config.accept_threads", val => "0"); -$recedit->set(conf => "proxy.config.exec_thread.affinity", val => "1"); +$recedit->set(conf => "proxy.config.accept_threads", val => "0"); +$recedit->set(conf => "proxy.config.exec_thread.affinity", val => "1"); # TLS #$recedit->set(conf => "proxy.config.ssl.server.cipher_suite", val => "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2:!RC4"); $recedit->set(conf => "proxy.config.ssl.hsts_max_age", val => "17280000"); #$recedit->set(conf => "proxy.config.ssl.max_record_size", val => "-1"); -$recedit->set(conf => "proxy.config.ssl.session_cache", val => "2"); -$recedit->set(conf => "proxy.config.ssl.ocsp.enabled", val => "1"); +$recedit->set(conf => "proxy.config.ssl.session_cache", val => "2"); +$recedit->set(conf => "proxy.config.ssl.ocsp.enabled", val => "1"); $recedit->set(conf => "proxy.config.http2.stream_priority_enabled", val => "1"); # Cache setup -$recedit->set(conf => "proxy.config.cache.ram_cache.size", val => "1536M"); -$recedit->set(conf => "proxy.config.cache.ram_cache_cutoff", val => "4M"); -$recedit->set(conf => "proxy.config.cache.limits.http.max_alts", val => "4"); -$recedit->set(conf => "proxy.config.cache.dir.sync_frequency", val => "600"); # 10 minutes intervals +$recedit->set(conf => "proxy.config.cache.ram_cache.size", val => "1536M"); +$recedit->set(conf => "proxy.config.cache.ram_cache_cutoff", val => "4M"); +$recedit->set(conf => "proxy.config.cache.limits.http.max_alts", val => "4"); +$recedit->set(conf => "proxy.config.cache.dir.sync_frequency", val => "600"); # 10 minutes intervals $recedit->set(conf => "proxy.config.http.cache.ignore_client_cc_max_age", val => "1"); -$recedit->set(conf => "proxy.config.allocator.hugepages", val => "1"); +$recedit->set(conf => "proxy.config.allocator.hugepages", val => "1"); # HTTP caching related stuff -$recedit->set(conf => "proxy.config.http.cache.required_headers", val => "1"); -$recedit->set(conf => "proxy.config.http.insert_request_via_str", val => "1"); -$recedit->set(conf => "proxy.config.http.insert_response_via_str", val => "2"); -$recedit->set(conf => "proxy.config.http.negative_caching_enabled", val => "1"); +$recedit->set(conf => "proxy.config.http.cache.required_headers", val => "1"); +$recedit->set(conf => "proxy.config.http.insert_request_via_str", val => "1"); +$recedit->set(conf => "proxy.config.http.insert_response_via_str", val => "2"); +$recedit->set(conf => "proxy.config.http.negative_caching_enabled", val => "1"); $recedit->set(conf => "proxy.config.http.negative_caching_lifetime", val => "60"); -$recedit->set(conf => "proxy.config.http.chunking.size", val => "64k"); -$recedit->set(conf => "proxy.config.url_remap.pristine_host_hdr", val => "1"); +$recedit->set(conf => "proxy.config.http.chunking.size", val => "64k"); +$recedit->set(conf => "proxy.config.url_remap.pristine_host_hdr", val => "1"); # Timeouts -$recedit->set(conf => "proxy.config.http.keep_alive_no_activity_timeout_in", val => "300"); -$recedit->set(conf => "proxy.config.http.keep_alive_no_activity_timeout_out", val => "300"); +$recedit->set(conf => "proxy.config.http.keep_alive_no_activity_timeout_in", val => "300"); +$recedit->set(conf => "proxy.config.http.keep_alive_no_activity_timeout_out", val => "300"); $recedit->set(conf => "proxy.config.http.transaction_no_activity_timeout_out", val => "180"); -$recedit->set(conf => "proxy.config.http.transaction_no_activity_timeout_in", val => "180"); -$recedit->set(conf => "proxy.config.http.transaction_active_timeout_in", val => "180"); -$recedit->set(conf => "proxy.config.http.transaction_active_timeout_out", val => "180"); -$recedit->set(conf => "proxy.config.http.accept_no_activity_timeout", val => "30"); +$recedit->set(conf => "proxy.config.http.transaction_no_activity_timeout_in", val => "180"); +$recedit->set(conf => "proxy.config.http.transaction_active_timeout_in", val => "180"); +$recedit->set(conf => "proxy.config.http.transaction_active_timeout_out", val => "180"); +$recedit->set(conf => "proxy.config.http.accept_no_activity_timeout", val => "30"); # DNS / HostDB -$recedit->set(conf => "proxy.config.cache.hostdb.sync_frequency", val => "0"); +$recedit->set(conf => "proxy.config.cache.hostdb.sync_frequency", val => "0"); # Logging -$recedit->set(conf => "proxy.config.log.logging_enabled", val => "3"); -$recedit->set(conf => "proxy.config.log.max_space_mb_for_logs", val => "4096"); -$recedit->set(conf => "proxy.config.log.max_space_mb_headroom", val => "64"); +$recedit->set(conf => "proxy.config.log.logging_enabled", val => "3"); +$recedit->set(conf => "proxy.config.log.max_space_mb_for_logs", val => "4096"); +$recedit->set(conf => "proxy.config.log.max_space_mb_headroom", val => "64"); # Network -$recedit->set(conf => "proxy.config.net.connections_throttle", val => "10000"); -$recedit->set(conf => "proxy.config.net.sock_send_buffer_size_in", val => "4M"); +$recedit->set(conf => "proxy.config.net.connections_throttle", val => "10000"); +$recedit->set(conf => "proxy.config.net.sock_send_buffer_size_in", val => "4M"); $recedit->set(conf => "proxy.config.net.sock_recv_buffer_size_out", val => "4M"); -$recedit->set(conf => "proxy.config.net.poll_timeout", val => "30"); +$recedit->set(conf => "proxy.config.net.poll_timeout", val => "30"); # Local additions (typically not found in the records.config.default) -$recedit->set(conf => "proxy.config.dns.dedicated_thread", val => "0"); -$recedit->set(conf => "proxy.config.http_ui_enabled", val => "3"); -$recedit->set(conf => "proxy.config.http.server_max_connections", val =>"250"); +$recedit->set(conf => "proxy.config.dns.dedicated_thread", val => "0"); +$recedit->set(conf => "proxy.config.http_ui_enabled", val => "3"); +$recedit->set(conf => "proxy.config.http.server_max_connections", val => "250"); #$recedit->set(conf => "proxy.config.mlock_enabled", val => "2"); diff --git a/ci/rat-regex.txt b/ci/rat-regex.txt index 93a24d5b087..18c5c827e59 100644 --- a/ci/rat-regex.txt +++ b/ci/rat-regex.txt @@ -25,6 +25,7 @@ .*\.gold$ ^\.gitignore$ ^\.gitmodules$ +^\.perltidyrc$ ^\.indent.pro$ ^\.vimrc$ ^\.clang-.*$ diff --git a/lib/perl/examples/forw_proxy_conf.pl b/lib/perl/examples/forw_proxy_conf.pl index 9cdd79cdfba..ae25562b232 100755 --- a/lib/perl/examples/forw_proxy_conf.pl +++ b/lib/perl/examples/forw_proxy_conf.pl @@ -18,7 +18,6 @@ use Apache::TS::Config::Records; - ############################################################################ # Simple script, to show some minimum configuration changes typical for # a forward proxy. @@ -26,16 +25,16 @@ my $recedit = new Apache::TS::Config::Records(file => $fn); # Definitely tweak the memory config -$recedit->set(conf => "proxy.config.cache.ram_cache.size", val => "2048M"); +$recedit->set(conf => "proxy.config.cache.ram_cache.size", val => "2048M"); # These puts the server in forward proxy mode only. -$recedit->set(conf => "proxy.config.url_remap.remap_required", val => "0"); -$recedit->set(conf => "proxy.config.reverse_proxy.enabled", val => "0"); +$recedit->set(conf => "proxy.config.url_remap.remap_required", val => "0"); +$recedit->set(conf => "proxy.config.reverse_proxy.enabled", val => "0"); # Fine tuning, you might or might not want these $recedit->set(conf => "proxy.config.http.transaction_active_timeout_in", val => "1800"); -$recedit->set(conf => "proxy.config.dns.dedicated_thread", val => "1"); -$recedit->set(conf => "proxy.config.http.normalize_ae_gzip", val => "1"); +$recedit->set(conf => "proxy.config.dns.dedicated_thread", val => "1"); +$recedit->set(conf => "proxy.config.http.normalize_ae_gzip", val => "1"); # Write out the new config file (this won't overwrite your config $recedit->write(file => "$fn.new"); diff --git a/lib/perl/lib/Apache/TS/AdminClient.pm b/lib/perl/lib/Apache/TS/AdminClient.pm index 5f9aab63717..937d5ce310d 100644 --- a/lib/perl/lib/Apache/TS/AdminClient.pm +++ b/lib/perl/lib/Apache/TS/AdminClient.pm @@ -76,20 +76,16 @@ use constant { TS_ERR_FAIL => 12 }; - # Semi-intelligent way of finding the mgmtapi socket. -sub _find_socket { +sub _find_socket +{ my $path = shift || ""; my $name = shift || "mgmtapi.sock"; my @sockets_def = ( - $path, - Apache::TS::PREFIX . '/' . Apache::TS::REL_RUNTIMEDIR . '/' . 'mgmtapi.sock', - '/usr/local/var/trafficserver', - '/usr/local/var/run/trafficserver', - '/usr/local/var/run', - '/var/trafficserver', - '/var/run/trafficserver', - '/var/run', + $path, Apache::TS::PREFIX . '/' . Apache::TS::REL_RUNTIMEDIR . '/' . 'mgmtapi.sock', + '/usr/local/var/trafficserver', '/usr/local/var/run/trafficserver', + '/usr/local/var/run', '/var/trafficserver', + '/var/run/trafficserver', '/var/run', '/opt/ats/var/trafficserver', ); @@ -104,14 +100,14 @@ sub _find_socket { # # Constructor # -sub new { +sub new +{ my ($class, %args) = @_; my $self = {}; $self->{_socket_path} = _find_socket($args{socket_path}); - $self->{_socket} = undef; - croak -"Unable to locate socket, please pass socket_path with the management api socket location to Apache::TS::AdminClient" + $self->{_socket} = undef; + croak "Unable to locate socket, please pass socket_path with the management api socket location to Apache::TS::AdminClient" if (!$self->{_socket_path}); if ((!-r $self->{_socket_path}) or (!-w $self->{_socket_path}) or (!-S $self->{_socket_path})) { croak "Unable to open $self->{_socket_path} for reads or writes"; @@ -128,7 +124,8 @@ sub new { # # Destructor # -sub DESTROY { +sub DESTROY +{ my $self = shift; return $self->close_socket(); } @@ -136,15 +133,15 @@ sub DESTROY { # # Open the socket (Unix domain) # -sub open_socket { +sub open_socket +{ my $self = shift; my %args = @_; if (defined($self->{_socket})) { if ($args{force} || $args{reopen}) { $self->close_socket(); - } - else { + } else { return undef; } } @@ -152,7 +149,7 @@ sub open_socket { $self->{_socket} = IO::Socket::UNIX->new( Type => SOCK_STREAM, Peer => $self->{_socket_path} - ) or croak("Error opening socket - $@"); + ) or croak("Error opening socket - $@"); return undef unless defined($self->{_socket}); $self->{_select}->add($self->{_socket}); @@ -160,7 +157,8 @@ sub open_socket { return $self; } -sub close_socket { +sub close_socket +{ my $self = shift; # if socket doesn't exist, return as there's nothing to do. @@ -177,10 +175,11 @@ sub close_socket { # # Do reads()'s on our Unix domain socket, takes an optional timeout, in ms's. # -sub _do_read { - my $self = shift; - my $timeout = shift || 1/1000.0; # 1ms by default - my $res = ""; +sub _do_read +{ + my $self = shift; + my $timeout = shift || 1 / 1000.0; # 1ms by default + my $res = ""; while ($self->{_select}->can_read($timeout)) { my $rc = $self->{_socket}->sysread($res, 1024, length($res)); @@ -199,14 +198,14 @@ sub _do_read { return $res || undef; } - # # Get (read) a stat out of the local manager. Note that the assumption is # that you are calling this with an existing stats "name". # -sub get_stat { +sub get_stat +{ my ($self, $stat) = @_; - my $res = ""; + my $res = ""; return undef unless defined($self->{_socket}); return undef unless $self->{_select}->can_write(10); @@ -219,7 +218,7 @@ sub get_stat { my $msg = pack("ll/Z", TS_RECORD_GET, $stat); $self->{_socket}->print(pack("l/a", $msg)); $res = $self->_do_read(); - return undef unless defined($res); # Don't proceed on read failure. + return undef unless defined($res); # Don't proceed on read failure. # The response format is: # MGMT_MARSHALL_INT: message length @@ -235,12 +234,10 @@ sub get_stat { if ($type == TS_REC_INT || $type == TS_REC_COUNTER) { my ($ival) = unpack("q", $value); return $ival; - } - elsif ($type == TS_REC_FLOAT) { + } elsif ($type == TS_REC_FLOAT) { my ($fval) = unpack("f", $value); return $fval; - } - elsif ($type == TS_REC_STRING) { + } elsif ($type == TS_REC_STRING) { my ($sval) = unpack("Z*", $value); return $sval; } diff --git a/lib/perl/lib/Apache/TS/Config.pm b/lib/perl/lib/Apache/TS/Config.pm index 4e9e7aa6289..d4a50078542 100644 --- a/lib/perl/lib/Apache/TS/Config.pm +++ b/lib/perl/lib/Apache/TS/Config.pm @@ -31,7 +31,7 @@ our $VERSION = "1.0"; # Constants use constant { - TS_CONF_UNMODIFIED => 0, - TS_CONF_MODIFIED => 1, - TS_CONF_REMOVED => 2 + TS_CONF_UNMODIFIED => 0, + TS_CONF_MODIFIED => 1, + TS_CONF_REMOVED => 2 }; diff --git a/lib/perl/lib/Apache/TS/Config/Records.pm b/lib/perl/lib/Apache/TS/Config/Records.pm index 5f691138610..078aab87d16 100644 --- a/lib/perl/lib/Apache/TS/Config/Records.pm +++ b/lib/perl/lib/Apache/TS/Config/Records.pm @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - ############################################################################ # This is a simple module to let you read, modify and add to an Apache # Traffic Server records.config file. The idea is that you would write a @@ -25,7 +24,6 @@ # perldoc for more details. ############################################################################ - package Apache::TS::Config::Records; use Apache::TS::Config; @@ -38,22 +36,22 @@ use Carp; our $VERSION = "1.0"; - # # Constructor # -sub new { +sub new +{ my ($class, %args) = @_; my $self = {}; - my $fn = $args{file}; + my $fn = $args{file}; $fn = $args{filename} unless defined($fn); - $fn = "-" unless defined($fn); + $fn = "-" unless defined($fn); - $self->{_filename} = $fn; # Filename to open when loading and saving - $self->{_configs} = []; # Storage, and to to preserve order - $self->{_lookup} = {}; # For faster lookup, indexes into the above - $self->{_ix} = -1; # Empty + $self->{_filename} = $fn; # Filename to open when loading and saving + $self->{_configs} = []; # Storage, and to to preserve order + $self->{_lookup} = {}; # For faster lookup, indexes into the above + $self->{_ix} = -1; # Empty bless $self, $class; $self->load() if $self->{_filename}; @@ -61,16 +59,16 @@ sub new { return $self; } - # # Load a records.config file # -sub load { +sub load +{ my $self = shift; my %args = @_; - my $fn = $args{file}; + my $fn = $args{file}; - $fn = $args{filename} unless defined($fn); + $fn = $args{filename} unless defined($fn); $fn = $self->{_filename} unless defined($fn); open(FH, "<$fn") || die "Can't open file $fn for reading"; @@ -88,14 +86,14 @@ sub load { } } - # # Get an existing configuration line, as an anon array. # -sub get { +sub get +{ my $self = shift; my %args = @_; - my $c = $args{conf}; + my $c = $args{conf}; $c = $args{config} unless defined($c); my $ix = $self->{_lookup}->{$c}; @@ -104,26 +102,26 @@ sub get { return $self->{_configs}->[$ix]; } - # # Modify one configuration value # -sub set { +sub set +{ my $self = shift; my %args = @_; - my $c = $args{conf}; - my $v = $args{val}; + my $c = $args{conf}; + my $v = $args{val}; $c = $args{config} unless defined($c); - $v = $args{value} unless defined($v); + $v = $args{value} unless defined($v); my $ix = $self->{_lookup}->{$c}; if (!defined($ix)) { - my $type = $args{type}; + my $type = $args{type}; - $type = "INT" unless defined($type); - $self->append(line => "CONFIG $c $type $v"); + $type = "INT" unless defined($type); + $self->append(line => "CONFIG $c $type $v"); } else { my $val = $self->{_configs}->[$ix]; @@ -132,14 +130,14 @@ sub set { } } - # # Remove a configuration from the file. # -sub remove { +sub remove +{ my $self = shift; my %args = @_; - my $c = $args{conf}; + my $c = $args{conf}; $c = $args{config} unless defined($c); @@ -148,11 +146,11 @@ sub remove { $self->{_configs}->[$ix]->[2] = TS_CONF_REMOVED if defined($ix); } - # # Append anything to the "end" of the configuration. # -sub append { +sub append +{ my $self = shift; my %args = @_; my $line = $args{line}; @@ -170,17 +168,17 @@ sub append { $self->{_lookup}->{$p[1]} = $self->{_ix} if ($#p == 3) && (($p[0] eq "LOCAL") || ($p[0] eq "CONFIG")); } - # # Write the new configuration file to STDOUT, or provided # -sub write { +sub write +{ my $self = shift; my %args = @_; - my $fn = $args{file}; + my $fn = $args{file}; $fn = $args{filename} unless defined($fn); - $fn = "-" unless defined($fn); + $fn = "-" unless defined($fn); if ($fn ne "-") { close(STDOUT); diff --git a/plugins/experimental/url_sig/genkeys.pl b/plugins/experimental/url_sig/genkeys.pl index ae5bc0723d0..38cc5235ecc 100755 --- a/plugins/experimental/url_sig/genkeys.pl +++ b/plugins/experimental/url_sig/genkeys.pl @@ -17,11 +17,11 @@ # limitations under the License. my $len = 32; -my @chars = ( 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '_' ); -foreach my $i ( 0 .. 15 ) { +my @chars = ('a' .. 'z', 'A' .. 'Z', '0' .. '9', '_'); +foreach my $i (0 .. 15) { my $string = ""; - foreach ( 1 .. $len ) { - $string .= $chars[ rand @chars ]; + foreach (1 .. $len) { + $string .= $chars[rand @chars]; } print "key" . $i . " = " . $string . "\n"; } diff --git a/plugins/experimental/url_sig/sign.pl b/plugins/experimental/url_sig/sign.pl index 6de4cc62eb8..7cf3850dd0c 100755 --- a/plugins/experimental/url_sig/sign.pl +++ b/plugins/experimental/url_sig/sign.pl @@ -22,50 +22,50 @@ use MIME::Base64::URLSafe (); use strict; use warnings; -my $key = undef; -my $string = undef; -my $useparts = undef; -my $result = undef; -my $duration = undef; -my $keyindex = undef; -my $verbose = 0; -my $url = undef; -my $client = undef; -my $algorithm = 1; +my $key = undef; +my $string = undef; +my $useparts = undef; +my $result = undef; +my $duration = undef; +my $keyindex = undef; +my $verbose = 0; +my $url = undef; +my $client = undef; +my $algorithm = 1; my $pathparams = 0; my $sig_anchor = undef; -my $proxy = undef; -my $scheme = "http://"; +my $proxy = undef; +my $scheme = "http://"; $result = GetOptions( - "url=s" => \$url, - "useparts=s" => \$useparts, - "duration=i" => \$duration, - "key=s" => \$key, - "client=s" => \$client, - "algorithm=i" => \$algorithm, - "keyindex=i" => \$keyindex, - "verbose" => \$verbose, - "pathparams" => \$pathparams, - "proxy=s" => \$proxy, - "siganchor=s" => \$sig_anchor + "url=s" => \$url, + "useparts=s" => \$useparts, + "duration=i" => \$duration, + "key=s" => \$key, + "client=s" => \$client, + "algorithm=i" => \$algorithm, + "keyindex=i" => \$keyindex, + "verbose" => \$verbose, + "pathparams" => \$pathparams, + "proxy=s" => \$proxy, + "siganchor=s" => \$sig_anchor ); -if ( !defined($key) || !defined($url) || !defined($duration) || !defined($keyindex) ) { - &help(); - exit(1); -} -if ( defined($proxy) ) { - if ($proxy !~ /http\:\/\/.*\:\d\d/) { +if (!defined($key) || !defined($url) || !defined($duration) || !defined($keyindex)) { &help(); - } + exit(1); +} +if (defined($proxy)) { + if ($proxy !~ /http\:\/\/.*\:\d\d/) { + &help(); + } } if ($url =~ m/^https/) { - $url =~ s/^https:\/\///; - $scheme = "https://"; + $url =~ s/^https:\/\///; + $scheme = "https://"; } else { - $url =~ s/^http:\/\///; + $url =~ s/^http:\/\///; } my $url_prefix = $url; @@ -77,87 +77,90 @@ my @inactive_parts = (); my $query_params = undef; -my $urlHasParams = index($url,"?"); -my $file = undef; +my $urlHasParams = index($url, "?"); +my $file = undef; my @parts = (split(/\//, $url)); my $parts_size = scalar(@parts); if ($pathparams) { - if (scalar(@parts) > 1) { - $file = pop @parts; - } else { - print STDERR "\nERROR: No file segment in the path when using --pathparams.\n\n"; - &help(); - exit 1; - } - if($urlHasParams) { - $file = (split(/\?/, $file))[0]; - } - $parts_size = scalar(@parts); + if (scalar(@parts) > 1) { + $file = pop @parts; + } else { + print STDERR "\nERROR: No file segment in the path when using --pathparams.\n\n"; + &help(); + exit 1; + } + if ($urlHasParams) { + $file = (split(/\?/, $file))[0]; + } + $parts_size = scalar(@parts); } if ($urlHasParams > 0) { - if ( ! $pathparams) { - ($parts[$parts_size -1], $query_params) = (split(/\?/, $parts[$parts_size - 1])); - } else { - $query_params = (split(/\?/, $url))[1]; - } + if (!$pathparams) { + ($parts[$parts_size - 1], $query_params) = (split(/\?/, $parts[$parts_size - 1])); + } else { + $query_params = (split(/\?/, $url))[1]; + } } foreach my $part (@parts) { - if ( length($useparts) > $i ) { - $part_active = substr( $useparts, $i++, 1 ); - } - if ($part_active) { - $string .= $part . "/"; - } - else { - $inactive_parts[$j] = $part; - } - $j++; + if (length($useparts) > $i) { + $part_active = substr($useparts, $i++, 1); + } + if ($part_active) { + $string .= $part . "/"; + } else { + $inactive_parts[$j] = $part; + } + $j++; } my $signing_signature = undef; chop($string); if ($pathparams) { - if ( defined($client) ) { - $signing_signature = ";C=" . $client . ";E=" . ( time() + $duration ) . ";A=" . $algorithm . ";K=" . $keyindex . ";P=" . $useparts . ";S="; - $string .= $signing_signature; - } - else { - $signing_signature = ";E=" . ( time() + $duration ) . ";A=" . $algorithm . ";K=" . $keyindex . ";P=" . $useparts . ";S="; - $string .= $signing_signature; - } -} else { - if ( defined($client) ) { - if ($urlHasParams > 0) { - $signing_signature = "?$query_params" . "&C=" . $client . "&E=" . ( time() + $duration ) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; - $string .= $signing_signature; - } - else { - $signing_signature = "?C=" . $client . "&E=" . ( time() + $duration ) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; - $string .= $signing_signature; - } - } - else { - if ($urlHasParams > 0) { - $signing_signature = "?$query_params" . "&E=" . ( time() + $duration ) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; - $string .= $signing_signature; + if (defined($client)) { + $signing_signature = + ";C=" . $client . ";E=" . (time() + $duration) . ";A=" . $algorithm . ";K=" . $keyindex . ";P=" . $useparts . ";S="; + $string .= $signing_signature; + } else { + $signing_signature = ";E=" . (time() + $duration) . ";A=" . $algorithm . ";K=" . $keyindex . ";P=" . $useparts . ";S="; + $string .= $signing_signature; } - else { - $signing_signature = "?E=" . ( time() + $duration ) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; - $string .= $signing_signature; +} else { + if (defined($client)) { + if ($urlHasParams > 0) { + $signing_signature = + "?$query_params" . "&C=" + . $client . "&E=" + . (time() + $duration) . "&A=" + . $algorithm . "&K=" + . $keyindex . "&P=" + . $useparts . "&S="; + $string .= $signing_signature; + } else { + $signing_signature = + "?C=" . $client . "&E=" . (time() + $duration) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; + $string .= $signing_signature; + } + } else { + if ($urlHasParams > 0) { + $signing_signature = + "?$query_params" . "&E=" . (time() + $duration) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; + $string .= $signing_signature; + } else { + $signing_signature = "?E=" . (time() + $duration) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; + $string .= $signing_signature; + } } - } } my $digest; -if ( $algorithm == 1 ) { - $digest = hmac_sha1_hex( $string, $key ); -} -else { - $digest = hmac_md5_hex( $string, $key ); +if ($algorithm == 1) { + $digest = hmac_sha1_hex($string, $key); +} else { + $digest = hmac_md5_hex($string, $key); } $verbose && print "\nSigned String: $string\n\n"; @@ -165,82 +168,96 @@ $verbose && print "\nsigning_signature: $signing_signature\n"; $verbose && print "\ndigest: $digest\n"; -if ($urlHasParams == -1) { # no application query parameters. - if ( ! defined($proxy)) { - if ( ! $pathparams) { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . $signing_signature . $digest . "'\n\n"; - } else { - my $index = rindex($url, '/'); - $url = substr($url,0,$index); - my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); - if (defined($sig_anchor)) { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/$file" . "'\n\n"; +if ($urlHasParams == -1) { # no application query parameters. + if (!defined($proxy)) { + if (!$pathparams) { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . $signing_signature . $digest . "'\n\n"; } else { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . "/" . $encoded . "/$file" . "'\n\n"; + my $index = rindex($url, '/'); + $url = substr($url, 0, $index); + my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); + if (defined($sig_anchor)) { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/$file" . "'\n\n"; + } else { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . "/" . $encoded . "/$file" . "'\n\n"; + } } - } } else { - if ( ! $pathparams) { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . $signing_signature . $digest . - "'\n\n"; - } else { - my $index = rindex($url, '/'); - $url = substr($url,0,$index); - my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); - if (defined($sig_anchor)) { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/$file" . "'\n\n"; + if (!$pathparams) { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . $signing_signature . $digest . "'\n\n"; } else { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . "/" . $encoded . "/$file" . "'\n\n"; + my $index = rindex($url, '/'); + $url = substr($url, 0, $index); + my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); + if (defined($sig_anchor)) { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" + . $url + . ";${sig_anchor}=" + . $encoded + . "/$file" . "'\n\n"; + } else { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . "/" . $encoded . "/$file" . "'\n\n"; + } } - } } -} else { # has application parameters. +} else { # has application parameters. $url = (split(/\?/, $url))[0]; - if ( ! defined($proxy)) { - if ( ! $pathparams) { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . $signing_signature . $digest . "'\n\n"; - } else { - my $index = rindex($url, '/'); - $url = substr($url,0,$index); - my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); - if (defined($sig_anchor)) { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/" . $file . "?$query_params" - . "'\n\n"; + if (!defined($proxy)) { + if (!$pathparams) { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . $signing_signature . $digest . "'\n\n"; } else { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . "/" . $encoded . "/" . $file . "?$query_params" - . "'\n\n"; + my $index = rindex($url, '/'); + $url = substr($url, 0, $index); + my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); + if (defined($sig_anchor)) { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" + . $url + . ";${sig_anchor}=" + . $encoded . "/" + . $file + . "?$query_params" . "'\n\n"; + } else { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . "/" . $encoded . "/" . $file . "?$query_params" + . "'\n\n"; + } } - } } else { - if ( ! $pathparams) { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . $signing_signature . $digest . - "'\n\n"; - } else { - my $index = rindex($url, '/'); - $url = substr($url,0,$index); - my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); - if (defined($sig_anchor)) { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/" . $file . "?$query_params" - . "'\n\n"; + if (!$pathparams) { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . $signing_signature . $digest . "'\n\n"; } else { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . "/" . $encoded . "/$file?$query_params" . "'\n\n"; + my $index = rindex($url, '/'); + $url = substr($url, 0, $index); + my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); + if (defined($sig_anchor)) { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" + . $url + . ";${sig_anchor}=" + . $encoded . "/" + . $file + . "?$query_params" . "'\n\n"; + } else { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" + . $url . "/" + . $encoded + . "/$file?$query_params" . "'\n\n"; + } } - } } } -sub help { - print "sign.pl - Example signing utility in perl for signed URLs\n"; - print "Usage: \n"; - print " ./sign.pl --url \\ \n"; - print " --useparts \\ \n"; - print " --algorithm \\ \n"; - print " --duration \\ \n"; - print " --keyindex \\ \n"; - print " [--client ] \\ \n"; - print " --key \\ \n"; - print " [--verbose] \n"; - print " [--pathparams] \n"; - print " [--proxy ] ex value: http://myproxy:80\n"; - print "\n"; +sub help +{ + print "sign.pl - Example signing utility in perl for signed URLs\n"; + print "Usage: \n"; + print " ./sign.pl --url \\ \n"; + print " --useparts \\ \n"; + print " --algorithm \\ \n"; + print " --duration \\ \n"; + print " --keyindex \\ \n"; + print " [--client ] \\ \n"; + print " --key \\ \n"; + print " [--verbose] \n"; + print " [--pathparams] \n"; + print " [--proxy ] ex value: http://myproxy:80\n"; + print "\n"; } diff --git a/proxy/http/test_http_client.pl b/proxy/http/test_http_client.pl index 68e6d9dde52..24a62811e3d 100644 --- a/proxy/http/test_http_client.pl +++ b/proxy/http/test_http_client.pl @@ -28,7 +28,6 @@ sub make_doc_filename($); sub make_doc_http_filename($); - ########################################################### # # global configuration parameters @@ -44,54 +43,39 @@ ########################################################### sub process_input_http_requests_file($$$) { - my ($filename, $proxy_name, $proxy_port) = @_; - my ($input, $host_name, $host_port, $request, $line); - - #open input file for read - unless (open input, "<$filename") - { - print "cannot open $filename: $!\n"; - return; - } - - while ($line = ) - { - $request .= $line; - #replace \n with \r\n - if (not $line =~ m/\r/) - { - $line =~ s/\n/\r\n/; - } - if ($line =~ m/host/i) - { - ($_, $host_name, $host_port) = split( /:/, $line, 3); - if (not $host_port) - { - $host_port = 80; - } - } - elsif (length($line) <= 2 && $line == "\n") - { - $request .= $line; - if ($proxy_name and $proxy_port) - { - $request = make_proxy_request( - $request, - $host_name, - $host_port, - $proxy_name, - $proxy_port); - spawn_http_request($proxy_name, $proxy_port, $request); - } - else - { - print $request; - spawn_http_request($host_name, $host_port, $request); - } - $request = ""; - } - } - return; + my ($filename, $proxy_name, $proxy_port) = @_; + my ($input, $host_name, $host_port, $request, $line); + + #open input file for read + unless (open input, "<$filename") { + print "cannot open $filename: $!\n"; + return; + } + + while ($line = ) { + $request .= $line; + #replace \n with \r\n + if (not $line =~ m/\r/) { + $line =~ s/\n/\r\n/; + } + if ($line =~ m/host/i) { + ($_, $host_name, $host_port) = split(/:/, $line, 3); + if (not $host_port) { + $host_port = 80; + } + } elsif (length($line) <= 2 && $line == "\n") { + $request .= $line; + if ($proxy_name and $proxy_port) { + $request = make_proxy_request($request, $host_name, $host_port, $proxy_name, $proxy_port); + spawn_http_request($proxy_name, $proxy_port, $request); + } else { + print $request; + spawn_http_request($host_name, $host_port, $request); + } + $request = ""; + } + } + return; } ########################################################### # @@ -100,21 +84,18 @@ ($$$) ########################################################### sub spawn_http_request($$$) { - my($hostname, $hostport, $request) = @_; - - my ($pid); - if (!defined ($pid = fork)) - { - print "fork failed", "\n"; - exit; - } - elsif ($pid) - { # parent - return; - } - # else, I am the child - do_http_request ($hostname, $hostport, $request); - exit; + my ($hostname, $hostport, $request) = @_; + + my ($pid); + if (!defined($pid = fork)) { + print "fork failed", "\n"; + exit; + } elsif ($pid) { # parent + return; + } + # else, I am the child + do_http_request($hostname, $hostport, $request); + exit; } ########################################################### # @@ -123,33 +104,31 @@ ($$$) ########################################################### sub spawn_http_request($$$) { - my ($hostname, $hostport, $request) = @_; - my ($line); - my ($iaddr, $paddr, $proto); - - $hostname =~ s/\s//g; - - $iaddr = inet_aton($hostname) or die "no host: $hostname", "\n"; - $paddr = sockaddr_in($hostport, $iaddr); - $proto = getprotobyname('tcp'); - - unless (socket(Host, PF_INET, SOCK_STREAM, $proto)) - { - print "socket: $!", "\n"; - exit; - } - unless (connect(Host, $paddr)) - { - print "connect: $!", "\n"; - exit; - } - syswrite Host, $request, length($request); - #process response - process_http_response($request, $Host, 1, 1); - print "request is done\n"; - close (Host); - - return; + my ($hostname, $hostport, $request) = @_; + my ($line); + my ($iaddr, $paddr, $proto); + + $hostname =~ s/\s//g; + + $iaddr = inet_aton($hostname) or die "no host: $hostname", "\n"; + $paddr = sockaddr_in($hostport, $iaddr); + $proto = getprotobyname('tcp'); + + unless (socket(Host, PF_INET, SOCK_STREAM, $proto)) { + print "socket: $!", "\n"; + exit; + } + unless (connect(Host, $paddr)) { + print "connect: $!", "\n"; + exit; + } + syswrite Host, $request, length($request); + #process response + process_http_response($request, $Host, 1, 1); + print "request is done\n"; + close(Host); + + return; } ########################################################### # @@ -165,62 +144,52 @@ ($$$) ########################################################### sub process_http_response($$$$) { - my ($request, $Host, $save_doc_flag, $save_http_flag) = @_; - my ($doc_filename, $http_filename); - - my ($doc_filename) = make_doc_filename($request); - my ($doc_http_filename) = make_doc_http_filename($request); - - print $doc_filename, ' ', $doc_http_filename, "\n"; - - my ($doc_file, $doc_http_file); - ######################## - # open files for write # - ######################## - if ($save_doc_flag) - { - unless (open doc_file, ">$doc_filename") - { - print "cannot open $doc_filename for write", "\n"; - return; - } - } - if ($save_http_flag) - { - unless (open doc_http_file, ">$doc_http_filename") - { - print "cannot open $doc_http_filename for write", "\n"; - return; - } - } - ############################## - # write http header and body # - ############################## - my ($http_header) = 1; - my ($doc_body) = 0; - my ($line); - - while ($line = ) - { - if ($http_header) - { - if ($save_http_flag) - { - print doc_http_file $line; - } - if (length($line) <= 2 && $line == "\n") - { - close doc_http_file; - $http_header = 0; - $doc_body = 1; - } - } - elsif ($save_doc_flag) - { - print doc_file $line; - } - } - return; + my ($request, $Host, $save_doc_flag, $save_http_flag) = @_; + my ($doc_filename, $http_filename); + + my ($doc_filename) = make_doc_filename($request); + my ($doc_http_filename) = make_doc_http_filename($request); + + print $doc_filename, ' ', $doc_http_filename, "\n"; + + my ($doc_file, $doc_http_file); + ######################## + # open files for write # + ######################## + if ($save_doc_flag) { + unless (open doc_file, ">$doc_filename") { + print "cannot open $doc_filename for write", "\n"; + return; + } + } + if ($save_http_flag) { + unless (open doc_http_file, ">$doc_http_filename") { + print "cannot open $doc_http_filename for write", "\n"; + return; + } + } + ############################## + # write http header and body # + ############################## + my ($http_header) = 1; + my ($doc_body) = 0; + my ($line); + + while ($line = ) { + if ($http_header) { + if ($save_http_flag) { + print doc_http_file $line; + } + if (length($line) <= 2 && $line == "\n") { + close doc_http_file; + $http_header = 0; + $doc_body = 1; + } + } elsif ($save_doc_flag) { + print doc_file $line; + } + } + return; } ########################################################### # @@ -234,21 +203,20 @@ ($$$$) ########################################################### sub make_proxy_request($$$$$) { - my ($request, $host_name, $host_port, $proxy_name, $proxy_port) = @_; - my ($proxy_request) = $request; + my ($request, $host_name, $host_port, $proxy_name, $proxy_port) = @_; + my ($proxy_request) = $request; - my ($url_prefix) = "http:\/\/$host_name\/"; - $url_prefix =~ s/\s//g; + my ($url_prefix) = "http:\/\/$host_name\/"; + $url_prefix =~ s/\s//g; - if ($host_port != 80) - { - $url_prefix .= ":$host_port\/"; - } - $url_prefix =~ s/\s//g; + if ($host_port != 80) { + $url_prefix .= ":$host_port\/"; + } + $url_prefix =~ s/\s//g; - $proxy_request =~ s/\//$url_prefix/; + $proxy_request =~ s/\//$url_prefix/; - return ($proxy_request); + return ($proxy_request); } ########################################################### # @@ -258,45 +226,44 @@ ($$$$$) ########################################################### sub make_doc_filename($) { - my ($request) = @_; - my ($doc_filename); - my ($host_name); - - ($_, $host_name) = split (/host:/i, $request, 2); - ($host_name, $_) = split (/ /, $host_name); - #replace every . with _ - $host_name =~ s/\./_/g; - - print $request, "\n"; - print $host_name, "\n"; - - ($_, $doc_filename) = split (/ /, $request, 2); - #remove scheme://host_name if this is a proxy request -# if ($doc_filename =~ m/:\/\//) -# { -# -# } -# -# @@@@@@@@ + my ($request) = @_; + my ($doc_filename); + my ($host_name); + + ($_, $host_name) = split(/host:/i, $request, 2); + ($host_name, $_) = split(/ /, $host_name); + #replace every . with _ + $host_name =~ s/\./_/g; - ($_, $doc_filename) = split (/\//, $doc_filename, 2); - $doc_filename =~ s/\//_/g; - #remove any white spaces - $doc_filename =~ s/\s//g; + print $request, "\n"; + print $host_name, "\n"; - print "doc name is: ", $doc_filename, "\n"; + ($_, $doc_filename) = split(/ /, $request, 2); + #remove scheme://host_name if this is a proxy request + # if ($doc_filename =~ m/:\/\//) + # { + # + # } + # + # @@@@@@@@ - if (length($doc_filename) <= 1) - { - $doc_filename = 'default.html'; - } + ($_, $doc_filename) = split(/\//, $doc_filename, 2); + $doc_filename =~ s/\//_/g; + #remove any white spaces + $doc_filename =~ s/\s//g; - $doc_filename = $host_name . '_' . $doc_filename; + print "doc name is: ", $doc_filename, "\n"; - #remove any white spaces - $doc_filename =~ s/\s//g; + if (length($doc_filename) <= 1) { + $doc_filename = 'default.html'; + } - return ($doc_filename); + $doc_filename = $host_name . '_' . $doc_filename; + + #remove any white spaces + $doc_filename =~ s/\s//g; + + return ($doc_filename); } ########################################################### # @@ -305,43 +272,34 @@ ($) ########################################################### sub make_doc_http_filename($) { - my ($request) = @_; - my ($doc_http_filename); + my ($request) = @_; + my ($doc_http_filename); - $doc_http_filename = make_doc_filename($request); - $doc_http_filename .= '.http'; + $doc_http_filename = make_doc_filename($request); + $doc_http_filename .= '.http'; - return ($doc_http_filename); + return ($doc_http_filename); } ########################################################### # # main entry point # ########################################################### -if ($#ARGV != 1 and $#ARGV != 3) -{ - print 'no proxy : test_http_client ', "\n"; - print 'use proxy: test_http_client '; - print ' ', "\n"; - exit; +if ($#ARGV != 1 and $#ARGV != 3) { + print 'no proxy : test_http_client ', "\n"; + print 'use proxy: test_http_client '; + print ' ', "\n"; + exit; } -if ($#ARGV == 1) -{ - my ($infile, $nusers) = @ARGV; - process_input_http_requests_file($infile, "", ""); -} -elsif ($#ARGV == 3) -{ - my ($infile, $nusers, $proxy_name, $proxy_port) = @ARGV; - process_input_http_requests_file($infile, $proxy_name, $proxy_port); +if ($#ARGV == 1) { + my ($infile, $nusers) = @ARGV; + process_input_http_requests_file($infile, "", ""); +} elsif ($#ARGV == 3) { + my ($infile, $nusers, $proxy_name, $proxy_port) = @ARGV; + process_input_http_requests_file($infile, $proxy_name, $proxy_port); } print "\n"; exit; - - - - - diff --git a/proxy/http/test_proxy.pl b/proxy/http/test_proxy.pl index 94de3752ee4..646547703e7 100644 --- a/proxy/http/test_proxy.pl +++ b/proxy/http/test_proxy.pl @@ -29,16 +29,15 @@ sub make_doc_filename($); sub make_doc_http_filename($); - ########################################################### # # global configuration parameters # ########################################################### -glob ($number_of_users) = 1; -glob ($save_http_doc) = 0; #if false (0) don't save a copy - #of the doc and header files. -glob ($method) = "GET"; #method to use in http requests +glob($number_of_users) = 1; +glob($save_http_doc) = 0; #if false (0) don't save a copy + #of the doc and header files. +glob($method) = "GET"; #method to use in http requests ########################################################### # @@ -48,13 +47,13 @@ ########################################################### sub compare_files($$$) { - my ($dfile, $pfile, $log_file) = @_; - @args = ("diff", $dfile, $pfile, ">>", $log_file); + my ($dfile, $pfile, $log_file) = @_; + @args = ("diff", $dfile, $pfile, ">>", $log_file); - #diff returns 0 if files are identical - $is_diff = system (@args); + #diff returns 0 if files are identical + $is_diff = system(@args); - return ($is_diff); + return ($is_diff); } ########################################################### # @@ -68,21 +67,18 @@ ($$$) ########################################################### sub spawn_task($$$$) { - my($hostname, $hostport, $request, $run_task) = @_; - - my ($pid); - if (!defined ($pid = fork)) - { - print "fork failed", "\n"; - exit; - } - elsif ($pid) - { # parent - return; - } - # else, I am the child - run_task ($hostname, $hostport, $request); - exit; + my ($hostname, $hostport, $request, $run_task) = @_; + + my ($pid); + if (!defined($pid = fork)) { + print "fork failed", "\n"; + exit; + } elsif ($pid) { # parent + return; + } + # else, I am the child + run_task($hostname, $hostport, $request); + exit; } ########################################################### # @@ -91,236 +87,214 @@ ($$$$) ########################################################### sub run_proxy_keep_alive { - my ($proxy_host_name, $proxy_port, -} + my ( + $proxy_host_name, $proxy_port,; + } -@@@@@@@@@@ + @@@@@@@@@@ ########################################################### -# -# subroutine: do_http_request hostname request -# + # + # subroutine: do_http_request hostname request + # ########################################################### -sub do_http_request($$$) -{ - my ($hostname, $hostport, $request) = @_; - my ($line); - my ($iaddr, $paddr, $proto); - - $hostname =~ s/\s//g; - - $iaddr = inet_aton($hostname) or die "no host: $hostname", "\n"; - $paddr = sockaddr_in($hostport, $iaddr); - $proto = getprotobyname('tcp'); - - unless (socket(Host, PF_INET, SOCK_STREAM, $proto)) - { - print "socket: $!", "\n"; - exit; - } - unless (connect(Host, $paddr)) - { - print "connect: $!", "\n"; - exit; - } - syswrite Host, $request, length($request); - #process response - process_http_response($request, $Host, 1, 1); - print "request is done\n"; - close (Host); - - return; -} + sub do_http_request($$$) + { + my ($hostname, $hostport, $request) = @_; + my ($line); + my ($iaddr, $paddr, $proto); + + $hostname =~ s/\s//g; + + $iaddr = inet_aton($hostname) or die "no host: $hostname", "\n"; + $paddr = sockaddr_in($hostport, $iaddr); + $proto = getprotobyname('tcp'); + + unless (socket(Host, PF_INET, SOCK_STREAM, $proto)) { + print "socket: $!", "\n"; + exit; + } + unless (connect(Host, $paddr)) { + print "connect: $!", "\n"; + exit; + } + syswrite Host, $request, length($request); + #process response + process_http_response($request, $Host, 1, 1); + print "request is done\n"; + close(Host); + + return; + } ########################################################### -# -# subroutine: process_http_response -# request, -# host_socket, -# save_doc_flag, -# save_http_flag -# -# options for save doc -# - save http response header in doc.http -# - save http doc in a unique file + # + # subroutine: process_http_response + # request, + # host_socket, + # save_doc_flag, + # save_http_flag + # + # options for save doc + # - save http response header in doc.http + # - save http doc in a unique file ########################################################### -sub process_http_response($$$$) -{ - my ($request, $Host, $save_doc_flag, $save_http_flag) = @_; - my ($doc_filename, $http_filename); - - my ($doc_filename) = make_doc_filename($request); - my ($doc_http_filename) = make_doc_http_filename($request); - - print $doc_filename, ' ', $doc_http_filename, "\n"; - - my ($doc_file, $doc_http_file); - ######################## - # open files for write # - ######################## - if ($save_doc_flag) - { - unless (open doc_file, ">$doc_filename") - { - print "cannot open $doc_filename for write", "\n"; - return; - } - } - if ($save_http_flag) - { - unless (open doc_http_file, ">$doc_http_filename") - { - print "cannot open $doc_http_filename for write", "\n"; - return; - } - } - ############################## - # write http header and body # - ############################## - my ($http_header) = 1; - my ($doc_body) = 0; - my ($line); - - while ($line = ) - { - if ($http_header) - { - if ($save_http_flag) - { - print doc_http_file $line; - } - if (length($line) <= 2 && $line == "\n") - { - close doc_http_file; - $http_header = 0; - $doc_body = 1; - } - } - elsif ($save_doc_flag) - { - print doc_file $line; - } - } - return; -} + sub process_http_response($$$$) + { + my ($request, $Host, $save_doc_flag, $save_http_flag) = @_; + my ($doc_filename, $http_filename); + + my ($doc_filename) = make_doc_filename($request); + my ($doc_http_filename) = make_doc_http_filename($request); + + print $doc_filename, ' ', $doc_http_filename, "\n"; + + my ($doc_file, $doc_http_file); + ######################## + # open files for write # + ######################## + if ($save_doc_flag) { + unless (open doc_file, ">$doc_filename") { + print "cannot open $doc_filename for write", "\n"; + return; + } + } + if ($save_http_flag) { + unless (open doc_http_file, ">$doc_http_filename") { + print "cannot open $doc_http_filename for write", "\n"; + return; + } + } + ############################## + # write http header and body # + ############################## + my ($http_header) = 1; + my ($doc_body) = 0; + my ($line); + + while ($line = ) { + if ($http_header) { + if ($save_http_flag) { + print doc_http_file $line; + } + if (length($line) <= 2 && $line == "\n") { + close doc_http_file; + $http_header = 0; + $doc_body = 1; + } + } elsif ($save_doc_flag) { + print doc_file $line; + } + } + return; + } ########################################################### -# -# subroutine: make_proxy_request -# request -# host_name -# host_port -# proxy_name -# proxy_port -# + # + # subroutine: make_proxy_request + # request + # host_name + # host_port + # proxy_name + # proxy_port + # ########################################################### -sub make_proxy_request($$$$$) -{ - my ($request, $host_name, $host_port, $proxy_name, $proxy_port) = @_; - my ($proxy_request) = $request; + sub make_proxy_request($$$$$) + { + my ($request, $host_name, $host_port, $proxy_name, $proxy_port) = @_; + my ($proxy_request) = $request; - my ($url_prefix) = "http:\/\/$host_name\/"; - $url_prefix =~ s/\s//g; + my ($url_prefix) = "http:\/\/$host_name\/"; + $url_prefix =~ s/\s//g; - if ($host_port != 80) - { - $url_prefix .= ":$host_port\/"; - } - $url_prefix =~ s/\s//g; + if ($host_port != 80) { + $url_prefix .= ":$host_port\/"; + } + $url_prefix =~ s/\s//g; - $proxy_request =~ s/\//$url_prefix/; + $proxy_request =~ s/\//$url_prefix/; - return ($proxy_request); -} + return ($proxy_request); + } ########################################################### -# -# subroutine: make_doc_filename request -# -# file name is: + # + # subroutine: make_doc_filename request + # + # file name is: ########################################################### -sub make_doc_filename($) -{ - my ($request) = @_; - my ($doc_filename); - my ($host_name); - - ($_, $host_name) = split (/host:/i, $request, 2); - ($host_name, $_) = split (/ /, $host_name); - #replace every . with _ - $host_name =~ s/\./_/g; - - print $request, "\n"; - print $host_name, "\n"; - - ($_, $doc_filename) = split (/ /, $request, 2); - #remove scheme://host_name if this is a proxy request -# if ($doc_filename =~ m/:\/\//) -# { -# -# } -# -# @@@@@@@@ - - ($_, $doc_filename) = split (/\//, $doc_filename, 2); - $doc_filename =~ s/\//_/g; - #remove any white spaces - $doc_filename =~ s/\s//g; - - print "doc name is: ", $doc_filename, "\n"; - - if (length($doc_filename) <= 1) - { - $doc_filename = 'default.html'; - } - - $doc_filename = $host_name . '_' . $doc_filename; - - #remove any white spaces - $doc_filename =~ s/\s//g; - - return ($doc_filename); -} + sub make_doc_filename($) + { + my ($request) = @_; + my ($doc_filename); + my ($host_name); + + ($_, $host_name) = split(/host:/i, $request, 2); + ($host_name, $_) = split(/ /, $host_name); + #replace every . with _ + $host_name =~ s/\./_/g; + + print $request, "\n"; + print $host_name, "\n"; + + ($_, $doc_filename) = split(/ /, $request, 2); + #remove scheme://host_name if this is a proxy request + # if ($doc_filename =~ m/:\/\//) + # { + # + # } + # + # @@@@@@@@ + + ($_, $doc_filename) = split(/\//, $doc_filename, 2); + $doc_filename =~ s/\//_/g; + #remove any white spaces + $doc_filename =~ s/\s//g; + + print "doc name is: ", $doc_filename, "\n"; + + if (length($doc_filename) <= 1) { + $doc_filename = 'default.html'; + } + + $doc_filename = $host_name . '_' . $doc_filename; + + #remove any white spaces + $doc_filename =~ s/\s//g; + + return ($doc_filename); + } ########################################################### -# -# subroutine: make_doc_filename request -# + # + # subroutine: make_doc_filename request + # ########################################################### -sub make_doc_http_filename($) -{ - my ($request) = @_; - my ($doc_http_filename); + sub make_doc_http_filename($) + { + my ($request) = @_; + my ($doc_http_filename); - $doc_http_filename = make_doc_filename($request); - $doc_http_filename .= '.http'; + $doc_http_filename = make_doc_filename($request); + $doc_http_filename .= '.http'; - return ($doc_http_filename); -} + return ($doc_http_filename); + } ########################################################### -# -# main entry point -# + # + # main entry point + # ########################################################### -if ($#ARGV != 1 and $#ARGV != 3) -{ - print 'no proxy : test_http_client ', "\n"; - print 'use proxy: test_http_client '; - print ' ', "\n"; - exit; -} - -if ($#ARGV == 1) -{ - my ($infile, $nusers) = @ARGV; - process_input_http_requests_file($infile, "", ""); -} -elsif ($#ARGV == 3) -{ - my ($infile, $nusers, $proxy_name, $proxy_port) = @ARGV; - process_input_http_requests_file($infile, $proxy_name, $proxy_port); -} - -print "\n"; -exit; - - - - - + if ($#ARGV != 1 and $#ARGV != 3) { + print 'no proxy : test_http_client ', "\n"; + print 'use proxy: test_http_client '; + print ' ', "\n"; + exit; + } + + if ($#ARGV == 1) { + my ($infile, $nusers) = @ARGV; + process_input_http_requests_file($infile, "", ""); + } elsif ($#ARGV == 3) { + my ($infile, $nusers, $proxy_name, $proxy_port) = @ARGV; + process_input_http_requests_file($infile, $proxy_name, $proxy_port); + } + + print "\n"; + exit; diff --git a/tools/changelog.pl b/tools/changelog.pl index ba62f093bd6..0de170e193d 100755 --- a/tools/changelog.pl +++ b/tools/changelog.pl @@ -22,105 +22,96 @@ use WWW::Curl::Easy; use JSON; -my $owner = shift; -my $repo = shift; +my $owner = shift; +my $repo = shift; my $milestone = shift; -my $auth = shift; -my $url = "https://api.github.com"; +my $auth = shift; +my $url = "https://api.github.com"; sub rate_fail { - print STDERR "You have exceeded your rate limit. Try using an auth token.\n"; - exit 2; + print STDERR "You have exceeded your rate limit. Try using an auth token.\n"; + exit 2; } sub milestone_lookup { - my $curl = shift; - my $url = shift; - my $owner = shift; - my $repo = shift; - my $milestone_title = shift; - my $endpoint = "/repos/$owner/$repo/milestones"; - - my $resp_body; - - $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); - $curl->setopt(CURLOPT_URL, $url . $endpoint); - - my $retcode = $curl->perform(); - if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 200) - { - my $milestones = from_json($resp_body); - foreach my $milestone (@{ $milestones }) - { - if ($milestone->{title} eq $milestone_title) - { - return $milestone->{number}; - } + my $curl = shift; + my $url = shift; + my $owner = shift; + my $repo = shift; + my $milestone_title = shift; + my $endpoint = "/repos/$owner/$repo/milestones"; + + my $resp_body; + + $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); + $curl->setopt(CURLOPT_URL, $url . $endpoint); + + my $retcode = $curl->perform(); + if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 200) { + my $milestones = from_json($resp_body); + foreach my $milestone (@{$milestones}) { + if ($milestone->{title} eq $milestone_title) { + return $milestone->{number}; + } + } + } elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) { + rate_fail(); } - } - elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) - { - rate_fail(); - } - undef; + undef; } sub is_merged { - my $curl = shift; - my $url = shift; - my $owner = shift; - my $repo = shift; - my $issue_id = shift; - my $endpoint = "/repos/$owner/$repo/pulls/$issue_id/merge"; - - my $resp_body; - - $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); - $curl->setopt(CURLOPT_URL, $url . $endpoint); - - my $retcode = $curl->perform(); - if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 204) { - return 1; - } - elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) - { - rate_fail(); - } - - undef; + my $curl = shift; + my $url = shift; + my $owner = shift; + my $repo = shift; + my $issue_id = shift; + my $endpoint = "/repos/$owner/$repo/pulls/$issue_id/merge"; + + my $resp_body; + + $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); + $curl->setopt(CURLOPT_URL, $url . $endpoint); + + my $retcode = $curl->perform(); + if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 204) { + return 1; + } elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) { + rate_fail(); + } + + undef; } sub issue_search { - my $curl = shift; - my $url = shift; - my $owner = shift; - my $repo = shift; - my $milestone_id = shift; - my $page = shift; - my $endpoint = "/repos/$owner/$repo/issues"; - - my $params = "milestone=$milestone_id&state=closed&page=$page"; - - my $resp_body; - - $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); - $curl->setopt(CURLOPT_URL, $url . $endpoint . '?' . $params); - - my $retcode = $curl->perform(); - if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 200) { - return from_json($resp_body); - } - elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) - { - rate_fail(); - } - - undef; + my $curl = shift; + my $url = shift; + my $owner = shift; + my $repo = shift; + my $milestone_id = shift; + my $page = shift; + my $endpoint = "/repos/$owner/$repo/issues"; + + my $params = "milestone=$milestone_id&state=closed&page=$page"; + + my $resp_body; + + $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); + $curl->setopt(CURLOPT_URL, $url . $endpoint . '?' . $params); + + my $retcode = $curl->perform(); + if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 200) { + return from_json($resp_body); + } elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) { + rate_fail(); + } + + undef; } my $curl = WWW::Curl::Easy->new; @@ -128,17 +119,15 @@ sub issue_search #$curl->setopt(CURLOPT_VERBOSE, 1); $curl->setopt(CURLOPT_HTTPHEADER, ['Accept: application/vnd.github.v3+json', 'User-Agent: Awesome-Octocat-App']); -if (defined($auth)) -{ - $curl->setopt(CURLOPT_USERPWD, $auth); +if (defined($auth)) { + $curl->setopt(CURLOPT_USERPWD, $auth); } my $milestone_id = milestone_lookup($curl, $url, $owner, $repo, $milestone); -if (!defined($milestone_id)) -{ - print STDERR "Milestone not found!\n"; - exit 1; +if (!defined($milestone_id)) { + print STDERR "Milestone not found!\n"; + exit 1; } my $issues; @@ -148,39 +137,33 @@ sub issue_search print STDERR "Looking for issues from Milestone $milestone\n"; do { - print STDERR "Page $page\n"; - $issues = issue_search($curl, $url, $owner, $repo, $milestone_id, $page); - foreach my $issue (@{ $issues }) - { - if (defined($issue)) - { - print STDERR "Issue #" . $issue->{number} . " - " . $issue->{title} . " "; - - if (!exists($issue->{pull_request})) - { - print STDERR "not a PR.\n"; - next; - } - - if (!is_merged($curl, $url, $owner, $repo, $issue->{number})) - { - print STDERR "not merged.\n"; - next; - } - - print STDERR "added.\n"; - push @{ $changelog }, {number => $issue->{number}, title => $issue->{title}}; + print STDERR "Page $page\n"; + $issues = issue_search($curl, $url, $owner, $repo, $milestone_id, $page); + foreach my $issue (@{$issues}) { + if (defined($issue)) { + print STDERR "Issue #" . $issue->{number} . " - " . $issue->{title} . " "; + + if (!exists($issue->{pull_request})) { + print STDERR "not a PR.\n"; + next; + } + + if (!is_merged($curl, $url, $owner, $repo, $issue->{number})) { + print STDERR "not merged.\n"; + next; + } + + print STDERR "added.\n"; + push @{$changelog}, {number => $issue->{number}, title => $issue->{title}}; + } } - } - $page++; -} while (scalar @{ $issues }); + $page++; +} while (scalar @{$issues}); -if (defined($changelog)) -{ - print "Changes with Apache Traffic Server $milestone\n"; +if (defined($changelog)) { + print "Changes with Apache Traffic Server $milestone\n"; - foreach my $issue (sort {$a->{number} <=> $b->{number}} @{ $changelog }) - { - print " #$issue->{number} - $issue->{title}\n"; - } + foreach my $issue (sort {$a->{number} <=> $b->{number}} @{$changelog}) { + print " #$issue->{number} - $issue->{title}\n"; + } } diff --git a/tools/compare_records.pl b/tools/compare_records.pl index cf0046360b3..eb16cd4b17d 100755 --- a/tools/compare_records.pl +++ b/tools/compare_records.pl @@ -38,35 +38,39 @@ use warnings; use Getopt::Long; -my($file1, $file2, $in_files, $help); +my ($file1, $file2, $in_files, $help); my %file1_settings; my %file2_settings; my $diff_metrics; -usage() if (@ARGV < 1 or - !GetOptions( - 'f=s@' => \$in_files, - 'm' => \$diff_metrics, - 'help|?' => \$help) or - defined $help); +usage() + if ( + @ARGV < 1 + or !GetOptions( + 'f=s@' => \$in_files, + 'm' => \$diff_metrics, + 'help|?' => \$help + ) + or defined $help + ); # Input file is mandatory die "\nTwo input files must be specified to compare\n" - unless defined $in_files; + unless defined $in_files; # Print the usage sub usage { - print "Unknown option: @_\n" if (@_); - print "Provide 2 files to compare configs or metrics.\n"; - print "By default this tool will diff only configs,\n"; - print "to get diff of metrics pass -m flag\n\n"; - print "Usage: compare_records.pl -m -f -f \n"; - print " -m to diff the metrics\n"; - print " -h for help\n\n"; - print "where the files are generated with e.g.\n\n"; - print " \$ traffic_ctl config match .\n"; - exit; + print "Unknown option: @_\n" if (@_); + print "Provide 2 files to compare configs or metrics.\n"; + print "By default this tool will diff only configs,\n"; + print "to get diff of metrics pass -m flag\n\n"; + print "Usage: compare_records.pl -m -f -f \n"; + print " -m to diff the metrics\n"; + print " -h for help\n\n"; + print "where the files are generated with e.g.\n\n"; + print " \$ traffic_ctl config match .\n"; + exit; } my @file_list = @$in_files; @@ -76,89 +80,88 @@ sub usage # Open input files if (defined $in_file1) { - open $file1, $in_file1 or die "Could not open $in_file1: $!"; + open $file1, $in_file1 or die "Could not open $in_file1: $!"; } if (defined $in_file2) { - open $file2, $in_file2 or die "Could not open $in_file2: $!"; + open $file2, $in_file2 or die "Could not open $in_file2: $!"; } # Read input files while (my $setting = <$file1>) { - chomp $setting; - my($record, $value) = split(/:/, $setting); - if (defined $diff_metrics) { - # Obtain only metrics, excluding configs - if ($record !~ /proxy.config/) { - $file1_settings{$record} = $value; - } - } else { - # Obtain only configs - if ($record =~ /proxy.config/) { - $file1_settings{$record} = $value; + chomp $setting; + my ($record, $value) = split(/:/, $setting); + if (defined $diff_metrics) { + # Obtain only metrics, excluding configs + if ($record !~ /proxy.config/) { + $file1_settings{$record} = $value; + } + } else { + # Obtain only configs + if ($record =~ /proxy.config/) { + $file1_settings{$record} = $value; + } } - } } close $file1; while (my $setting = <$file2>) { - chomp $setting; - my($record, $value) = split(/:/, $setting); - if (defined $diff_metrics) { - # Obtain only metrics, excluding configs - if ($record !~ /proxy.config/) { - $file2_settings{$record} = $value; - } - } else { - # Obtain only configs - if ($record =~ /proxy.config/) { - $file2_settings{$record} = $value; + chomp $setting; + my ($record, $value) = split(/:/, $setting); + if (defined $diff_metrics) { + # Obtain only metrics, excluding configs + if ($record !~ /proxy.config/) { + $file2_settings{$record} = $value; + } + } else { + # Obtain only configs + if ($record =~ /proxy.config/) { + $file2_settings{$record} = $value; + } } - } } close $file2; # Subroutine to compare configs/metrics and obtain common and difference between them sub compare_configs_or_metrics { - my($records1, $records2, $file) = @_; - my %common_settings; - my %diff_settings; - my %settings1 = %$records1; - my %settings2 = %$records2; - - foreach my $record(sort keys %settings1) { - if ($settings2{$record}) { - $common_settings{$record} = $settings1{$record}; - } else { - $diff_settings{$record} = $settings1{$record}; + my ($records1, $records2, $file) = @_; + my %common_settings; + my %diff_settings; + my %settings1 = %$records1; + my %settings2 = %$records2; + + foreach my $record (sort keys %settings1) { + if ($settings2{$record}) { + $common_settings{$record} = $settings1{$record}; + } else { + $diff_settings{$record} = $settings1{$record}; + } } - } - print "####################################################################################\n"; - print "Configs/metrics found only in $file\n"; - print "####################################################################################\n"; - foreach my $key(sort keys %diff_settings) - { - print "$key\n"; - } - return (\%common_settings); + print "####################################################################################\n"; + print "Configs/metrics found only in $file\n"; + print "####################################################################################\n"; + foreach my $key (sort keys %diff_settings) { + print "$key\n"; + } + return (\%common_settings); } # Subroutine to obtain changes in default values among common configs/metrics sub compare_default_values { - my($records1, $records2) = @_; - my %settings1 = %$records1; - my %settings2 = %$records2; - - foreach my $record(sort keys %settings1) { - if (defined $settings1{$record} && $settings2{$record}) { - if ($settings1{$record} ne $settings2{$record}) { - # Values doesn't match - print "$record default value changed from $settings1{$record} -> $settings2{$record}\n"; - } + my ($records1, $records2) = @_; + my %settings1 = %$records1; + my %settings2 = %$records2; + + foreach my $record (sort keys %settings1) { + if (defined $settings1{$record} && $settings2{$record}) { + if ($settings1{$record} ne $settings2{$record}) { + # Values doesn't match + print "$record default value changed from $settings1{$record} -> $settings2{$record}\n"; + } + } } - } } # Obtain common configs/metrics between two files @@ -171,9 +174,8 @@ sub compare_default_values print "####################################################################################\n"; print "Common configs/metrics between $in_file1 and $in_file2\n"; print "####################################################################################\n"; -foreach my $key(sort keys %common2_settings) -{ - print "$key\n"; +foreach my $key (sort keys %common2_settings) { + print "$key\n"; } # Compare common configs/metrics and obtain changes in default values diff --git a/tools/compare_servers.pl b/tools/compare_servers.pl index 7f1bcb86bb3..4c485da6c24 100755 --- a/tools/compare_servers.pl +++ b/tools/compare_servers.pl @@ -28,218 +28,232 @@ my $verbose = 0; #---------------------------------------------------------------------------- -sub usage() { - print STDERR "USAGE: compare_hosts.pl --verbose level --host1 testing_host --host2 valid_host --file url_file\n\n"; - print STDERR "\t--host1 The host running the newest version\n"; - print STDERR "\t--host2 The host running the older version\n"; - print STDERR "\t--file A file that contains a list of URLs\n"; - print STDERR "\t--verbose verbose level 1-3, 1 is the least verbose\n\n"; - print STDERR "Example:\n"; - print STDERR "\tcompare_hosts.pl --host1 new_ats --host2 old_ats --file top_1000_urls\n"; - exit 1; +sub usage() +{ + print STDERR "USAGE: compare_hosts.pl --verbose level --host1 testing_host --host2 valid_host --file url_file\n\n"; + print STDERR "\t--host1 The host running the newest version\n"; + print STDERR "\t--host2 The host running the older version\n"; + print STDERR "\t--file A file that contains a list of URLs\n"; + print STDERR "\t--verbose verbose level 1-3, 1 is the least verbose\n\n"; + print STDERR "Example:\n"; + print STDERR "\tcompare_hosts.pl --host1 new_ats --host2 old_ats --file top_1000_urls\n"; + exit 1; } #---------------------------------------------------------------------------- -sub compareHeaderNames($$) { - my($response1, $response2) = @_; +sub compareHeaderNames($$) +{ + my ($response1, $response2) = @_; - my @names1 = $response1->header_field_names; - my @names2 = $response2->header_field_names; + my @names1 = $response1->header_field_names; + my @names2 = $response2->header_field_names; - my %hash2; - $hash2{$_} = 1 for (@names2); - my %hash1; - $hash1{$_} = 1 for (@names1); + my %hash2; + $hash2{$_} = 1 for (@names2); + my %hash1; + $hash1{$_} = 1 for (@names1); - my $return_val = 0; # header names match + my $return_val = 0; # header names match - foreach my $name (@names1) { - if (!defined $hash2{$name}) { - print "\t\t- $name header not found on host2\n" if $verbose >= 2; - $return_val = 1; + foreach my $name (@names1) { + if (!defined $hash2{$name}) { + print "\t\t- $name header not found on host2\n" if $verbose >= 2; + $return_val = 1; + } } - } - foreach my $name (@names2) { - if (!defined $hash1{$name}) { - print "\t\t- $name header not found on host1\n" if $verbose >= 2; - $return_val = 1; + foreach my $name (@names2) { + if (!defined $hash1{$name}) { + print "\t\t- $name header not found on host1\n" if $verbose >= 2; + $return_val = 1; + } } - } - return $return_val; + return $return_val; } #---------------------------------------------------------------------------- -sub compareHeaderValues($$) { - my($response1, $response2) = @_; +sub compareHeaderValues($$) +{ + my ($response1, $response2) = @_; - my @test_headers = qw(ETag Cache-Control Connection Accept-Ranges Server Content-Type Access-Control-Allow-Methods Access-Control-Allow-Origin Strict-Transport-Security); - my $return_val = 0; # header valuse match + my @test_headers = + qw(ETag Cache-Control Connection Accept-Ranges Server Content-Type Access-Control-Allow-Methods Access-Control-Allow-Origin Strict-Transport-Security); + my $return_val = 0; # header valuse match - if ($verbose >= 3) { - foreach my $field ($response1->header_field_names) { - print "\t\t\t~ " . $field . ": " . $response1->header($field) . "\n"; - } + if ($verbose >= 3) { + foreach my $field ($response1->header_field_names) { + print "\t\t\t~ " . $field . ": " . $response1->header($field) . "\n"; + } - print "\t\tHost2: \n"; + print "\t\tHost2: \n"; - foreach my $field ($response2->header_field_names) { - print "\t\t\t~ " . $field . ": " . $response2->header($field) . "\n"; + foreach my $field ($response2->header_field_names) { + print "\t\t\t~ " . $field . ": " . $response2->header($field) . "\n"; + } } - } - - # Test specific headers that are defined above - foreach my $field (@test_headers) { - my $value1 = $response1->header($field); - my $value2 = $response2->header($field); - - if (defined $value1 && defined $value2) { - if ($value1 ne $value2) { - print "\t\t- $field: $value1 ne $value2\n" if $verbose; - print "\t\t\t - Via host1: " . $response1->header('Via') . " host2: " . $response2->header('Via') . "\n" if $verbose; - print "\t\t\t - Last-Modified host1: " . $response1->header('Last-Modified') . " host2: " . $response2->header('Last-Modified') . "\n" if $verbose; - if (defined $response2->header('Content-Encoding')) { - print "\t\t\t - Content-Encoding host1: " . $response1->header('Content-Encoding') . " host2: " . $response2->header('Content-Encoding') . "\n"; - } else { - print "\t\t\t - Content-Encoding host1: " . $response1->header('Content-Encoding') . " host2: ''\n"; + + # Test specific headers that are defined above + foreach my $field (@test_headers) { + my $value1 = $response1->header($field); + my $value2 = $response2->header($field); + + if (defined $value1 && defined $value2) { + if ($value1 ne $value2) { + print "\t\t- $field: $value1 ne $value2\n" if $verbose; + print "\t\t\t - Via host1: " . $response1->header('Via') . " host2: " . $response2->header('Via') . "\n" + if $verbose; + print "\t\t\t - Last-Modified host1: " + . $response1->header('Last-Modified') + . " host2: " + . $response2->header('Last-Modified') . "\n" + if $verbose; + if (defined $response2->header('Content-Encoding')) { + print "\t\t\t - Content-Encoding host1: " + . $response1->header('Content-Encoding') + . " host2: " + . $response2->header('Content-Encoding') . "\n"; + } else { + print "\t\t\t - Content-Encoding host1: " . $response1->header('Content-Encoding') . " host2: ''\n"; + } + $return_val = 1; + } else { + print "\t\t- $field: $value1 eq $value2\n" if $verbose >= 2; + } } - $return_val = 1; - } else { - print "\t\t- $field: $value1 eq $value2\n" if $verbose >= 2; - } } - } - return $return_val; + return $return_val; } #---------------------------------------------------------------------------- { - my %stats; - - $ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = '0'; - my($host1, $host2, $file); - GetOptions ("host1=s" => \$host1, - "host2=s" => \$host2, - "file=s" => \$file, - "verbose=f" => \$verbose) || die $!; - - usage() if (! defined $host1 || ! defined $host2 || ! defined $file); - - my $count = 0; - my $status_error = 0; - my $sha_error = 0; - my $header_names_mismatch = 0; - my $header_values_mismatch = 0; - - my $host1_addr = inet_ntoa(inet_aton($host1)); - my $host2_addr = inet_ntoa(inet_aton($host2)); - - print "Testing with host1: $host1 ($host1_addr) - host2: $host2 ($host2_addr)\n"; - print '-' x 78, "\n"; - - open(FILE, $file) || die $!; - - # Create a user agent object - my $ua1 = LWP::UserAgent->new(keep_alive => 100); - $ua1->agent("MyApp/0.1 "); - - # Create a user agent object - my $ua2 = LWP::UserAgent->new(keep_alive => 100); - $ua2->agent("MyApp/0.1 "); - - while (my $url = ) { - next if ($url =~ m|hc.l.yimg.com|); - chomp $url; - my $exit = 0; - - if ($url =~ m|(https?)://([^/]+)(.+)|) { - - my $scheme = $1; - my $host = $2; - my $path = $3; - - $count++; - print "Test $count - URL: $url\n"; - - my $port = 80; - $port = 443 if $scheme eq 'https'; - - my $request1 = HTTP::Request->new(GET => "${scheme}://${host1_addr}${path}"); - $request1->header('Host' => $host); - my $response1 = $ua1->request($request1); - - my $request2 = HTTP::Request->new(GET => "${scheme}://${host2_addr}${path}"); - $request2->header('Host' => $host); - $request2->header('Accept-Encoding' => 'deflate'); - my $response2 = $ua2->request($request2); - - print "\tStatus code for host1: " . $response1->code . " - host2: " . $response2->code . "\n" if $verbose; - - my $sha1 = Digest::SHA1->new; - $sha1->add($response1->content); - my $digest1 = $sha1->hexdigest; - open(FILE1, "> /tmp/tmp1"); - open(FILE2, "> /tmp/tmp2"); - print FILE1 $response1->content; - print FILE2 $response2->content; - close FILE1; - close FILE2; - #print $response1->content, "\n"; # for internal debugging - #print $response2->content, "\n"; # for internal debugging - - my $sha2 = Digest::SHA1->new; - $sha2->add($response2->content); - my $digest2 = $sha2->hexdigest; - - print "\tSHA hash for host1: $digest1 - host2: $digest2\n" if $verbose; - - # Build up stats - if ($response1->status_line eq $response2->status_line) { - - # Do the hashes - if ($digest1 eq $digest2) { - $stats{stat_line_match}->{$response1->code}->{sha_match}++; - print "\tResponse code: " . $response1->code . " - Status lines and SHA1 of response bodies match\n"; - } else { - $stats{stat_line_match}->{$response1->code}->{sha_nomatch}++; - print "\tResponse code: " . $response1->code . " - Status lines match SHA1 doesn't match\n"; - $sha_error++; - #$exit = 1 if $response1->code == 200; # for internal debugging - } - - # Compare the header field names - if (compareHeaderNames($response1, $response2) == 0) { - $stats{stat_line_match}->{$response1->code}->{field_names_match}++; - } else { - $stats{stat_line_match}->{$response1->code}->{field_names_nomatch}++; - $header_names_mismatch++; + my %stats; + + $ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = '0'; + my ($host1, $host2, $file); + GetOptions( + "host1=s" => \$host1, + "host2=s" => \$host2, + "file=s" => \$file, + "verbose=f" => \$verbose + ) || die $!; + + usage() if (!defined $host1 || !defined $host2 || !defined $file); + + my $count = 0; + my $status_error = 0; + my $sha_error = 0; + my $header_names_mismatch = 0; + my $header_values_mismatch = 0; + + my $host1_addr = inet_ntoa(inet_aton($host1)); + my $host2_addr = inet_ntoa(inet_aton($host2)); + + print "Testing with host1: $host1 ($host1_addr) - host2: $host2 ($host2_addr)\n"; + print '-' x 78, "\n"; + + open(FILE, $file) || die $!; + + # Create a user agent object + my $ua1 = LWP::UserAgent->new(keep_alive => 100); + $ua1->agent("MyApp/0.1 "); + + # Create a user agent object + my $ua2 = LWP::UserAgent->new(keep_alive => 100); + $ua2->agent("MyApp/0.1 "); + + while (my $url = ) { + next if ($url =~ m|hc.l.yimg.com|); + chomp $url; + my $exit = 0; + + if ($url =~ m|(https?)://([^/]+)(.+)|) { + + my $scheme = $1; + my $host = $2; + my $path = $3; + + $count++; + print "Test $count - URL: $url\n"; + + my $port = 80; + $port = 443 if $scheme eq 'https'; + + my $request1 = HTTP::Request->new(GET => "${scheme}://${host1_addr}${path}"); + $request1->header('Host' => $host); + my $response1 = $ua1->request($request1); + + my $request2 = HTTP::Request->new(GET => "${scheme}://${host2_addr}${path}"); + $request2->header('Host' => $host); + $request2->header('Accept-Encoding' => 'deflate'); + my $response2 = $ua2->request($request2); + + print "\tStatus code for host1: " . $response1->code . " - host2: " . $response2->code . "\n" if $verbose; + + my $sha1 = Digest::SHA1->new; + $sha1->add($response1->content); + my $digest1 = $sha1->hexdigest; + open(FILE1, "> /tmp/tmp1"); + open(FILE2, "> /tmp/tmp2"); + print FILE1 $response1->content; + print FILE2 $response2->content; + close FILE1; + close FILE2; + #print $response1->content, "\n"; # for internal debugging + #print $response2->content, "\n"; # for internal debugging + + my $sha2 = Digest::SHA1->new; + $sha2->add($response2->content); + my $digest2 = $sha2->hexdigest; + + print "\tSHA hash for host1: $digest1 - host2: $digest2\n" if $verbose; + + # Build up stats + if ($response1->status_line eq $response2->status_line) { + + # Do the hashes + if ($digest1 eq $digest2) { + $stats{stat_line_match}->{$response1->code}->{sha_match}++; + print "\tResponse code: " . $response1->code . " - Status lines and SHA1 of response bodies match\n"; + } else { + $stats{stat_line_match}->{$response1->code}->{sha_nomatch}++; + print "\tResponse code: " . $response1->code . " - Status lines match SHA1 doesn't match\n"; + $sha_error++; + #$exit = 1 if $response1->code == 200; # for internal debugging + } + + # Compare the header field names + if (compareHeaderNames($response1, $response2) == 0) { + $stats{stat_line_match}->{$response1->code}->{field_names_match}++; + } else { + $stats{stat_line_match}->{$response1->code}->{field_names_nomatch}++; + $header_names_mismatch++; + } + + # Compare the values of the header fields + if (compareHeaderValues($response1, $response2) == 0) { + $stats{stat_line_match}->{$response1->code}->{field_values_match}++; + } else { + $stats{stat_line_match}->{$response1->code}->{field_values_nomatch}++; + $header_values_mismatch++; + } + } else { + $status_error++; + $stats{stat_line_nomatch}++; + print "\tERROR: status lines don't match\n"; + } + + last if $exit; } - - # Compare the values of the header fields - if (compareHeaderValues($response1, $response2) == 0) { - $stats{stat_line_match}->{$response1->code}->{field_values_match}++; - } else { - $stats{stat_line_match}->{$response1->code}->{field_values_nomatch}++; - $header_values_mismatch++; - } - } else { - $status_error++; - $stats{stat_line_nomatch}++; - print "\tERROR: status lines don't match\n"; - } - - last if $exit; } - } - - print '-' x 78, "\n"; - print "SUMMARY:\n"; - print "URLs tested: $count\n"; - print "Status line mismatches: $status_error\n"; - print "SHA1 mismatches: $sha_error\n"; - print "Responses with header names mismatches: $header_names_mismatch\n"; - print "Responses with header values mismatches: $header_values_mismatch\n"; - print Dumper \%stats if $verbose; + + print '-' x 78, "\n"; + print "SUMMARY:\n"; + print "URLs tested: $count\n"; + print "Status line mismatches: $status_error\n"; + print "SHA1 mismatches: $sha_error\n"; + print "Responses with header names mismatches: $header_names_mismatch\n"; + print "Responses with header values mismatches: $header_values_mismatch\n"; + print Dumper \%stats if $verbose; } diff --git a/tools/freelist_diff.pl b/tools/freelist_diff.pl index 8f15a93d970..4e9e92d68e6 100755 --- a/tools/freelist_diff.pl +++ b/tools/freelist_diff.pl @@ -16,21 +16,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -sub usage { +sub usage +{ print "Usage: freelist_diff.pl dump1.txt dump2.txt\n"; } -sub int_meg { +sub int_meg +{ my $bytes = shift; - return $bytes / (1024*1024); + return $bytes / (1024 * 1024); } -sub load_file { +sub load_file +{ my $file = shift; my %data; open(DATA, $file) || return undef; - while() { + while () { my @items = split; chomp @items; @@ -49,11 +52,13 @@ sub load_file { while (my ($key, $value) = each(%{$data1})) { # before alloc [0], after alloc [1], before in-use [2], after in-use [3] - $diff{$key} = [ $value->[0], $data2->{$key}->[0], $value->[1], $data2->{$key}->[1], - # diff alloc [4], diff in-use [5] - $data2->{$key}->[0] - $value->[0], $data2->{$key}->[1] - $value->[1], - # type size [6] - $value->[2] ]; + $diff{$key} = [ + $value->[0], $data2->{$key}->[0], $value->[1], $data2->{$key}->[1], + # diff alloc [4], diff in-use [5] + $data2->{$key}->[0] - $value->[0], $data2->{$key}->[1] - $value->[1], + # type size [6] + $value->[2] + ]; } print "Sorted by in-use growth\n"; diff --git a/tools/http_load/merge_stats.pl b/tools/http_load/merge_stats.pl index 49e93a4f709..db406004907 100644 --- a/tools/http_load/merge_stats.pl +++ b/tools/http_load/merge_stats.pl @@ -20,54 +20,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -my $runs = 0; -my $fetches = 0; -my $conns = 0; -my $parallel = 0; -my $bytes = 0; -my $seconds = 0; -my $mean_bytes = 0; -my $fetches_sec = 0.0; -my $bytes_sec = 0.0; -my %msecs_connect = ( "mean" => 0.0, - "max" => 0.0, - "min" => 0.0 ); -my %msecs_response = ( "mean" => 0.0, - "max" => 0.0, - "min" => 0.0 ); - +my $runs = 0; +my $fetches = 0; +my $conns = 0; +my $parallel = 0; +my $bytes = 0; +my $seconds = 0; +my $mean_bytes = 0; +my $fetches_sec = 0.0; +my $bytes_sec = 0.0; +my %msecs_connect = ( + "mean" => 0.0, + "max" => 0.0, + "min" => 0.0 +); +my %msecs_response = ( + "mean" => 0.0, + "max" => 0.0, + "min" => 0.0 +); while (<>) { - my @c = split(); - if (/fetches on/) { - $fetches += $c[0]; - $conns += $c[3]; - $parallel += $c[5]; - $bytes += $c[8]; - $seconds += $c[11]; - $runs++; - } elsif (/mean bytes/) { - $mean_bytes += $c[0]; - } elsif (/fetches\/sec/) { - $fetches_sec += $c[0]; - $bytes_sec += $c[2]; - } elsif (/msecs\/connect/) { - $msecs_connect{"mean"} += $c[1]; - $msecs_connect{"max"} += $c[3]; - $msecs_connect{"min"} += $c[5]; - } elsif (/msecs\/first/) { - $msecs_response{"mean"} += $c[1]; - $msecs_response{"max"} += $c[3]; - $msecs_response{"min"} += $c[5]; - } + my @c = split(); + if (/fetches on/) { + $fetches += $c[0]; + $conns += $c[3]; + $parallel += $c[5]; + $bytes += $c[8]; + $seconds += $c[11]; + $runs++; + } elsif (/mean bytes/) { + $mean_bytes += $c[0]; + } elsif (/fetches\/sec/) { + $fetches_sec += $c[0]; + $bytes_sec += $c[2]; + } elsif (/msecs\/connect/) { + $msecs_connect{"mean"} += $c[1]; + $msecs_connect{"max"} += $c[3]; + $msecs_connect{"min"} += $c[5]; + } elsif (/msecs\/first/) { + $msecs_response{"mean"} += $c[1]; + $msecs_response{"max"} += $c[3]; + $msecs_response{"min"} += $c[5]; + } } print "Total runs: ", $runs, "\n"; printf "%d fetches on %d conns, %d max parallell, %.5e bytes in %d seconds\n", $fetches, $conns, $parallel, $bytes, $seconds / $runs; -print $mean_bytes/$runs, " mean bytes/fetch\n"; -printf "%.2f fetches/sec, %.5e bytes/sec\n", $fetches_sec, $bytes_sec; -print "msecs/connect: ", $msecs_connect{"mean"}/$runs, " mean, ", - $msecs_connect{"max"}/$runs, " max, ", $msecs_connect{"min"}/$runs, " min\n"; -print "msecs/first-response: ", $msecs_response{"mean"}/$runs, " mean, ", - $msecs_response{"max"}/$runs, " max, ", $msecs_response{"min"}/$runs, " min\n"; +print $mean_bytes/ $runs, " mean bytes/fetch\n"; +printf "%.2f fetches/sec, %.5e bytes/sec\n", $fetches_sec, $bytes_sec; +print "msecs/connect: ", $msecs_connect{"mean"} / $runs, " mean, ", + $msecs_connect{"max"} / $runs, " max, ", $msecs_connect{"min"} / $runs, " min\n"; +print "msecs/first-response: ", $msecs_response{"mean"} / $runs, " mean, ", + $msecs_response{"max"} / $runs, " max, ", $msecs_response{"min"} / $runs, " min\n"; diff --git a/tools/slow_log_report.pl b/tools/slow_log_report.pl index 9207b5906af..81fb52d47da 100755 --- a/tools/slow_log_report.pl +++ b/tools/slow_log_report.pl @@ -21,62 +21,72 @@ use warnings; #use Data::Dumper; -sub addStat($$$) { - my($stats, $key, $value) = @_; - #print "$key $value\n"; - $stats->{$key}->{total} = 0 if (! defined $stats->{$key}->{total}); - $stats->{$key}->{count} = 0 if (! defined $stats->{$key}->{count}); - return if (! ($value =~ m|^-?\d+\.?\d*$|)); - #print "$key\n"; - $stats->{$key}->{total} += $value if $value >= 0; - $stats->{$key}->{count}++ if $value >= 0; - push(@{$stats->{$key}->{values}}, $value) if $value >= 0; +sub addStat($$$) +{ + my ($stats, $key, $value) = @_; + #print "$key $value\n"; + $stats->{$key}->{total} = 0 if (!defined $stats->{$key}->{total}); + $stats->{$key}->{count} = 0 if (!defined $stats->{$key}->{count}); + return if (!($value =~ m|^-?\d+\.?\d*$|)); + #print "$key\n"; + $stats->{$key}->{total} += $value if $value >= 0; + $stats->{$key}->{count}++ if $value >= 0; + push(@{$stats->{$key}->{values}}, $value) if $value >= 0; } -sub displayStat($) { - my($stats) = @_; +sub displayStat($) +{ + my ($stats) = @_; + + printf("%25s %10s %10s %10s %10s %10s %10s %10s %10s\n", + 'key', 'total', 'count', 'mean', 'median', '95th', '99th', 'min', 'max'); + foreach my $key ( + 'ua_begin', 'ua_first_read', 'ua_read_header_done', 'cache_open_read_begin', + 'cache_open_read_end', 'dns_lookup_begin', 'dns_lookup_end', 'server_connect', + 'server_connect_end', 'server_first_read', 'server_read_header_done', 'server_close', + 'ua_close', 'sm_finish' + ) + { - printf("%25s %10s %10s %10s %10s %10s %10s %10s %10s\n", 'key', 'total', 'count', 'mean', 'median', '95th', '99th', 'min', 'max'); - foreach my $key ('ua_begin', 'ua_first_read', 'ua_read_header_done', 'cache_open_read_begin', 'cache_open_read_end', 'dns_lookup_begin', 'dns_lookup_end', 'server_connect', 'server_connect_end', 'server_first_read', 'server_read_header_done', 'server_close', 'ua_close', 'sm_finish') { + my $count = $stats->{$key}->{count}; + my $total = $stats->{$key}->{total}; + if (!defined $stats->{$key}->{values}) { + next; + #print "$key\n"; + #die $key; + } + my @sorted = sort {$a <=> $b} @{$stats->{$key}->{values}}; + my $median = $sorted[int($count / 2)]; + my $p95th = $sorted[int($count * .95)]; + my $p99th = $sorted[int($count * .99)]; + my $min = $sorted[0]; + my $max = $sorted[$count - 1]; + my $mean = 0; + $mean = $total / $count if $count > 0; - my $count = $stats->{$key}->{count}; - my $total = $stats->{$key}->{total}; - if (!defined $stats->{$key}->{values}) { - next; - #print "$key\n"; - #die $key; + printf("%25s %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f\n", + $key, $total, $count, $mean, $median, $p95th, $p99th, $min, $max); } - my @sorted = sort {$a <=> $b} @{$stats->{$key}->{values}}; - my $median = $sorted[int($count/2)]; - my $p95th = $sorted[int($count * .95)]; - my $p99th = $sorted[int($count * .99)]; - my $min = $sorted[0]; - my $max = $sorted[$count - 1]; - my $mean = 0; - $mean = $total / $count if $count > 0; - - printf("%25s %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f\n", $key, $total, $count, $mean, $median, $p95th, $p99th, $min, $max); - } - print "NOTE: Times are in seconds\n"; + print "NOTE: Times are in seconds\n"; } { - my %stats; + my %stats; - while (<>) { - chomp; - s/unique id/unique_id/; - s/server state/server_state/; - s/client state/client_state/; - if (m|Slow Request: .+ (ua_begin: .+)|) { - my %data = split(/: | /, $1); - foreach my $key (keys %data) { - next if (!defined $data{$key}); - #print "$key $data{$key}\n"; - addStat(\%stats, $key, $data{$key}); - } + while (<>) { + chomp; + s/unique id/unique_id/; + s/server state/server_state/; + s/client state/client_state/; + if (m|Slow Request: .+ (ua_begin: .+)|) { + my %data = split(/: | /, $1); + foreach my $key (keys %data) { + next if (!defined $data{$key}); + #print "$key $data{$key}\n"; + addStat(\%stats, $key, $data{$key}); + } + } } - } - displayStat(\%stats); + displayStat(\%stats); } diff --git a/tools/traffic_via.pl b/tools/traffic_via.pl index 6eee2a6cb13..e5a0f297cac 100755 --- a/tools/traffic_via.pl +++ b/tools/traffic_via.pl @@ -25,9 +25,9 @@ # 1. Pass Via Header with -s option \n"; # traffic_via [-s viaheader]"; # or -# 2. Pipe curl output +# 2. Pipe curl output # curl -v -H "X-Debug: Via" http://ats_server:port 2>&1| ./traffic_via.pl -# +# use strict; use warnings; @@ -40,8 +40,7 @@ #Proxy request header flags and titles my @proxy_header_array = ( { - "Request headers received from client:", - { + "Request headers received from client:", { 'I' => "If Modified Since (IMS)", 'C' => "cookie", 'E' => "error in request", @@ -49,115 +48,93 @@ 'N' => "no-cache", ' ' => "unknown?", }, - }, - { - "Result of Traffic Server cache lookup for URL:", - { - 'A' => "in cache, not acceptable (a cache \"MISS\")", - 'H' => "in cache, fresh (a cache \"HIT\")", - 'S' => "in cache, stale (a cache \"MISS\")", - 'R' => "in cache, fresh Ram hit (a cache \"HIT\")", - 'M' => "miss (a cache \"MISS\")", - ' ' => "unknown?", + }, { + "Result of Traffic Server cache lookup for URL:", { + 'A' => "in cache, not acceptable (a cache \"MISS\")", + 'H' => "in cache, fresh (a cache \"HIT\")", + 'S' => "in cache, stale (a cache \"MISS\")", + 'R' => "in cache, fresh Ram hit (a cache \"HIT\")", + 'M' => "miss (a cache \"MISS\")", + ' ' => "unknown?", }, - }, - { - "Response information received from origin server:", - { - 'E' => "error in response", - ' ' => "no server connection needed", - 'S' => "served", - 'N'=> "not-modified", + }, { + "Response information received from origin server:", { + 'E' => "error in response", + ' ' => "no server connection needed", + 'S' => "served", + 'N' => "not-modified", } - }, - { - "Result of document write-to-cache:", - { - 'U' => "updated old cache copy", - 'D' => "cached copy deleted", - 'W' => "written into cache (new copy)", - ' ' => "no cache write performed", + }, { + "Result of document write-to-cache:", { + 'U' => "updated old cache copy", + 'D' => "cached copy deleted", + 'W' => "written into cache (new copy)", + ' ' => "no cache write performed", }, - }, - { - "Proxy operation result:", - { - 'R' => "origin server revalidated", - ' ' => "unknown?", - 'S' => "served", - 'N' => "not-modified", + }, { + "Proxy operation result:", { + 'R' => "origin server revalidated", + ' ' => "unknown?", + 'S' => "served", + 'N' => "not-modified", }, - }, - { - "Error codes (if any):", - { - 'A' => "authorization failure", - 'H' => "header syntax unacceptable", - 'C' => "connection to server failed", - 'T' => "connection timed out", - 'S' => "server related error", - 'D' => "dns failure", - 'N' => "no error", - 'F' => "request forbidden", + }, { + "Error codes (if any):", { + 'A' => "authorization failure", + 'H' => "header syntax unacceptable", + 'C' => "connection to server failed", + 'T' => "connection timed out", + 'S' => "server related error", + 'D' => "dns failure", + 'N' => "no error", + 'F' => "request forbidden", }, - }, - { - "Tunnel info:", - { - ' ' => "no tunneling", - 'U' => "tunneling because of url (url suggests dynamic content)", - 'M' => "tunneling due to a method (e.g. CONNECT)", - 'O' => "tunneling because cache is turned off", - 'F' => "tunneling due to a header field (such as presence of If-Range header)", + }, { + "Tunnel info:", { + ' ' => "no tunneling", + 'U' => "tunneling because of url (url suggests dynamic content)", + 'M' => "tunneling due to a method (e.g. CONNECT)", + 'O' => "tunneling because cache is turned off", + 'F' => "tunneling due to a header field (such as presence of If-Range header)", }, - }, - { - "Cache type:", - { - 'I' => "icp", - ' ' => "cache miss or no cache lookup", - 'C' => "cache", + }, { + "Cache type:", { + 'I' => "icp", + ' ' => "cache miss or no cache lookup", + 'C' => "cache", }, - }, - { - "Cache lookup result:", - { - ' ' => "no cache lookup", - 'S' => "cache hit, but expired", - 'U' => "cache hit, but client forces revalidate (e.g. Pragma: no-cache)", - 'D' => "cache hit, but method forces revalidated (e.g. ftp, not anonymous)", - 'I' => "conditional miss (client sent conditional, fresh in cache, returned 412)", - 'H' => "cache hit", - 'M' => "cache miss (url not in cache)", - 'C' => "cache hit, but config forces revalidate", - 'N' => "conditional hit (client sent conditional, doc fresh in cache, returned 304)", + }, { + "Cache lookup result:", { + ' ' => "no cache lookup", + 'S' => "cache hit, but expired", + 'U' => "cache hit, but client forces revalidate (e.g. Pragma: no-cache)", + 'D' => "cache hit, but method forces revalidated (e.g. ftp, not anonymous)", + 'I' => "conditional miss (client sent conditional, fresh in cache, returned 412)", + 'H' => "cache hit", + 'M' => "cache miss (url not in cache)", + 'C' => "cache hit, but config forces revalidate", + 'N' => "conditional hit (client sent conditional, doc fresh in cache, returned 304)", }, - }, - { - "ICP status:", - { - ' ' => "no icp", - 'S' => "connection opened successfully", - 'F' => "connection open failed", + }, { + "ICP status:", { + ' ' => "no icp", + 'S' => "connection opened successfully", + 'F' => "connection open failed", }, - }, - { - "Parent proxy connection status:", - { - ' ' => "no parent proxy", - 'S' => "connection opened successfully", - 'F' => "connection open failed", + }, { + "Parent proxy connection status:", { + ' ' => "no parent proxy", + 'S' => "connection opened successfully", + 'F' => "connection open failed", }, - - }, - { - "Origin server connection status:", - { - ' ' => "no server connection", - 'S' => "connection opened successfully", - 'F' => "connection open failed", + + }, { + "Origin server connection status:", { + ' ' => "no server connection", + 'S' => "connection opened successfully", + 'F' => "connection open failed", }, - }, + }, ); ##Print script usage @@ -181,17 +158,22 @@ sub usage #Pattern matching for Via if ($element =~ /Via:(.*)\[(.*)\]/) { #Search and grep via header - $via_string = $2; + $via_string = $2; chomp($via_string); print "Via Header is [$via_string]"; decode_via_header($via_string); } } } else { - usage() if (!GetOptions('s=s' => \$via_header, - 'help|?' => \$help) or - defined $help); - + usage() + if ( + !GetOptions( + 's=s' => \$via_header, + 'help|?' => \$help + ) + or defined $help + ); + if (defined $via_header) { #if passed through commandline dashed argument print "Via Header is [$via_header]"; @@ -201,16 +183,17 @@ sub usage } #Subroutine to decode via header -sub decode_via_header { - my($header) = @_; +sub decode_via_header +{ + my ($header) = @_; my $hdrLength; my $newHeader; #Check via header syntax - if ($header =~ /([a-zA-Z: ]+)/) { + if ($header =~ /([a-zA-Z: ]+)/) { #Get via header length $hdrLength = length($header); - + # Valid Via header length is 24 or 6. # When Via header length is 24, it will have both proxy request header result and operational results. if ($hdrLength == 24) { @@ -220,7 +203,7 @@ sub decode_via_header { $newHeader = $header; } elsif ($hdrLength == 5) { # When Via header length is 5, it might be missing last field. Fill it and decode header. - my $newHeader = "$header"." "; + my $newHeader = "$header" . " "; } else { # Invalid header size, come out. print "\nInvalid VIA header. VIA header length should be 6 or 24 characters\n"; @@ -228,49 +211,50 @@ sub decode_via_header { } convert_header_to_array($newHeader); } - - + } -sub convert_header_to_array { +sub convert_header_to_array +{ my ($viaHeader) = @_; my @ResultArray; #Convert string header into character array while ($viaHeader =~ /(.)/g) { - #Only capital letters indicate flags - if ($1 !~ m/[a-z]+/) { - push(@ResultArray, $1); - } + #Only capital letters indicate flags + if ($1 !~ m/[a-z]+/) { + push(@ResultArray, $1); + } } print "\nVia Header details: \n"; - for (my $arrayIndex=0; $arrayIndex < scalar(@ResultArray); $arrayIndex++ ) { + for (my $arrayIndex = 0; $arrayIndex < scalar(@ResultArray); $arrayIndex++) { get_via_header_flags(\@proxy_header_array, $arrayIndex, $ResultArray[$arrayIndex]); } } #Get values from header arrays -sub get_via_header_flags { +sub get_via_header_flags +{ my ($arrayName, $inputIndex, $flag) = @_; my %flagValues; my @flagKeys; my %flags; my @keys; - + my @array = @$arrayName; - + %flagValues = %{$array[$inputIndex]}; - @flagKeys = keys (%flagValues); - - foreach my $keyEntry ( @flagKeys ) { - printf ("%-55s", $keyEntry); + @flagKeys = keys(%flagValues); + + foreach my $keyEntry (@flagKeys) { + printf("%-55s", $keyEntry); %flags = %{$flagValues{$keyEntry}}; - @keys = keys (%flags); - foreach my $key ( @keys ) { - if ($key =~ /$flag/) { - #print $flags{$key}; - printf("%s",$flags{$key}); - print "\n"; + @keys = keys(%flags); + foreach my $key (@keys) { + if ($key =~ /$flag/) { + #print $flags{$key}; + printf("%s", $flags{$key}); + print "\n"; } } } From 11110ec732dd4f130cbb73d3b8005056dd9d4ef0 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 4 Mar 2019 15:49:21 -0600 Subject: [PATCH 350/526] Fix #5108: Remove OS dependency in test_BufferWriterFormat.cc. --- src/tscore/unit_tests/test_BufferWriterFormat.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tscore/unit_tests/test_BufferWriterFormat.cc b/src/tscore/unit_tests/test_BufferWriterFormat.cc index 6e0062ee214..d6f28587f0f 100644 --- a/src/tscore/unit_tests/test_BufferWriterFormat.cc +++ b/src/tscore/unit_tests/test_BufferWriterFormat.cc @@ -522,7 +522,7 @@ TEST_CASE("bwstring std formats", "[libts][bwprint]") w.print("{}", ts::bwf::Errno(13)); REQUIRE(w.view() == "EACCES: Permission denied [13]"sv); w.reset().print("{}", ts::bwf::Errno(134)); - REQUIRE(w.view().substr(0, 22) == "Unknown: Unknown error"sv); + REQUIRE(w.view().substr(0, 9) == "Unknown: "sv); time_t t = 1528484137; // default is GMT From 7197a6dbbe176889e0ddd5770d6defe69e502ea3 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 8 Mar 2019 13:43:16 -0600 Subject: [PATCH 351/526] UrlRewrite: separate constructor and configuration loading for testability. --- proxy/ReverseProxy.cc | 6 +++--- proxy/http/remap/UrlRewrite.cc | 23 +++++--------------- proxy/http/remap/UrlRewrite.h | 38 +++++++++++++++++++++++----------- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/proxy/ReverseProxy.cc b/proxy/ReverseProxy.cc index 0164da968fd..2ab3d036065 100644 --- a/proxy/ReverseProxy.cc +++ b/proxy/ReverseProxy.cc @@ -64,7 +64,7 @@ init_reverse_proxy() rewrite_table = new UrlRewrite(); Note("remap.config loading ..."); - if (!rewrite_table->is_valid()) { + if (!rewrite_table->load()) { Fatal("remap.config failed to load"); } Note("remap.config finished loading"); @@ -121,7 +121,7 @@ struct UR_UpdateContinuation : public Continuation { bool urlRewriteVerify() { - return UrlRewrite().is_valid(); + return UrlRewrite().load(); } /** @@ -138,7 +138,7 @@ reloadUrlRewrite() Note("remap.config loading ..."); Debug("url_rewrite", "remap.config updated, reloading..."); newTable = new UrlRewrite(); - if (newTable->is_valid()) { + if (newTable->load()) { static const char *msg = "remap.config finished loading"; // Hold at least one lease, until we reload the configuration diff --git a/proxy/http/remap/UrlRewrite.cc b/proxy/http/remap/UrlRewrite.cc index 5c154d3a7f2..628c119fea0 100644 --- a/proxy/http/remap/UrlRewrite.cc +++ b/proxy/http/remap/UrlRewrite.cc @@ -1,6 +1,6 @@ /** @file - A brief file description + URL rewriting. @section license License @@ -49,20 +49,8 @@ SetHomePageRedirectFlag(url_mapping *new_mapping, URL &new_to_url) new_mapping->homePageRedirect = (from_path && !to_path) ? true : false; } -// -// CTOR / DTOR for the UrlRewrite class. -// -UrlRewrite::UrlRewrite() - : nohost_rules(0), - reverse_proxy(0), - ts_name(nullptr), - http_default_redirect_url(nullptr), - num_rules_forward(0), - num_rules_reverse(0), - num_rules_redirect_permanent(0), - num_rules_redirect_temporary(0), - num_rules_forward_with_recv_port(0), - _valid(false) +bool +UrlRewrite::load() { ats_scoped_str config_file_path; @@ -70,7 +58,7 @@ UrlRewrite::UrlRewrite() if (!config_file_path) { pmgmt->signalManager(MGMT_SIGNAL_CONFIG_ERROR, "Unable to find proxy.config.url_remap.filename"); Warning("%s Unable to locate remap.config. No remappings in effect", modulePrefix); - return; + return false; } this->ts_name = nullptr; @@ -99,6 +87,7 @@ UrlRewrite::UrlRewrite() } else { Warning("something failed during BuildTable() -- check your remap plugins!"); } + return _valid; } UrlRewrite::~UrlRewrite() @@ -662,8 +651,6 @@ UrlRewrite::InsertForwardMapping(mapping_type maptype, url_mapping *mapping, con int UrlRewrite::BuildTable(const char *path) { - BUILD_TABLE_INFO bti; - ink_assert(forward_mappings.empty()); ink_assert(reverse_mappings.empty()); ink_assert(permanent_redirects.empty()); diff --git a/proxy/http/remap/UrlRewrite.h b/proxy/http/remap/UrlRewrite.h index dafb1d31799..89661caecae 100644 --- a/proxy/http/remap/UrlRewrite.h +++ b/proxy/http/remap/UrlRewrite.h @@ -1,6 +1,6 @@ /** @file - A brief file description + URL rewriting. @section license License @@ -57,10 +57,24 @@ class UrlRewrite : public RefCountObj { public: using URLTable = std::unordered_map; - UrlRewrite(); + UrlRewrite() = default; ~UrlRewrite() override; + /** Load the configuration. + * + * This access data in librecords to obtain the information needed for loading the configuration. + * + * @return @c true if the instance state is valid, @c false if not. + */ + bool load(); + + /** Build the internal url write tables. + * + * @param path Path to configuration file. + * @return 0 on success, non-zero error code on failure. + */ int BuildTable(const char *path); + mapping_type Remap_redirect(HTTPHdr *request_header, URL *redirect_url); bool ReverseMap(HTTPHdr *response_header); void SetReverseFlag(int flag); @@ -182,20 +196,20 @@ class UrlRewrite : public RefCountObj mapping_container); } - int nohost_rules; - int reverse_proxy; + int nohost_rules = 0; + int reverse_proxy = 0; - char *ts_name; // Used to send redirects when no host info + char *ts_name = nullptr; // Used to send redirects when no host info - char *http_default_redirect_url; // Used if redirect in "referer" filtering was not defined properly - int num_rules_forward; - int num_rules_reverse; - int num_rules_redirect_permanent; - int num_rules_redirect_temporary; - int num_rules_forward_with_recv_port; + char *http_default_redirect_url = nullptr; // Used if redirect in "referer" filtering was not defined properly + int num_rules_forward = 0; + int num_rules_reverse = 0; + int num_rules_redirect_permanent = 0; + int num_rules_redirect_temporary = 0; + int num_rules_forward_with_recv_port = 0; private: - bool _valid; + bool _valid = false; bool _mappingLookup(MappingsStore &mappings, URL *request_url, int request_port, const char *request_host, int request_host_len, UrlMappingContainer &mapping_container); From 1b300dfed5292efbaee257f369aa8bc8757b65b4 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Fri, 8 Mar 2019 12:14:20 +0900 Subject: [PATCH 352/526] Run clang-format against the same files on both make clang-foramt and git pre-hook Include *.h.in on make clang-format Exclude lib/tsconfig and lib/yamlcpp on git/pre-hook --- include/ts/apidefs.h.in | 188 ++++++++++++++++++++-------------------- tools/clang-format.sh | 2 +- tools/git/pre-commit | 2 +- 3 files changed, 96 insertions(+), 96 deletions(-) diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 2b1e5d2074f..e546a562f77 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -283,7 +283,7 @@ typedef enum { // Putting the SSL hooks in the same enum space // So both sets of hooks can be set by the same Hook function TS_SSL_FIRST_HOOK, - TS_VCONN_START_HOOK = TS_SSL_FIRST_HOOK, + TS_VCONN_START_HOOK = TS_SSL_FIRST_HOOK, TS_VCONN_PRE_ACCEPT_HOOK = TS_VCONN_START_HOOK, // Deprecated but compatible for now. TS_VCONN_CLOSE_HOOK, TS_SSL_CLIENT_HELLO_HOOK, @@ -394,90 +394,90 @@ typedef enum { */ typedef enum { - TS_EVENT_NONE = 0, - TS_EVENT_IMMEDIATE = 1, - TS_EVENT_TIMEOUT = 2, - TS_EVENT_ERROR = 3, - TS_EVENT_CONTINUE = 4, - - TS_EVENT_VCONN_READ_READY = 100, - TS_EVENT_VCONN_WRITE_READY = 101, - TS_EVENT_VCONN_READ_COMPLETE = 102, - TS_EVENT_VCONN_WRITE_COMPLETE = 103, - TS_EVENT_VCONN_EOS = 104, - TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, - TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, - TS_EVENT_VCONN_START = 107, - TS_EVENT_VCONN_CLOSE = 108, - TS_EVENT_VCONN_OUTBOUND_START = 109, - TS_EVENT_VCONN_OUTBOUND_CLOSE = 110, - TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible - - TS_EVENT_NET_CONNECT = 200, - TS_EVENT_NET_CONNECT_FAILED = 201, - TS_EVENT_NET_ACCEPT = 202, - TS_EVENT_NET_ACCEPT_FAILED = 204, - - TS_EVENT_INTERNAL_206 = 206, - TS_EVENT_INTERNAL_207 = 207, - TS_EVENT_INTERNAL_208 = 208, - TS_EVENT_INTERNAL_209 = 209, - TS_EVENT_INTERNAL_210 = 210, - TS_EVENT_INTERNAL_211 = 211, - TS_EVENT_INTERNAL_212 = 212, - - TS_EVENT_HOST_LOOKUP = 500, - - TS_EVENT_CACHE_OPEN_READ = 1102, - TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, - TS_EVENT_CACHE_OPEN_WRITE = 1108, - TS_EVENT_CACHE_OPEN_WRITE_FAILED = 1109, - TS_EVENT_CACHE_REMOVE = 1112, - TS_EVENT_CACHE_REMOVE_FAILED = 1113, - TS_EVENT_CACHE_SCAN = 1120, - TS_EVENT_CACHE_SCAN_FAILED = 1121, - TS_EVENT_CACHE_SCAN_OBJECT = 1122, - TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED = 1123, - TS_EVENT_CACHE_SCAN_OPERATION_FAILED = 1124, - TS_EVENT_CACHE_SCAN_DONE = 1125, - TS_EVENT_CACHE_LOOKUP = 1126, - TS_EVENT_CACHE_READ = 1127, - TS_EVENT_CACHE_DELETE = 1128, - TS_EVENT_CACHE_WRITE = 1129, - TS_EVENT_CACHE_WRITE_HEADER = 1130, - TS_EVENT_CACHE_CLOSE = 1131, - TS_EVENT_CACHE_LOOKUP_READY = 1132, - TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, - TS_EVENT_CACHE_READ_READY = 1134, - TS_EVENT_CACHE_READ_COMPLETE = 1135, - - TS_EVENT_INTERNAL_1200 = 1200, - - TS_EVENT_SSL_SESSION_GET = 2000, - TS_EVENT_SSL_SESSION_NEW = 2001, - TS_EVENT_SSL_SESSION_REMOVE = 2002, - - TS_EVENT_AIO_DONE = 3900, - - TS_EVENT_HTTP_CONTINUE = 60000, - TS_EVENT_HTTP_ERROR = 60001, - TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, - TS_EVENT_HTTP_OS_DNS = 60003, - TS_EVENT_HTTP_SEND_REQUEST_HDR = 60004, - TS_EVENT_HTTP_READ_CACHE_HDR = 60005, - TS_EVENT_HTTP_READ_RESPONSE_HDR = 60006, - TS_EVENT_HTTP_SEND_RESPONSE_HDR = 60007, - TS_EVENT_HTTP_REQUEST_TRANSFORM = 60008, - TS_EVENT_HTTP_RESPONSE_TRANSFORM = 60009, - TS_EVENT_HTTP_SELECT_ALT = 60010, - TS_EVENT_HTTP_TXN_START = 60011, - TS_EVENT_HTTP_TXN_CLOSE = 60012, - TS_EVENT_HTTP_SSN_START = 60013, - TS_EVENT_HTTP_SSN_CLOSE = 60014, - TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, - TS_EVENT_HTTP_PRE_REMAP = 60016, - TS_EVENT_HTTP_POST_REMAP = 60017, - TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60018, + TS_EVENT_NONE = 0, + TS_EVENT_IMMEDIATE = 1, + TS_EVENT_TIMEOUT = 2, + TS_EVENT_ERROR = 3, + TS_EVENT_CONTINUE = 4, + + TS_EVENT_VCONN_READ_READY = 100, + TS_EVENT_VCONN_WRITE_READY = 101, + TS_EVENT_VCONN_READ_COMPLETE = 102, + TS_EVENT_VCONN_WRITE_COMPLETE = 103, + TS_EVENT_VCONN_EOS = 104, + TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, + TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, + TS_EVENT_VCONN_START = 107, + TS_EVENT_VCONN_CLOSE = 108, + TS_EVENT_VCONN_OUTBOUND_START = 109, + TS_EVENT_VCONN_OUTBOUND_CLOSE = 110, + TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible + + TS_EVENT_NET_CONNECT = 200, + TS_EVENT_NET_CONNECT_FAILED = 201, + TS_EVENT_NET_ACCEPT = 202, + TS_EVENT_NET_ACCEPT_FAILED = 204, + + TS_EVENT_INTERNAL_206 = 206, + TS_EVENT_INTERNAL_207 = 207, + TS_EVENT_INTERNAL_208 = 208, + TS_EVENT_INTERNAL_209 = 209, + TS_EVENT_INTERNAL_210 = 210, + TS_EVENT_INTERNAL_211 = 211, + TS_EVENT_INTERNAL_212 = 212, + + TS_EVENT_HOST_LOOKUP = 500, + + TS_EVENT_CACHE_OPEN_READ = 1102, + TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, + TS_EVENT_CACHE_OPEN_WRITE = 1108, + TS_EVENT_CACHE_OPEN_WRITE_FAILED = 1109, + TS_EVENT_CACHE_REMOVE = 1112, + TS_EVENT_CACHE_REMOVE_FAILED = 1113, + TS_EVENT_CACHE_SCAN = 1120, + TS_EVENT_CACHE_SCAN_FAILED = 1121, + TS_EVENT_CACHE_SCAN_OBJECT = 1122, + TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED = 1123, + TS_EVENT_CACHE_SCAN_OPERATION_FAILED = 1124, + TS_EVENT_CACHE_SCAN_DONE = 1125, + TS_EVENT_CACHE_LOOKUP = 1126, + TS_EVENT_CACHE_READ = 1127, + TS_EVENT_CACHE_DELETE = 1128, + TS_EVENT_CACHE_WRITE = 1129, + TS_EVENT_CACHE_WRITE_HEADER = 1130, + TS_EVENT_CACHE_CLOSE = 1131, + TS_EVENT_CACHE_LOOKUP_READY = 1132, + TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, + TS_EVENT_CACHE_READ_READY = 1134, + TS_EVENT_CACHE_READ_COMPLETE = 1135, + + TS_EVENT_INTERNAL_1200 = 1200, + + TS_EVENT_SSL_SESSION_GET = 2000, + TS_EVENT_SSL_SESSION_NEW = 2001, + TS_EVENT_SSL_SESSION_REMOVE = 2002, + + TS_EVENT_AIO_DONE = 3900, + + TS_EVENT_HTTP_CONTINUE = 60000, + TS_EVENT_HTTP_ERROR = 60001, + TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, + TS_EVENT_HTTP_OS_DNS = 60003, + TS_EVENT_HTTP_SEND_REQUEST_HDR = 60004, + TS_EVENT_HTTP_READ_CACHE_HDR = 60005, + TS_EVENT_HTTP_READ_RESPONSE_HDR = 60006, + TS_EVENT_HTTP_SEND_RESPONSE_HDR = 60007, + TS_EVENT_HTTP_REQUEST_TRANSFORM = 60008, + TS_EVENT_HTTP_RESPONSE_TRANSFORM = 60009, + TS_EVENT_HTTP_SELECT_ALT = 60010, + TS_EVENT_HTTP_TXN_START = 60011, + TS_EVENT_HTTP_TXN_CLOSE = 60012, + TS_EVENT_HTTP_SSN_START = 60013, + TS_EVENT_HTTP_SSN_CLOSE = 60014, + TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, + TS_EVENT_HTTP_PRE_REMAP = 60016, + TS_EVENT_HTTP_POST_REMAP = 60017, + TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60018, TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60100, TS_EVENT_LIFECYCLE_PORTS_READY = 60101, @@ -488,16 +488,16 @@ typedef enum { TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60106, TS_EVENT_LIFECYCLE_SHUTDOWN = 60107, - TS_EVENT_INTERNAL_60200 = 60200, - TS_EVENT_INTERNAL_60201 = 60201, - TS_EVENT_INTERNAL_60202 = 60202, - TS_EVENT_SSL_CERT = 60203, - TS_EVENT_SSL_SERVERNAME = 60204, - TS_EVENT_SSL_VERIFY_SERVER = 60205, - TS_EVENT_SSL_VERIFY_CLIENT = 60206, - TS_EVENT_SSL_CLIENT_HELLO = 60207, + TS_EVENT_INTERNAL_60200 = 60200, + TS_EVENT_INTERNAL_60201 = 60201, + TS_EVENT_INTERNAL_60202 = 60202, + TS_EVENT_SSL_CERT = 60203, + TS_EVENT_SSL_SERVERNAME = 60204, + TS_EVENT_SSL_VERIFY_SERVER = 60205, + TS_EVENT_SSL_VERIFY_CLIENT = 60206, + TS_EVENT_SSL_CLIENT_HELLO = 60207, - TS_EVENT_MGMT_UPDATE = 60300 + TS_EVENT_MGMT_UPDATE = 60300 } TSEvent; #define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */ diff --git a/tools/clang-format.sh b/tools/clang-format.sh index ff97232fd86..6c37d5df5eb 100755 --- a/tools/clang-format.sh +++ b/tools/clang-format.sh @@ -77,7 +77,7 @@ EOF echo "or alternatively, undefine the FORMAT environment variable" exit 1 else - for file in $(find $DIR -iname \*.[ch] -o -iname \*.cc); do + for file in $(find $DIR -iname \*.[ch] -o -iname \*.cc -o -iname \*.h.in); do echo $file ${FORMAT} -i $file done diff --git a/tools/git/pre-commit b/tools/git/pre-commit index 3aee8e7420d..748215b85a8 100755 --- a/tools/git/pre-commit +++ b/tools/git/pre-commit @@ -47,7 +47,7 @@ patch_file=$(mktemp -t clang-format.XXXXXXXXXX) trap "rm -f $patch_file" 0 1 2 3 5 15 # Loop over all files that are changed, and produce a diff file -git diff-index --cached --diff-filter=ACMR --name-only HEAD | while read file; do +git diff-index --cached --diff-filter=ACMR --name-only HEAD | grep -vE "lib/tsconfig|lib/yamlcpp" | while read file; do case "$file" in *.cc | *.c | *.h | *.h.in) ${FORMAT} "$file" | diff -u "$file" - >> "$patch_file" From fa537711fec0f70916cb5fc2d6aa72590ed4708c Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Fri, 8 Mar 2019 17:18:48 +0000 Subject: [PATCH 353/526] cdniuc is not a manditory claim With Internet Draft 16 for uri signing, the cdniuc claim is not manditory. It took the place of the manditory sub claim in draft 12, and the manditory nature of the sub claim was still in effect. This change allows for tokens to not contain the cdniuc claim and also renews the cdniuc and cdnistd claim on token renewal. --- plugins/experimental/uri_signing/jwt.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index aeaa218424f..020d0cac0f8 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -193,7 +193,12 @@ jwt_check_uri(const char *cdniuc, const char *uri) static const char CONT_URI_HASH_STR[] = "hash"; static const char CONT_URI_REGEX_STR[] = "regex"; - if (!cdniuc || !*cdniuc || !uri) { + /* If cdniuc is not provided, skip uri check */ + if (!cdniuc || !*cdniuc) { + return true; + } + + if (!uri) { return false; } @@ -292,9 +297,11 @@ renew(struct jwt *jwt, const char *iss, cjose_jwk_t *jwk, const char *alg, const renew_copy_real(new_json, "nbf", jwt->nbf); renew_copy_real(new_json, "iat", now()); /* issued now */ renew_copy_string(new_json, "jti", jwt->jti); + renew_copy_string(new_json, "cdniuc", jwt->cdniuc); renew_copy_integer(new_json, "cdniv", jwt->cdniv); renew_copy_integer(new_json, "cdniets", jwt->cdniets); renew_copy_integer(new_json, "cdnistt", jwt->cdnistt); + renew_copy_integer(new_json, "cdnistd", jwt->cdnistd); char *pt = json_dumps(new_json, JSON_COMPACT); json_decref(new_json); From cb2382dbcaf0a3be8fd0ca391d3aa24343f69614 Mon Sep 17 00:00:00 2001 From: scw00 Date: Tue, 26 Feb 2019 16:38:17 +0800 Subject: [PATCH 354/526] Fixed the compatibility with previous cache verison --- iocore/cache/Cache.cc | 12 +++++- iocore/cache/I_CacheDefs.h | 2 +- proxy/hdrs/HTTP.cc | 81 ++++++++++++++++++++++++++++++++++++++ proxy/hdrs/HTTP.h | 1 + 4 files changed, 94 insertions(+), 2 deletions(-) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 87b6f1898a8..dd408c9e9e5 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -2139,10 +2139,20 @@ CacheVC::is_pread_capable() static void unmarshal_helper(Doc *doc, Ptr &buf, int &okay) { + using UnmarshalFunc = int(char *buf, int len, RefCountObj *block_ref); + UnmarshalFunc *unmarshal_func = &HTTPInfo::unmarshal; + ts::VersionNumber version(doc->v_major, doc->v_minor); + + // introduced by https://github.com/apache/trafficserver/pull/4874, this is used to distinguish the doc version + // before and after #4847 + if (version < CACHE_DB_VERSION) { + unmarshal_func = &HTTPInfo::unmarshal_v24_1; + } + char *tmp = doc->hdr(); int len = doc->hlen; while (len > 0) { - int r = HTTPInfo::unmarshal(tmp, len, buf.get()); + int r = unmarshal_func(tmp, len, buf.get()); if (r < 0) { ink_assert(!"CacheVC::handleReadDone unmarshal failed"); okay = 0; diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h index 619d20c158f..cdd39bb2199 100644 --- a/iocore/cache/I_CacheDefs.h +++ b/iocore/cache/I_CacheDefs.h @@ -33,7 +33,7 @@ #define CACHE_ALT_REMOVED -2 static const uint8_t CACHE_DB_MAJOR_VERSION = 24; -static const uint8_t CACHE_DB_MINOR_VERSION = 1; +static const uint8_t CACHE_DB_MINOR_VERSION = 2; // This is used in various comparisons because otherwise if the minor version is 0, // the compile fails because the condition is always true or false. Running it through // VersionNumber prevents that. diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 8faf57eca0f..50726d0702d 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -2136,6 +2136,87 @@ HTTPInfo::unmarshal(char *buf, int len, RefCountObj *block_ref) return alt->m_unmarshal_len; } +int +HTTPInfo::unmarshal_v24_1(char *buf, int len, RefCountObj *block_ref) +{ + HTTPCacheAlt *alt = (HTTPCacheAlt *)buf; + int orig_len = len; + + if (alt->m_magic == CACHE_ALT_MAGIC_ALIVE) { + // Already unmarshaled, must be a ram cache + // it + ink_assert(alt->m_unmarshal_len > 0); + ink_assert(alt->m_unmarshal_len <= len); + return alt->m_unmarshal_len; + } else if (alt->m_magic != CACHE_ALT_MAGIC_MARSHALED) { + ink_assert(!"HTTPInfo::unmarshal bad magic"); + return -1; + } + + ink_assert(alt->m_unmarshal_len < 0); + alt->m_magic = CACHE_ALT_MAGIC_ALIVE; + ink_assert(alt->m_writeable == 0); + len -= HTTP_ALT_MARSHAL_SIZE; + + if (alt->m_frag_offset_count > HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS) { + // stuff that didn't fit in the integral slots. + int extra = sizeof(FragOffset) * alt->m_frag_offset_count - sizeof(alt->m_integral_frag_offsets); + char *extra_src = buf + reinterpret_cast(alt->m_frag_offsets); + // Actual buffer size, which must be a power of two. + // Well, technically not, because we never modify an unmarshalled fragment + // offset table, but it would be a nasty bug should that be done in the + // future. + int bcount = HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS * 2; + + while (bcount < alt->m_frag_offset_count) { + bcount *= 2; + } + alt->m_frag_offsets = + static_cast(ats_malloc(bcount * sizeof(FragOffset))); // WRONG - must round up to next power of 2. + memcpy(alt->m_frag_offsets, alt->m_integral_frag_offsets, sizeof(alt->m_integral_frag_offsets)); + memcpy(alt->m_frag_offsets + HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS, extra_src, extra); + len -= extra; + } else if (alt->m_frag_offset_count > 0) { + alt->m_frag_offsets = alt->m_integral_frag_offsets; + } else { + alt->m_frag_offsets = nullptr; // should really already be zero. + } + + HdrHeap *heap = (HdrHeap *)(alt->m_request_hdr.m_heap ? (buf + (intptr_t)alt->m_request_hdr.m_heap) : nullptr); + HTTPHdrImpl *hh = nullptr; + int tmp; + if (heap != nullptr) { + tmp = heap->unmarshal(len, HDR_HEAP_OBJ_HTTP_HEADER, (HdrHeapObjImpl **)&hh, block_ref); + if (hh == nullptr || tmp < 0) { + ink_assert(!"HTTPInfo::request unmarshal failed"); + return -1; + } + len -= tmp; + alt->m_request_hdr.m_heap = heap; + alt->m_request_hdr.m_http = hh; + alt->m_request_hdr.m_mime = hh->m_fields_impl; + alt->m_request_hdr.m_url_cached.m_heap = heap; + } + + heap = (HdrHeap *)(alt->m_response_hdr.m_heap ? (buf + (intptr_t)alt->m_response_hdr.m_heap) : nullptr); + if (heap != nullptr) { + tmp = heap->unmarshal(len, HDR_HEAP_OBJ_HTTP_HEADER, (HdrHeapObjImpl **)&hh, block_ref); + if (hh == nullptr || tmp < 0) { + ink_assert(!"HTTPInfo::response unmarshal failed"); + return -1; + } + len -= tmp; + + alt->m_response_hdr.m_heap = heap; + alt->m_response_hdr.m_http = hh; + alt->m_response_hdr.m_mime = hh->m_fields_impl; + } + + alt->m_unmarshal_len = orig_len - len; + + return alt->m_unmarshal_len; +} + // bool HTTPInfo::check_marshalled(char* buf, int len) // Checks a marhshalled HTTPInfo buffer to make // sure it's sane. Returns true if sane, false otherwise diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h index a5fc4e69589..a3adb759a40 100644 --- a/proxy/hdrs/HTTP.h +++ b/proxy/hdrs/HTTP.h @@ -1375,6 +1375,7 @@ class HTTPInfo inkcoreapi int marshal_length(); inkcoreapi int marshal(char *buf, int len); static int unmarshal(char *buf, int len, RefCountObj *block_ref); + static int unmarshal_v24_1(char *buf, int len, RefCountObj *block_ref); void set_buffer_reference(RefCountObj *block_ref); int get_handle(char *buf, int len); From 9c9052ab7747b11a79bdd0c7cfa1e4a60de6d009 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Mon, 11 Mar 2019 10:06:09 -0700 Subject: [PATCH 355/526] Updates to the STATUS file --- STATUS | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/STATUS b/STATUS index 8bc06306f48..2b3c2e9edd6 100644 --- a/STATUS +++ b/STATUS @@ -6,11 +6,13 @@ The current version of this file can be found at: * https://github.com/apache/trafficserver/blob/master/STATUS Release history: - 8.0.2 : Release on , 2019 + 8.0.3 : Release on , 2019 + 8.0.2 : Release on Jan 29th, 2019 8.0.1 : Release on Nov 29th, 2018 8.0.0 : Release on Sep 25th, 2018 - 7.1.6 : Released on , 2018 + 7.1.7 : Released on , 2019 + 7.1.6 : Released on Jan 29th, 2019 7.1.5 : Released on Nov 24th, 2018 7.1.4 : Released on Aug 1st, 2018 7.1.3 : Released on Apr 16th, 2018 From 497a33e44d244c117c3e38414192f4fb3405d843 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Wed, 6 Mar 2019 11:01:35 -0800 Subject: [PATCH 356/526] Fixed memory leaks in test_IntrusiveHashMap --- .../unit_tests/test_IntrusiveHashMap.cc | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/tscore/unit_tests/test_IntrusiveHashMap.cc b/src/tscore/unit_tests/test_IntrusiveHashMap.cc index 5e615c0e8d1..1675239b703 100644 --- a/src/tscore/unit_tests/test_IntrusiveHashMap.cc +++ b/src/tscore/unit_tests/test_IntrusiveHashMap.cc @@ -128,9 +128,11 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") Map::iterator idx; // Erase all the non-"dup" and see if the range is still correct. - map.apply([&map](Thing &thing) { - if (thing._payload != "dup"sv) - map.erase(map.iterator_for(&thing)); + map.apply([&map](Thing *thing) { + if (thing->_payload != "dup"sv) { + map.erase(map.iterator_for(thing)); + delete thing; + } }); r = map.equal_range("dup"sv); REQUIRE(r.first != r.second); @@ -155,7 +157,7 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") // Some more involved tests. TEST_CASE("IntrusiveHashMapManyStrings", "[IntrusiveHashMap]") { - std::vector strings; + std::vector strings; std::uniform_int_distribution char_gen{'a', 'z'}; std::uniform_int_distribution length_gen{20, 40}; @@ -167,12 +169,12 @@ TEST_CASE("IntrusiveHashMapManyStrings", "[IntrusiveHashMap]") strings.reserve(N); for (int i = 0; i < N; ++i) { auto len = length_gen(randu); - char *s = static_cast(malloc(len + 1)); + std::string s; + s.reserve(len); for (decltype(len) j = 0; j < len; ++j) { - s[j] = char_gen(randu); + s += char_gen(randu); } - s[len] = 0; - strings.push_back({s, size_t(len)}); + strings.push_back(s); } // Fill the IntrusiveHashMap @@ -233,4 +235,6 @@ TEST_CASE("IntrusiveHashMapManyStrings", "[IntrusiveHashMap]") } } REQUIRE(miss_p == false); + + ihm.apply([](Thing *thing) { delete thing; }); }; From f683b767405908c22c2fdd2a9d38cbfe9e8c335a Mon Sep 17 00:00:00 2001 From: Aaron Canary Date: Fri, 8 Mar 2019 13:56:01 -0600 Subject: [PATCH 357/526] Fix test_AcidPtr needed to call destructor --- src/tscore/unit_tests/test_AcidPtr.cc | 17 +++++++++++++++++ src/tscore/unit_tests/test_Extendible.cc | 24 ++++++++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/tscore/unit_tests/test_AcidPtr.cc b/src/tscore/unit_tests/test_AcidPtr.cc index e3f89017ae0..f33af7bdf50 100644 --- a/src/tscore/unit_tests/test_AcidPtr.cc +++ b/src/tscore/unit_tests/test_AcidPtr.cc @@ -94,3 +94,20 @@ TEST_CASE("AcidPtr Isolation") } CHECK(*p.getPtr() == 42); } + +TEST_CASE("AcidPtr Abort") +{ + AcidPtr p; + { + AcidCommitPtr w(p); + *w = 40; + } + CHECK(*p.getPtr() == 40); + { + AcidCommitPtr w = p; + *w += 1; + w.abort(); + CHECK(w == nullptr); + } + CHECK(*p.getPtr() == 40); +} diff --git a/src/tscore/unit_tests/test_Extendible.cc b/src/tscore/unit_tests/test_Extendible.cc index 6a0900ecf28..31669292725 100644 --- a/src/tscore/unit_tests/test_Extendible.cc +++ b/src/tscore/unit_tests/test_Extendible.cc @@ -209,13 +209,28 @@ TEST_CASE("Extendible", "") CHECK(testField::alive == 0); } + INFO("AcidPtr AcidCommitPtr malloc ptr int"); + { + void *mem = malloc(sizeof(AcidPtr)); + AcidPtr &reader = *(new (mem) AcidPtr); + { + auto writer = reader.startCommit(); + CHECK(*writer == 0); + *writer = 1; + CHECK(*writer == 1); + CHECK(*reader.getPtr().get() == 0); + // end of scope writer, commit to reader + } + CHECK(*reader.getPtr().get() == 1); + reader.~AcidPtr(); + free(mem); + } INFO("AcidPtr AcidCommitPtr casting"); { - void *mem = malloc(sizeof(AcidPtr)); - new (mem) AcidPtr(); - AcidPtr &reader = *static_cast *>(mem); + void *mem = malloc(sizeof(AcidPtr)); + AcidPtr &reader = *(new (mem) AcidPtr); { - AcidCommitPtr writer = AcidCommitPtr(reader); + auto writer = reader.startCommit(); CHECK(writer->arr[0] == 1); CHECK(reader.getPtr()->arr[0] == 1); writer->arr[0] = 99; @@ -223,6 +238,7 @@ TEST_CASE("Extendible", "") CHECK(reader.getPtr()->arr[0] == 1); } CHECK(reader.getPtr()->arr[0] == 99); + reader.~AcidPtr(); free(mem); } INFO("ACIDPTR block-free reader") From 42fec9a22288b3932783566de7006070b2daf151 Mon Sep 17 00:00:00 2001 From: dyrock Date: Mon, 11 Mar 2019 16:11:27 -0500 Subject: [PATCH 358/526] Separate P_SSLUtil and P_SSLClientUtils includes --- iocore/net/P_SSLUtils.h | 3 +-- iocore/net/SSLConfig.cc | 1 + src/traffic_server/traffic_server.cc | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index 648ed8035ff..bc6fbf21219 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -31,8 +31,7 @@ #include "tscore/ink_config.h" #include "tscore/Diags.h" - -#include "P_SSLClientUtils.h" +#include "records/I_RecCore.h" #include "P_SSLCertLookup.h" struct SSLConfigParams; diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index cb477f8f142..491bd784627 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -42,6 +42,7 @@ #include "P_Net.h" #include "P_SSLUtils.h" +#include "P_SSLClientUtils.h" #include "P_SSLCertLookup.h" #include "SSLDiags.h" #include "SSLSessionCache.h" diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 7aba78d17ca..c35f00c9f90 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -96,6 +96,7 @@ extern "C" int plock(int); #include "HTTP2.h" #include "tscore/ink_config.h" #include "P_SSLSNI.h" +#include "P_SSLClientUtils.h" #include "tscore/ink_cap.h" From 3d08c28471b05b7f46d3dc84d64945a433be3571 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 11 Mar 2019 13:53:34 +0000 Subject: [PATCH 359/526] Augment wildcard_match to allow underscore in domain name --- src/tscore/X509HostnameValidator.cc | 2 +- tests/gold_tests/tls/ssl/wild-signed.pem | 18 +++++++++++++++ tests/gold_tests/tls/ssl/wild.key | 28 ++++++++++++++++++++++++ tests/gold_tests/tls/tls_verify.test.py | 28 ++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 tests/gold_tests/tls/ssl/wild-signed.pem create mode 100644 tests/gold_tests/tls/ssl/wild.key diff --git a/src/tscore/X509HostnameValidator.cc b/src/tscore/X509HostnameValidator.cc index ab3a7800f5e..73f03b01428 100644 --- a/src/tscore/X509HostnameValidator.cc +++ b/src/tscore/X509HostnameValidator.cc @@ -166,7 +166,7 @@ wildcard_match(const unsigned char *prefix, size_t prefix_len, const unsigned ch * permitted characters and only matches a single label */ for (p = wildcard_start; p != wildcard_end; ++p) { - if (!(('0' <= *p && *p <= '9') || ('A' <= *p && *p <= 'Z') || ('a' <= *p && *p <= 'z') || *p == '-')) { + if (!(('a' <= *p && *p <= 'z') || ('A' <= *p && *p <= 'Z') || ('0' <= *p && *p <= '9') || *p == '-' || *p == '_')) { return false; } } diff --git a/tests/gold_tests/tls/ssl/wild-signed.pem b/tests/gold_tests/tls/ssl/wild-signed.pem new file mode 100644 index 00000000000..d6f25ff7086 --- /dev/null +++ b/tests/gold_tests/tls/ssl/wild-signed.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC2TCCAkICCQCTw2t3s0sIrjANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTkwMzExMTQ1ODEzWhcNMjkwMzA4MTQ1ODEzWjBAMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxDzANBgNVBAoMBmFwYWNoZTETMBEGA1UEAwwKKi53aWxk +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM51MZvJ4anfZkoF +w+yLI+qtSamz/nOEeaNYvpJIzfH/qG04hxujCMfodknWoElBcJfzJ6I+RTDev4DZ +QFiDaetwQv4/+H17fTc2RJR1PdOKA06rRfm76P6CrlRttWFYLfcUTooTUoQUBFGu +EFTBGWUpFWU4fVXbYKLT63PzEkCeeHU5wf7Xz88pTYI21ZWnQQt2ll+zXLgSHFCP +m9XO5Q/Dr++XpFI2joCYj1GL/y3mku26sdK6ZYxRo9oseYiVX8K9T6Oeo4O32CcT +3ZE4QMa1jXcb5r4Y6VBdneZkur10CW2+yDT9MX5dKBeeuB9UUTYClt7lJdncl5DH +GdLrV9kCAwEAATANBgkqhkiG9w0BAQsFAAOBgQCRpZoeIRSF83Iiy7COW59KG9qq +T9pSCbJnqJr9ri1usmgcO9mfb7lQt94RXCtNz+tF85ciDE3zXcM6nJyosgStQEbb +AkGcW0z29Lc+Owc6SYtN6o4fEgPM61SSABtdG26jhpzMjT8MZNXluqXJ6tR0HWki +Rep/Bm0POzeUeKb8GA== +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/wild.key b/tests/gold_tests/tls/ssl/wild.key new file mode 100644 index 00000000000..a7b57205e2b --- /dev/null +++ b/tests/gold_tests/tls/ssl/wild.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOdTGbyeGp32ZK +BcPsiyPqrUmps/5zhHmjWL6SSM3x/6htOIcbowjH6HZJ1qBJQXCX8yeiPkUw3r+A +2UBYg2nrcEL+P/h9e303NkSUdT3TigNOq0X5u+j+gq5UbbVhWC33FE6KE1KEFARR +rhBUwRllKRVlOH1V22Ci0+tz8xJAnnh1OcH+18/PKU2CNtWVp0ELdpZfs1y4EhxQ +j5vVzuUPw6/vl6RSNo6AmI9Ri/8t5pLturHSumWMUaPaLHmIlV/CvU+jnqODt9gn +E92ROEDGtY13G+a+GOlQXZ3mZLq9dAltvsg0/TF+XSgXnrgfVFE2Apbe5SXZ3JeQ +xxnS61fZAgMBAAECggEARyRdFtjHGRkxDzrTW5RKqRhTdNXgTYANxjrTWGccCFLX +f+NlsyFH6lLxR9pcW0HggYu5UY/xmbh39vdl09pcylNh0mjKwLqn2DmsAhgwWM+K ++jXMpBSbYfA4EEHJqaSQGj72HWAxI/Ad2OOJHxt3G8O/aqS/k7FHqHQsA0V0Oa2D +NFLscCQxzH3IzUhbhZhtoCrtNfFkxSgx7RQbHMHNxMqaNU3MtKafQ1KJqR+SuuQL +donKPL6A0Xi8MKOc8qsyrGtKL/+EEY0W77SRJAkY8N29X1oMNPnGzLsicarwhHLX +1KT+rxBISJ/tbGuTvaJNbCepvO6I6QLZshQZt3MuAQKBgQDvbdSoEzfDO3vcbphH +QMGjASMkGP7T/goCvFQwUGN03GtMnoHuYbbfpZ6iOcYVGefU6v0LDOqhczTwKFVq +44msdQt/T1k9A+wHp6/1SnxObJT8wJoGmJO9qryc0yoRYt3lv20jl3GSC5xMjrJL +2Ko9DwNZaZNTlOFLJzzOOSUqwQKBgQDcvy7zupQivgK90gFId/iRjXz0UuVWseGo +osDL+BV4w4jTltLUE6AMVRQCqF85O1spCcn3AKjU+kWj0fIcB38y2UZRv6Moy/u8 +w5fRAhHONqNPwb9yOLNplLdm7lOE6mm+nYVgauh13nca8bxeNKmRZe7tyEu2i/4o +kZqTecvrGQKBgAm/QuUEw0Rja4txxSlBbaChLzkM+3LN6MJrwFGnNCVRw9x+p3N4 +7uTz7R1VlMbPIyz71AlbIUIpWoJcYf3T/YrTyQAJzuw4+KbnILavrZfTu8z+Wkbi +d0FFbiBESHYkvDvaKytDww/bASXsuT11OJj7v3soXSMN8I4KruMGWIkBAoGBAMQl +GNo2ylQIlDUIul0jRPpIR2RtmBytmH6Yh0l2GdYhoJ2qIZGSEp+CpXIrG9ml1T2k +1hGlQ19jNqf27/NZ8ftDtskCyD6C6h9ziJ2OAjZCtGA1HyCmIz1IiKJsWEf9ZpKa +Mx5WQFIjp5+IdsEaeCWa9m/Qjv4YbHCt2DT8f2ZZAoGBALzd1ju4Sf5k+eP4QDO9 +jWRRRyjHIrT8RX4Hb3YZWJmW8edxjlV63GKVyuwv+PNyaP0hrLpVEw7h9tAYaIoL +DAjqph02ujDBe5oJrOMrO3IkxVqHj2PQFBnFR0MCbkQ9/+vyTt7ftmwXwQQFeDVW +YcOEYXCqeaAzjN9t0xidlkX7 +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/tls_verify.test.py b/tests/gold_tests/tls/tls_verify.test.py index 609f56c278f..52571e6c5cc 100644 --- a/tests/gold_tests/tls/tls_verify.test.py +++ b/tests/gold_tests/tls/tls_verify.test.py @@ -30,6 +30,7 @@ ts = Test.MakeATSProcess("ts", select_ports=False) server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server_wild = Test.MakeOriginServer("server_wild", ssl=True, options = {"--key": "{0}/wild.key".format(Test.RunDirectory), "--cert": "{0}/wild-signed.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -41,6 +42,7 @@ server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) server_bar.addResponse("sessionlog.json", request_bar_header, response_header) server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) +server_wild.addResponse("sessionlog.json", request_bar_header, response_header) # add ssl materials like key, certificates for the server ts.addSSLfile("ssl/signed-foo.pem") @@ -51,6 +53,8 @@ ts.addSSLfile("ssl/server.key") ts.addSSLfile("ssl/signer.pem") ts.addSSLfile("ssl/signer.key") +ts.addSSLfile("ssl/wild.key") +ts.addSSLfile("ssl/wild-signed.pem") ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( @@ -63,6 +67,10 @@ 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://foo.wild.com/ https://127.0.0.1:{0}'.format(server_wild.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://foo_bar.wild.com/ https://127.0.0.1:{0}'.format(server_wild.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' @@ -89,6 +97,9 @@ '- fqdn: bar.com', ' verify_server_policy: ENFORCED', ' verify_server_properties: ALL', + '- fqdn: "*.wild.com"', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', '- fqdn: bad_bar.com', ' verify_server_policy: ENFORCED', ' verify_server_properties: ALL' @@ -99,12 +110,15 @@ tr.Setup.Copy("ssl/signed-foo.pem") tr.Setup.Copy("ssl/signed-bar.key") tr.Setup.Copy("ssl/signed-bar.pem") +tr.Setup.Copy("ssl/wild-signed.pem") +tr.Setup.Copy("ssl/wild.key") tr.Processes.Default.Command = "curl -v -k -H \"host: foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server_foo) tr.Processes.Default.StartBefore(server_bar) tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(server_wild) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts @@ -124,6 +138,20 @@ tr3.StillRunningAfter = server tr3.StillRunningAfter = ts +tr4 = Test.AddTestRun("Exercise-wildcard-cert-name-check") +tr4.Processes.Default.Command = "curl -v -k -H \"host: foo.wild.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr4.ReturnCode = 0 +tr4.StillRunningAfter = server +tr4.StillRunningAfter = ts + +tr5 = Test.AddTestRun("Exercise-wildcard-cert-underscore-name-check") +tr5.Processes.Default.Command = "curl -v -k -H \"host: foo_bar.wild.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr5.ReturnCode = 0 +tr5.StillRunningAfter = server +tr5.StillRunningAfter = ts + # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bad_bar.com\) not in certificate", "Make sure bad_bar name checked failed.") From f8bda563c9791b5242d3b82339039f9eb6a8c39a Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Fri, 8 Feb 2019 16:36:13 -0600 Subject: [PATCH 360/526] Remove error-prone mirror enum in code handling TS API SSL hooks. --- iocore/net/SSLNetVConnection.cc | 18 +++++------ iocore/net/SSLUtils.cc | 6 ++-- proxy/InkAPIInternal.h | 56 ++++++++++++++++++--------------- src/traffic_server/InkAPI.cc | 4 +-- 4 files changed, 44 insertions(+), 40 deletions(-) diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index a93869b5a75..bc3cd8ec459 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -1124,7 +1124,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) { if (!curHook) { Debug("ssl", "Initialize preaccept curHook from NULL"); - curHook = ssl_hooks->get(TS_VCONN_START_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_START_HOOK)); } else { curHook = curHook->next(); } @@ -1375,7 +1375,7 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE) { if (!curHook) { Debug("ssl", "Initialize outbound connect curHook from NULL"); - curHook = ssl_hooks->get(TS_VCONN_OUTBOUND_START_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_OUTBOUND_START_HOOK)); } else { curHook = curHook->next(); } @@ -1694,7 +1694,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) case HANDSHAKE_HOOKS_CLIENT_HELLO: case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_CLIENT_HELLO_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_CLIENT_HELLO_HOOK)); } else { curHook = curHook->next(); } @@ -1708,14 +1708,14 @@ SSLNetVConnection::callHooks(TSEvent eventId) // The server verify event addresses ATS to origin handshake // All the other events are for client to ATS if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_VERIFY_SERVER_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_VERIFY_SERVER_HOOK)); } else { curHook = curHook->next(); } break; case HANDSHAKE_HOOKS_SNI: if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_SERVERNAME_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SERVERNAME_HOOK)); } else { curHook = curHook->next(); } @@ -1726,7 +1726,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) case HANDSHAKE_HOOKS_CERT: case HANDSHAKE_HOOKS_CERT_INVOKE: if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_CERT_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_CERT_HOOK)); } else { curHook = curHook->next(); } @@ -1739,7 +1739,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) case HANDSHAKE_HOOKS_CLIENT_CERT: case HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE: if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_VERIFY_CLIENT_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_VERIFY_CLIENT_HOOK)); } else { curHook = curHook->next(); } @@ -1749,14 +1749,14 @@ SSLNetVConnection::callHooks(TSEvent eventId) if (eventId == TS_EVENT_VCONN_CLOSE) { sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; if (curHook == nullptr) { - curHook = ssl_hooks->get(TS_VCONN_CLOSE_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_CLOSE_HOOK)); } else { curHook = curHook->next(); } } else if (eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE) { sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; if (curHook == nullptr) { - curHook = ssl_hooks->get(TS_VCONN_OUTBOUND_CLOSE_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_OUTBOUND_CLOSE_HOOK)); } else { curHook = curHook->next(); } diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index c83b3143803..41411ab8c92 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -193,7 +193,7 @@ ssl_get_cached_session(SSL *ssl, const unsigned char *id, int len, int *copy) Debug("ssl.session_cache.get", "ssl_get_cached_session cached session '%s' context %p", printable_buf, SSL_get_SSL_CTX(ssl)); } - APIHook *hook = ssl_hooks->get(TS_SSL_SESSION_INTERNAL_HOOK); + APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); while (hook) { hook->invoke(TS_EVENT_SSL_SESSION_GET, &sid); hook = hook->m_link.next; @@ -241,7 +241,7 @@ ssl_new_cached_session(SSL *ssl, SSL_SESSION *sess) session_cache->insertSession(sid, sess); // Call hook after new session is created - APIHook *hook = ssl_hooks->get(TS_SSL_SESSION_INTERNAL_HOOK); + APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); while (hook) { hook->invoke(TS_EVENT_SSL_SESSION_NEW, &sid); hook = hook->m_link.next; @@ -258,7 +258,7 @@ ssl_rm_cached_session(SSL_CTX *ctx, SSL_SESSION *sess) SSLSessionID sid(id, len); // Call hook before session is removed - APIHook *hook = ssl_hooks->get(TS_SSL_SESSION_INTERNAL_HOOK); + APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); while (hook) { hook->invoke(TS_EVENT_SSL_SESSION_REMOVE, &sid); hook = hook->m_link.next; diff --git a/proxy/InkAPIInternal.h b/proxy/InkAPIInternal.h index 7955c532328..ccfd08bb559 100644 --- a/proxy/InkAPIInternal.h +++ b/proxy/InkAPIInternal.h @@ -156,7 +156,7 @@ APIHooks::is_empty() const maximum hook ID so the valid ids are 0..(N-1) in the standard C array style. */ template class FeatureAPIHooks { @@ -194,14 +194,14 @@ class FeatureAPIHooks APIHooks m_hooks[N]; }; -template FeatureAPIHooks::FeatureAPIHooks() : hooks_p(false) {} +template FeatureAPIHooks::FeatureAPIHooks() : hooks_p(false) {} -template FeatureAPIHooks::~FeatureAPIHooks() +template FeatureAPIHooks::~FeatureAPIHooks() { this->clear(); } -template +template void FeatureAPIHooks::clear() { @@ -211,7 +211,7 @@ FeatureAPIHooks::clear() hooks_p = false; } -template +template void FeatureAPIHooks::prepend(ID id, INKContInternal *cont) { @@ -221,7 +221,7 @@ FeatureAPIHooks::prepend(ID id, INKContInternal *cont) } } -template +template void FeatureAPIHooks::append(ID id, INKContInternal *cont) { @@ -231,14 +231,14 @@ FeatureAPIHooks::append(ID id, INKContInternal *cont) } } -template +template APIHook * FeatureAPIHooks::get(ID id) const { return likely(is_valid(id)) ? m_hooks[id].get() : nullptr; } -template +template void FeatureAPIHooks::invoke(ID id, int event, void *data) { @@ -247,14 +247,14 @@ FeatureAPIHooks::invoke(ID id, int event, void *data) } } -template +template bool FeatureAPIHooks::has_hooks() const { return hooks_p; } -template +template bool FeatureAPIHooks::is_valid(ID id) { @@ -265,22 +265,26 @@ class HttpAPIHooks : public FeatureAPIHooks { }; -typedef enum { - TS_SSL_INTERNAL_FIRST_HOOK, - TS_VCONN_START_INTERNAL_HOOK = TS_SSL_INTERNAL_FIRST_HOOK, - TS_VCONN_CLOSE_INTERNAL_HOOK, - TS_SSL_CLIENT_HELLO_INTERNAL_HOOK, - TS_SSL_CERT_INTERNAL_HOOK, - TS_SSL_SERVERNAME_INTERNAL_HOOK, - TS_SSL_VERIFY_SERVER_INTERNAL_HOOK, - TS_SSL_VERIFY_CLIENT_INTERNAL_HOOK, - TS_SSL_SESSION_INTERNAL_HOOK, - TS_VCONN_OUTBOUND_START_INTERNAL_HOOK, - TS_VCONN_OUTBOUND_CLOSE_INTERNAL_HOOK, - TS_SSL_INTERNAL_LAST_HOOK -} TSSslHookInternalID; - -class SslAPIHooks : public FeatureAPIHooks +class TSSslHookInternalID +{ +public: + constexpr TSSslHookInternalID(TSHttpHookID id) : _id(id - TS_SSL_FIRST_HOOK) {} + + constexpr operator int() const { return _id; } + + static const int NUM = TS_SSL_LAST_HOOK - TS_SSL_FIRST_HOOK + 1; + + constexpr bool + is_in_bounds() const + { + return (_id >= 0) && (_id < NUM); + } + +private: + const int _id; +}; + +class SslAPIHooks : public FeatureAPIHooks { }; diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 62cda080053..c2ac86055b7 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -4696,8 +4696,8 @@ TSHttpHookAdd(TSHttpHookID id, TSCont contp) icontp = reinterpret_cast(contp); - if (id >= TS_SSL_FIRST_HOOK && id <= TS_SSL_LAST_HOOK) { - TSSslHookInternalID internalId = static_cast(id - TS_SSL_FIRST_HOOK); + TSSslHookInternalID internalId{id}; + if (internalId.is_in_bounds()) { ssl_hooks->append(internalId, icontp); } else { // Follow through the regular HTTP hook framework http_global_hooks->append(id, icontp); From e27bcec1ba191668000b7f8d0cdaacc9c0e3c0da Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Wed, 13 Mar 2019 21:46:17 +0800 Subject: [PATCH 361/526] Fix memleak in ProcessManager::stop() --- mgmt/ProcessManager.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc index e8e6866fc00..35d2b3cbfc3 100644 --- a/mgmt/ProcessManager.cc +++ b/mgmt/ProcessManager.cc @@ -126,7 +126,9 @@ ProcessManager::stop() ats_free(sig); } - ats_free(mgmt_signal_queue); + LLQ *tmp_queue = mgmt_signal_queue; + mgmt_signal_queue = nullptr; + delete_queue(tmp_queue); } /* From 33bbe3432a1a58ff526fe097424c17514d431b3c Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 5 Mar 2019 17:51:31 +0000 Subject: [PATCH 362/526] Ensure queued HostDB requests get rescheduled back to the original thread. --- iocore/cache/CacheWrite.cc | 10 ++-------- iocore/cache/P_CacheInternal.h | 3 +-- iocore/hostdb/HostDB.cc | 12 ++++++++++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index 178ec08ddb7..48d16cb1f2b 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -358,7 +358,7 @@ Vol::aggWriteDone(int event, Event *e) CacheVC *c = nullptr; while ((c = sync.dequeue())) { if (UINT_WRAP_LTE(c->write_serial + 2, header->write_serial)) { - c->initial_thread->schedule_imm_signal(c, AIO_EVENT_DONE); + eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); } else { sync.push(c); // put it back on the front break; @@ -1019,11 +1019,7 @@ Vol::aggWrite(int event, void * /* e ATS_UNUSED */) ink_assert(false); while ((c = agg.dequeue())) { agg_todo_size -= c->agg_len; - if (c->initial_thread != nullptr) { - c->initial_thread->schedule_imm_signal(c, AIO_EVENT_DONE); - } else { - eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); - } + eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); } return EVENT_CONT; } @@ -1086,8 +1082,6 @@ Vol::aggWrite(int event, void * /* e ATS_UNUSED */) while ((c = tocall.dequeue())) { if (event == EVENT_CALL && c->mutex->thread_holding == mutex->thread_holding) { ret = EVENT_RETURN; - } else if (c->initial_thread != nullptr) { - c->initial_thread->schedule_imm_signal(c, AIO_EVENT_DONE); } else { eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); } diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h index d82aa347324..8ed84757789 100644 --- a/iocore/cache/P_CacheInternal.h +++ b/iocore/cache/P_CacheInternal.h @@ -440,7 +440,6 @@ struct CacheVC : public CacheVConnection { // NOTE: NOTE: NOTE: If vio is NOT the start, then CHANGE the // size_to_init initialization VIO vio; - EThread *initial_thread; // initial thread open_XX was called on CacheFragType frag_type; CacheHTTPInfo *info; CacheHTTPInfoVector *write_vector; @@ -549,9 +548,9 @@ new_CacheVC(Continuation *cont) CacheVC *c = THREAD_ALLOC(cacheVConnectionAllocator, t); c->vector.data.data = &c->vector.data.fast_data[0]; c->_action = cont; - c->initial_thread = t->tt == DEDICATED ? nullptr : t; c->mutex = cont->mutex; c->start_time = Thread::get_hrtime(); + c->setThreadAffinity(t); ink_assert(c->trigger == nullptr); Debug("cache_new", "new %p", c); #ifdef CACHE_STAT_PAGES diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 73c80141c8d..ed7ba5faeea 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -1578,7 +1578,8 @@ int HostDBContinuation::set_check_pending_dns() { Queue &q = hostDB.pending_dns_for_hash(hash.hash); - HostDBContinuation *c = q.head; + this->setThreadAffinity(this_ethread()); + HostDBContinuation *c = q.head; for (; c; c = (HostDBContinuation *)c->link.next) { if (hash.hash == c->hash.hash) { Debug("hostdb", "enqueuing additional request"); @@ -1606,8 +1607,15 @@ HostDBContinuation::remove_trigger_pending_dns() } c = n; } + EThread *thread = this_ethread(); while ((c = qq.dequeue())) { - c->handleEvent(EVENT_IMMEDIATE, nullptr); + // resume all queued HostDBCont in the thread associated with the netvc to avoid nethandler locking issues. + EThread *affinity_thread = c->getThreadAffinity(); + if (!affinity_thread || affinity_thread == thread) { + c->handleEvent(EVENT_IMMEDIATE, nullptr); + } else { + eventProcessor.schedule_imm(c); + } } } From 0d55280c07bd54a2438c9a9ffc901372b1975c97 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 18 Oct 2018 17:33:06 -0600 Subject: [PATCH 363/526] Adds the log tag cqint for internal requests --- doc/admin-guide/logging/formatting.en.rst | 25 ++++++++++++--------- proxy/http/HttpSM.cc | 12 +++++----- proxy/http/HttpSM.h | 1 + proxy/logging/Log.cc | 5 +++++ proxy/logging/LogAccess.cc | 27 ++++++++++------------- proxy/logging/LogAccess.h | 1 + 6 files changed, 39 insertions(+), 32 deletions(-) diff --git a/doc/admin-guide/logging/formatting.en.rst b/doc/admin-guide/logging/formatting.en.rst index 5295d6bd308..1a4bce0f9ce 100644 --- a/doc/admin-guide/logging/formatting.en.rst +++ b/doc/admin-guide/logging/formatting.en.rst @@ -528,20 +528,25 @@ Plugin Details .. _piid: .. _pitag: +.. _cqint: Logging fields which may be used to obtain details of plugins involved in the transaction. -===== ============ ============================================================ -Field Source Description -===== ============ ============================================================ -piid Proxy Plugin Plugin ID for the current transaction. This is set for - plugin driven transactions via - :c:func:`TSHttpConnectWithPluginId`. -pitag Proxy Plugin Plugin tag for the current transaction. This is set for - plugin driven transactions via - :c:func:`TSHttpConnectWithPluginId`. -===== ============ ============================================================ +===== ================ ============================================================ +Field Source Description +===== ================ ============================================================ +piid Proxy Plugin Plugin ID for the current transaction. This is set for + plugin driven transactions via + :c:func:`TSHttpConnectWithPluginId`. +pitag Proxy Plugin Plugin tag for the current transaction. This is set for + plugin driven transactions via + :c:func:`TSHttpConnectWithPluginId`. +cqint Client Request If a request was generated internally (via a plugin), then + this has a value of ``1``, otherwise ``0``. This can be + useful when tracking internal only requests, such as those + generated by the ``authproxy`` plugin. +===== ================ ============================================================ .. _admin-logging-fields-proto: diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 5b0fa9afd88..81ec27fff47 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -460,10 +460,13 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader _client_transaction_id = ua_txn->get_transaction_id(); { auto p = ua_txn->get_parent(); + if (p) { _client_connection_id = p->connection_id(); } } + // We've already verified that the netvc is !nullptr above, and netvc == ua_txn->get_netvc() + is_internal = netvc->get_is_internal_request(); // Collect log & stats information client_tcp_reused = !(ua_txn->is_first_transaction()); @@ -3274,13 +3277,8 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) // set the ua_txn into half close mode // only external POSTs should be subject to this logic; ruling out internal POSTs here - bool is_eligible_post_request = (t_state.method == HTTP_WKSIDX_POST); - if (is_eligible_post_request) { - NetVConnection *vc = ua_txn->get_netvc(); - if (vc) { - is_eligible_post_request &= !vc->get_is_internal_request(); - } - } + bool is_eligible_post_request = ((t_state.method == HTTP_WKSIDX_POST) && !is_internal); + if ((is_eligible_post_request || t_state.client_info.pipeline_possible == true) && c->producer->vc_type != HT_STATIC && event == VC_EVENT_WRITE_COMPLETE) { ua_txn->set_half_close_flag(true); diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index 21f707170d8..8ceb79edd7a 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -539,6 +539,7 @@ class HttpSM : public Continuation // Info about client's SSL connection. bool client_ssl_reused = false; bool client_connection_is_ssl = false; + bool is_internal = false; const char *client_protocol = "-"; const char *client_sec_protocol = "-"; const char *client_cipher_suite = "-"; diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc index 2e6d8fa5b1f..b3a2354bb0c 100644 --- a/proxy/logging/Log.cc +++ b/proxy/logging/Log.cc @@ -497,6 +497,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("cqssr", field); + field = new LogField("client_req_is_internal", "cqint", LogField::sINT, &LogAccess::marshal_client_req_is_internal, + &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("cqint", field); + field = new LogField("client_sec_protocol", "cqssv", LogField::STRING, &LogAccess::marshal_client_security_protocol, (LogField::UnmarshalFunc)&LogAccess::unmarshal_str); global_field_list.add(field, false); diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index a9a6fe7d82b..ab1f4a43b06 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -1689,37 +1689,34 @@ int LogAccess::marshal_client_req_tcp_reused(char *buf) { if (buf) { - int64_t tcp_reused; - tcp_reused = m_http_sm->client_tcp_reused; - marshal_int(buf, tcp_reused); + marshal_int(buf, m_http_sm->client_tcp_reused ? 1 : 0); } return INK_MIN_ALIGN; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - int LogAccess::marshal_client_req_is_ssl(char *buf) { if (buf) { - int64_t is_ssl; - is_ssl = m_http_sm->client_connection_is_ssl; - marshal_int(buf, is_ssl); + marshal_int(buf, m_http_sm->client_connection_is_ssl ? 1 : 0); } return INK_MIN_ALIGN; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - int LogAccess::marshal_client_req_ssl_reused(char *buf) { if (buf) { - int64_t ssl_session_reused; - ssl_session_reused = m_http_sm->client_ssl_reused; - marshal_int(buf, ssl_session_reused); + marshal_int(buf, m_http_sm->client_ssl_reused ? 1 : 0); + } + return INK_MIN_ALIGN; +} + +int +LogAccess::marshal_client_req_is_internal(char *buf) +{ + if (buf) { + marshal_int(buf, m_http_sm->is_internal ? 1 : 0); } return INK_MIN_ALIGN; } diff --git a/proxy/logging/LogAccess.h b/proxy/logging/LogAccess.h index fa221cacb4e..2436178e665 100644 --- a/proxy/logging/LogAccess.h +++ b/proxy/logging/LogAccess.h @@ -149,6 +149,7 @@ class LogAccess inkcoreapi int marshal_client_req_tcp_reused(char *); // INT inkcoreapi int marshal_client_req_is_ssl(char *); // INT inkcoreapi int marshal_client_req_ssl_reused(char *); // INT + inkcoreapi int marshal_client_req_is_internal(char *); // INT inkcoreapi int marshal_client_security_protocol(char *); // STR inkcoreapi int marshal_client_security_cipher_suite(char *); // STR inkcoreapi int marshal_client_finish_status_code(char *); // INT From 405ce9f5791746a5b91e90db3d7649aa8ba2b2e5 Mon Sep 17 00:00:00 2001 From: Valentin Gutierrez Date: Fri, 15 Mar 2019 11:49:22 +0100 Subject: [PATCH 364/526] doc: Fix ssl.server.{cert,private_key}.path examples Quotes are not needed and they actually force ATS to use a wrong path: ERROR: failed to load certificate chain from /usr/"/opt/ts/etc/ssl/certs/"/rsa.crt --- doc/admin-guide/security/index.en.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/admin-guide/security/index.en.rst b/doc/admin-guide/security/index.en.rst index 22d59b8ff01..29d9da1a8e1 100644 --- a/doc/admin-guide/security/index.en.rst +++ b/doc/admin-guide/security/index.en.rst @@ -109,8 +109,8 @@ Client/Traffic Server connections, you must do the following: #. Set the appropriate base path for your SSL certificates and private keys in :file:`records.config`. :: - CONFIG proxy.config.ssl.server.cert.path STRING "/opt/ts/etc/ssl/certs/" - CONFIG proxy.config.ssl.server.private_key.path STRING "/opt/ts/etc/ssl/keys/" + CONFIG proxy.config.ssl.server.cert.path STRING /opt/ts/etc/ssl/certs/ + CONFIG proxy.config.ssl.server.private_key.path STRING /opt/ts/etc/ssl/keys/ #. Add an entry to :file:`ssl_multicert.config` for each certificate and key which your Traffic Server system will be using to terminate SSL connections From 63f341c447822442f3f207c4bb3a676023e13b6e Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Thu, 14 Mar 2019 11:40:35 +0900 Subject: [PATCH 365/526] Fix Makefile.am to run clang-tidy --- proxy/Makefile.am | 5 ++- src/Makefile.am | 1 - src/tscore/Makefile.am | 92 +------------------------------------- src/tscpp/api/Makefile.am | 4 ++ src/tscpp/util/Makefile.am | 7 ++- src/wccp/Makefile.am | 2 + 6 files changed, 15 insertions(+), 96 deletions(-) diff --git a/proxy/Makefile.am b/proxy/Makefile.am index 0bd93d99822..1965837408d 100644 --- a/proxy/Makefile.am +++ b/proxy/Makefile.am @@ -62,7 +62,7 @@ libproxy_a_SOURCES = \ ProxyClientSession.cc \ ProxyClientSession.h \ ProxyClientTransaction.cc \ - ProxyClientTransaction \ + ProxyClientTransaction.h \ ReverseProxy.cc \ ReverseProxy.h \ StatPages.cc \ @@ -164,3 +164,6 @@ install-data-hook: chown -R $(pkgsysuser):$(pkgsysgroup) $(DESTDIR)$(pkgsysconfdir) $(DESTDIR)$(pkgdatadir);\ fi -echo " $(PACKAGE_VERSION)" > $(DESTDIR)$(pkgsysconfdir)/trafficserver-release + +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) diff --git a/src/Makefile.am b/src/Makefile.am index 8e42189a73d..0d7f07973ae 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -43,4 +43,3 @@ include traffic_logcat/Makefile.inc clang-tidy-local: $(DIST_SOURCES) $(CXX_Clang_Tidy) - $(CC_Clang_Tidy) diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index 52047452d0f..cd089720dad 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -48,162 +48,74 @@ libtscore_la_LIBADD = \ -lc libtscore_la_SOURCES = \ - Allocator.h \ AcidPtr.cc \ - AcidPtr.h \ Arena.cc \ - Arena.h \ ArgParser.cc \ - ArgParser.h \ BaseLogFile.cc \ - BaseLogFile.h \ - BufferWriter.h \ - BufferWriterForward.h \ BufferWriterFormat.cc \ ConsistentHash.cc \ - ConsistentHash.h \ ContFlags.cc \ - ContFlags.h \ CryptoHash.cc \ - CryptoHash.h \ - defalloc.h \ Diags.cc \ - Diags.h \ EventNotify.cc \ - EventNotify.h \ - Extendible.h \ fastlz.c \ - fastlz.h \ Hash.cc \ HashFNV.cc \ - HashFNV.h \ - Hash.h \ HashMD5.cc \ - HashMD5.h \ HashSip.cc \ - HashSip.h \ - History.h \ HostLookup.cc \ - HostLookup.h \ hugepages.cc \ - hugepages.h \ - I_Layout.h \ - ink_aiocb.h \ - ink_align.h \ - ink_apidefs.h \ ink_args.cc \ - ink_args.h \ ink_assert.cc \ - ink_assert.h \ - ink_atomic.h \ ink_base64.cc \ - ink_base64.h \ ink_cap.cc \ - ink_cap.h \ ink_code.cc \ - ink_code.h \ ink_defs.cc \ - ink_defs.h \ InkErrno.cc \ - InkErrno.h \ ink_error.cc \ - ink_error.h \ - ink_exception.h \ ink_file.cc \ - ink_file.h \ ink_hrtime.cc \ - ink_hrtime.h \ ink_inet.cc \ - ink_inet.h \ - ink_inout.h \ - ink_llqueue.h \ - ink_lockfile.h \ - INK_MD5.h \ ink_memory.cc \ - ink_memory.h \ ink_mutex.cc \ - ink_mutex.h \ - ink_platform.h \ ink_queue.cc \ - ink_queue.h \ ink_queue_utils.cc \ ink_rand.cc \ - ink_rand.h \ ink_res_init.cc \ ink_res_mkquery.cc \ - ink_resolver.h \ ink_resource.cc \ - ink_resource.h \ ink_rwlock.cc \ - ink_rwlock.h \ ink_sock.cc \ - ink_sock.h \ ink_sprintf.cc \ - ink_sprintf.h \ ink_stack_trace.cc \ - ink_stack_trace.h \ ink_string.cc \ ink_string++.cc \ - ink_string.h \ - ink_string++.h \ ink_sys_control.cc \ - ink_sys_control.h \ ink_syslog.cc \ - ink_syslog.h \ ink_thread.cc \ - ink_thread.h \ ink_time.cc \ - ink_time.h \ ink_uuid.cc \ - ink_uuid.h \ IpMap.cc \ IpMapConf.cc \ - IpMapConf.h \ - IpMap.h \ - I_Version.h \ - JeAllocator.h \ JeAllocator.cc \ Layout.cc \ - List.h \ llqueue.cc \ lockfile.cc \ MatcherUtils.cc \ - MatcherUtils.h \ - MemSpan.h \ MemArena.cc \ - MemArena.h \ MMH.cc \ - MMH.h \ - MT_hashtable.h \ ParseRules.cc \ - ParseRules.h \ - PriorityQueue.h \ - Ptr.h \ RbTree.cc \ - RbTree.h \ Regex.cc \ - Regex.h \ Regression.cc \ - Regression.h \ - Result.h \ runroot.cc \ - runroot.h \ signals.cc \ - signals.h \ - SimpleTokenizer.h \ SourceLocation.cc \ - SourceLocation.h \ - TestBox.h \ TextBuffer.cc \ - TextBuffer.h \ Tokenizer.cc \ - Tokenizer.h \ - Trie.h \ - TsBuffer.h \ - ts_file. h ts_file.cc \ + ts_file.cc \ Version.cc \ - X509HostnameValidator.cc \ - X509HostnameValidator.h + X509HostnameValidator.cc BufferWriterFormat.o : AM_CPPFLAGS += -Wno-char-subscripts diff --git a/src/tscpp/api/Makefile.am b/src/tscpp/api/Makefile.am index 182c4322218..1232e92d287 100644 --- a/src/tscpp/api/Makefile.am +++ b/src/tscpp/api/Makefile.am @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +include $(top_srcdir)/build/tidy.mk + lib_LTLIBRARIES = libtscppapi.la libtscppapi_la_CPPFLAGS = $(AM_CPPFLAGS) -I $(abs_top_srcdir)/include @@ -47,3 +49,5 @@ libtscppapi_la_SOURCES = \ utils.cc \ utils_internal.cc +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) diff --git a/src/tscpp/util/Makefile.am b/src/tscpp/util/Makefile.am index 8f0f4428aa8..9e972b4efb8 100644 --- a/src/tscpp/util/Makefile.am +++ b/src/tscpp/util/Makefile.am @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +include $(top_srcdir)/build/tidy.mk + check_PROGRAMS = test_tscpputil TESTS = $(check_PROGRAMS) @@ -27,9 +29,7 @@ AM_CPPFLAGS += -I$(abs_top_srcdir)/include libtscpputil_la_LDFLAGS = -no-undefined -version-info @TS_LIBTOOL_VERSION@ libtscpputil_la_SOURCES = \ - IntrusiveDList.h \ - PostScript.h \ - TextView.h TextView.cc + TextView.cc test_tscpputil_CPPFLAGS = $(AM_CPPFLAGS)\ -I$(abs_top_srcdir)/tests/include @@ -48,4 +48,3 @@ clean-local: clang-tidy-local: $(DIST_SOURCES) $(CXX_Clang_Tidy) - diff --git a/src/wccp/Makefile.am b/src/wccp/Makefile.am index 982f9f95c8a..5e57c031fea 100644 --- a/src/wccp/Makefile.am +++ b/src/wccp/Makefile.am @@ -41,3 +41,5 @@ libwccp_a_SOURCES = \ WccpStatic.cc \ WccpUtil.h +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) From 0933b163b063cfe162864ee287d567197a67f429 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Fri, 15 Mar 2019 13:49:44 +0900 Subject: [PATCH 366/526] Run clang-tidy one by one --- build/tidy.mk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build/tidy.mk b/build/tidy.mk index 16f16817724..9829c0d1aac 100644 --- a/build/tidy.mk +++ b/build/tidy.mk @@ -23,12 +23,12 @@ Clang_Tidy_Options = -fix -fix-errors -header-filter=.* Clang_Tidy_CC_Files = $(filter %.c, $(sort $(1))) Clang_Tidy_CXX_Files = $(filter %.cc, $(sort $(1))) -#clang-tidy rules. We expect these to be actions with something like -#$(DIST_SOURCES) as the dependencies.rules. Note that $DIST_SOURCES -#is not an automake API, it is an implementation detail, but it ought -#to be stable enough. +# clang-tidy rules. We expect these to be actions with something like +# $(DIST_SOURCES) as the dependencies.rules. Note that $DIST_SOURCES +# is not an automake API, it is an implementation detail, but it ought +# to be stable enough. # -#All this clearly requires GNU make. +# All this clearly requires GNU make. -CXX_Clang_Tidy = $(CLANG_TIDY) $(Clang_Tidy_Options) $(call Clang_Tidy_CXX_Files,$^) -- $(CXXCOMPILE) -x c++ -CC_Clang_Tidy = $(CLANG_TIDY) $(Clang_Tidy_Options) $(call Clang_Tidy_CC_Files,$^) -- $(COMPILE) -x c +CXX_Clang_Tidy = $(foreach tidy_target, $(call Clang_Tidy_CXX_Files,$^), $(CLANG_TIDY) $(Clang_Tidy_Options) $(tidy_target) -- $(CXXCOMPILE) -x c++;) +CC_Clang_Tidy = $(foreach tidy_target, $(call Clang_Tidy_CC_Files,$^), $(CLANG_TIDY) $(Clang_Tidy_Options) $(tidy_target) -- $(COMPILE) -x c;) From e6cb7bfaec1cf3ff00baecf2a2ab8a07addab2bf Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 12 Mar 2019 18:30:01 +0800 Subject: [PATCH 367/526] Set the block's m_water_level in Arena:free even if the block is not the last block --- src/tscore/Arena.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tscore/Arena.cc b/src/tscore/Arena.cc index 23469e08b4f..1667923d4f5 100644 --- a/src/tscore/Arena.cc +++ b/src/tscore/Arena.cc @@ -133,12 +133,12 @@ Arena::free(void *mem, size_t size) b = m_blocks; while (b->next) { + if (b->m_water_level == ((char *)mem + size)) { + b->m_water_level = (char *)mem; + return; + } b = b->next; } - - if (b->m_water_level == ((char *)mem + size)) { - b->m_water_level = (char *)mem; - } } } From 0cce83ca4aff2dccad595bc5a4d45b351e5e24dc Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Mon, 4 Mar 2019 19:37:11 +0000 Subject: [PATCH 368/526] add --with-jansson and --with-cjose options, document sample commands for building and configuring both locally --- build/cjose.m4 | 47 ++++++++++++++++++++++ build/hiredis.m4 | 2 +- build/jansson.m4 | 47 ++++++++++++++++++++++ configure.ac | 26 ++++-------- plugins/experimental/uri_signing/README.md | 39 ++++++++++++++++++ 5 files changed, 141 insertions(+), 20 deletions(-) create mode 100644 build/cjose.m4 create mode 100644 build/jansson.m4 diff --git a/build/cjose.m4 b/build/cjose.m4 new file mode 100644 index 00000000000..6bead14cd3e --- /dev/null +++ b/build/cjose.m4 @@ -0,0 +1,47 @@ +dnl -------------------------------------------------------- -*- autoconf -*- +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. + +dnl +dnl cjose.m4: Trafficserver's cjose autoconf macros +dnl + +dnl +dnl TS_CHECK_CJOSE: look for cjose libraries and headers +dnl + +AC_DEFUN([TS_CHECK_CJOSE], [ +AC_MSG_CHECKING([for --with-cjose]) + AC_ARG_WITH( + [cjose], + [AS_HELP_STRING([--with-cjose=DIR], [use a specific cjose library])], + [ LDFLAGS="$LDFLAGS -L$with_cjose/lib"; + CFLAGS="$CFLAGS -I$with_cjose/include/"; + CPPFLAGS="$CPPFLAGS -I$with_cjose/include/"; + AC_MSG_RESULT([$with_cjose]) + ], + [ AC_MSG_RESULT([no])] + ) + + AC_CHECK_HEADERS([cjose/cjose.h], [ + AC_MSG_CHECKING([whether cjose is dynamic]) + TS_LINK_WITH_FLAGS_IFELSE([-fPIC -lcjose -ljansson -lcrypto],[AC_LANG_PROGRAM( + [#include ], + [(void) cjose_jws_import("", 0, NULL);])], + [AC_MSG_RESULT([yes]); LIBCJOSE=-lcjose], + [AC_MSG_RESULT([no]); LIBCJOSE=-l:libcjose.a]) + ], + [LIBCJOSE=]) +]) diff --git a/build/hiredis.m4 b/build/hiredis.m4 index ffbb8c96860..e49ad7925ce 100644 --- a/build/hiredis.m4 +++ b/build/hiredis.m4 @@ -33,7 +33,7 @@ AC_ARG_WITH(hiredis, [AC_HELP_STRING([--with-hiredis=DIR],[use a specific hiredi if test "$withval" != "no"; then case "$withval" in *":"*) - hidredis_include="`echo $withval |sed -e 's/:.*$//'`" + hiredis_include="`echo $withval |sed -e 's/:.*$//'`" hiredis_ldflags="`echo $withval |sed -e 's/^.*://'`" AC_MSG_CHECKING(checking for hiredis includes in $hiredis_include libs in $hiredis_ldflags ) ;; diff --git a/build/jansson.m4 b/build/jansson.m4 new file mode 100644 index 00000000000..07987c5acf7 --- /dev/null +++ b/build/jansson.m4 @@ -0,0 +1,47 @@ +dnl -------------------------------------------------------- -*- autoconf -*- +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. + +dnl +dnl jansson.m4: Trafficserver's jansson autoconf macros +dnl + +dnl +dnl TS_CHECK_JANSSON: look for jansson libraries and headers +dnl + +AC_DEFUN([TS_CHECK_JANSSON], [ + AC_MSG_CHECKING([for --with-jansson]) + AC_ARG_WITH( + [jansson], + [AS_HELP_STRING([--with-jansson], [use a specific jansson library])], + [ LDFLAGS="$LDFLAGS -L$with_jansson/lib"; + CFLAGS="$CFLAGS -I$with_jansson/include/"; + CPPFLAGS="$CPPFLAGS -I$with_jansson/include/"; + AC_MSG_RESULT([$with_jansson]) + ], + [ AC_MSG_RESULT([no])] + ) + + AC_CHECK_HEADERS([jansson.h], [ + AC_MSG_CHECKING([whether jansson is dynamic]) + TS_LINK_WITH_FLAGS_IFELSE([-fPIC -ljansson],[AC_LANG_PROGRAM( + [#include ], + [(void) json_object();])], + [AC_MSG_RESULT([yes]); LIBJANSSON=-ljansson], + [AC_MSG_RESULT([no]); LIBJANSSON=-l:libjansson.a]) + ], + [LIBJANSSON=]) +]) diff --git a/configure.ac b/configure.ac index 7ab2ccdf18a..75aebc581ff 100644 --- a/configure.ac +++ b/configure.ac @@ -1304,27 +1304,15 @@ TS_CHECK_LUAJIT # Enable experimental/uri_singing plugin # This is here, instead of above, because it needs to know if PCRE is available. # -AC_CHECK_HEADERS([jansson.h], [ - AC_MSG_CHECKING([whether jansson is dynamic]) - TS_LINK_WITH_FLAGS_IFELSE([-fPIC -ljansson],[AC_LANG_PROGRAM( - [#include ], - [(void) json_object();])], - [AC_MSG_RESULT([yes]); LIBJANSSON=-ljansson], - [AC_MSG_RESULT([no]); LIBJANSSON=-l:libjansson.a]) - ], - [LIBJANSSON=]) - -AC_CHECK_HEADERS([cjose/cjose.h], [ - AC_MSG_CHECKING([whether cjose is dynamic]) - TS_LINK_WITH_FLAGS_IFELSE([-fPIC -lcjose],[AC_LANG_PROGRAM( - [#include ], - [(void) cjose_jws_import("", 0, NULL);])], - [AC_MSG_RESULT([yes]); LIBCJOSE=-lcjose], - [AC_MSG_RESULT([no]); LIBCJOSE=-l:libcjose.a]) - ], - [LIBCJOSE=]) + +#### Check for optional jansson library (uri_signing) +TS_CHECK_JANSSON + AC_CHECK_LIB([crypto],[HMAC],[has_libcrypto=1],[has_libcrypto=0]) +#### Check for optional cjose library (uri_signing) +TS_CHECK_CJOSE + AM_CONDITIONAL([BUILD_URI_SIGNING_PLUGIN], [test ! -z "${LIBCJOSE}" -a ! -z "${LIBJANSSON}" -a "x${enable_pcre}" = "xyes" -a "x${has_libcrypto}" = "x1"]) AC_SUBST([LIBCJOSE]) AC_SUBST([LIBJANSSON]) diff --git a/plugins/experimental/uri_signing/README.md b/plugins/experimental/uri_signing/README.md index 5dc8789465b..8180586bf9a 100644 --- a/plugins/experimental/uri_signing/README.md +++ b/plugins/experimental/uri_signing/README.md @@ -185,3 +185,42 @@ plugin. If you would like to statically link them, you will need to ensure that they are compiled with the `-fPIC` flag in their CFLAGs. If the archives have PIC, the build scripts will automatically statically link them. + +Here are some sample commands for building jansson, cjose and trafficserver +locally using static linking. This assumes all source is under ${HOME}/git. + +### Sample + +If using local jansson: + + cd ${HOME}/git + git clone https://github.com/akheron/jansson.git + cd jansson + autoreconf -i + ./configure --disable-shared CC="gcc -fpic" + make -j`nproc` + + # Needed for ATS configure + ln -s src/.libs lib + ln -s src include + +If using local cjose: + + cd ${HOME}/git + git clone https://github.com/cisco/cjose.git + cd cjose + autoreconf -i + ./configure --with-jansson=${HOME}/git/jansson --disable-shared CC="gcc -fpic" + make -j`nproc` + + # Needed for ATS configure + ln -s src/.libs lib + +ATS: + + cd ${HOME}/git/ + git clone https://github.com/apache/trafficserver.git + cd trafficserver + autoreconf -i + ./configure --enable-experimental-plugins --with-jansson=${HOME}/git/jansson --with-cjose=${HOME}/git/cjose + make -j`nproc` From ede4df1d9f53ce650984a2de98a99a24366a57e3 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 19 Mar 2019 08:32:51 +0900 Subject: [PATCH 369/526] Ignore check programs --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 0aeedac9e89..0c7f2854e9a 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ iocore/eventsystem/test_MIOBufferWriter iocore/hostdb/test_RefCountCache proxy/hdrs/test_mime +proxy/hdrs/test_proxy_hdrs proxy/http/test_proxy_http proxy/http2/test_Huffmancode proxy/http2/test_Http2DependencyTree @@ -108,6 +109,7 @@ proxy/logging/test_LogUtils proxy/logging/test_LogUtils2 plugins/header_rewrite/header_rewrite_test +plugins/experimental/cookie_remap/test_cookiejar plugins/experimental/esi/*_test plugins/experimental/slice/test_* plugins/experimental/sslheaders/test_sslheaders From 307596508b76fbfb7bb928b111bd2c86523161a0 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Thu, 14 Mar 2019 14:51:41 -0700 Subject: [PATCH 370/526] Updated traffic_via to have consistency between output and error messages. Also, updated the test to be able to run from the command line and give some output. --- src/traffic_via/test_traffic_via | 7 ++++++- src/traffic_via/tests/[u c s f p eS;tNc p s ] | 2 +- src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] | 2 +- src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] | 2 +- src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] | 2 +- src/traffic_via/tests/[uScRs f p eN;t cCHp s ] | 2 +- src/traffic_via/tests/long rubbish via code2 | 2 +- src/traffic_via/tests/rubbish | 2 +- src/traffic_via/tests/short | 2 +- src/traffic_via/traffic_via.cc | 3 ++- 10 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/traffic_via/test_traffic_via b/src/traffic_via/test_traffic_via index fb9fb0347b6..9b3995f659b 100755 --- a/src/traffic_via/test_traffic_via +++ b/src/traffic_via/test_traffic_via @@ -20,11 +20,16 @@ set -e # exit on error TMPDIR=${TMPDIR:-/tmp} tmpfile=$(mktemp "$TMPDIR/via.XXXXXX") -srcdir=$(cd $srcdir && pwd) +if [ ! -z "$srcdir"]; then + srcdir=$(cd $srcdir && pwd) +else + srcdir=$(pwd) +fi find $srcdir/tests -type f | while read f ; do name=$(basename "$f") + echo "testing $name" ./traffic_via "$name" > "$tmpfile" 2>&1 || true diff -u "$tmpfile" "$srcdir/tests/$name" done diff --git a/src/traffic_via/tests/[u c s f p eS;tNc p s ] b/src/traffic_via/tests/[u c s f p eS;tNc p s ] index 95cf7a7d9d5..e662dc0e814 100644 --- a/src/traffic_via/tests/[u c s f p eS;tNc p s ] +++ b/src/traffic_via/tests/[u c s f p eS;tNc p s ] @@ -1,4 +1,4 @@ -Via header is [u c s f p eS;tNc p s ], Length is 24 +Via header is [u c s f p eS;tNc p s ], Length is 22 Via Header Details: Request headers received from client :unknown Result of Traffic Server cache lookup for URL :no cache lookup diff --git a/src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] b/src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] index a5f3238c4b0..0755883b90f 100644 --- a/src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] +++ b/src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] @@ -1,4 +1,4 @@ -Via header is [uIcRs f p eN;t cCHp s ], Length is 24 +Via header is [uIcRs f p eN;t cCHp s ], Length is 22 Via Header Details: Request headers received from client :IMS Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") diff --git a/src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] b/src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] index 6af0055ed2e..6ecfc64517a 100644 --- a/src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] +++ b/src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] @@ -1,4 +1,4 @@ -Via header is [uIcRs f p eN;t cCNp s ], Length is 24 +Via header is [uIcRs f p eN;t cCNp s ], Length is 22 Via Header Details: Request headers received from client :IMS Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") diff --git a/src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] b/src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] index 1e494c63175..6ee096a1889 100644 --- a/src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] +++ b/src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] @@ -1,4 +1,4 @@ -Via header is [uScMsSf pSeN;t cCMp sS], Length is 24 +Via header is [uScMsSf pSeN;t cCMp sS], Length is 22 Via Header Details: Request headers received from client :simple request (not conditional) Result of Traffic Server cache lookup for URL :miss (a cache "MISS") diff --git a/src/traffic_via/tests/[uScRs f p eN;t cCHp s ] b/src/traffic_via/tests/[uScRs f p eN;t cCHp s ] index d7d394c6e59..ea96fdb54fa 100644 --- a/src/traffic_via/tests/[uScRs f p eN;t cCHp s ] +++ b/src/traffic_via/tests/[uScRs f p eN;t cCHp s ] @@ -1,4 +1,4 @@ -Via header is [uScRs f p eN;t cCHp s ], Length is 24 +Via header is [uScRs f p eN;t cCHp s ], Length is 22 Via Header Details: Request headers received from client :simple request (not conditional) Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") diff --git a/src/traffic_via/tests/long rubbish via code2 b/src/traffic_via/tests/long rubbish via code2 index d3abd3d95ba..37366829fbe 100644 --- a/src/traffic_via/tests/long rubbish via code2 +++ b/src/traffic_via/tests/long rubbish via code2 @@ -12,6 +12,6 @@ traffic_via: Invalid VIA header character: i traffic_via: Invalid VIA header character: a traffic_via: Invalid VIA header character: o traffic_via: Invalid VIA header character: d -Via header is long rubbish via code2, Length is 22 +Via header is [long rubbish via code2], Length is 22 Via Header Details: Error codes (if any) :Invalid sequence diff --git a/src/traffic_via/tests/rubbish b/src/traffic_via/tests/rubbish index 348f372e606..599670db921 100644 --- a/src/traffic_via/tests/rubbish +++ b/src/traffic_via/tests/rubbish @@ -1,4 +1,4 @@ -Via header is rubbish, Length is 7 +Via header is [rubbish], Length is 7 Invalid VIA header. VIA header length should be 6 or 22 characters Valid via header format is [ucsfpe:tcps] diff --git a/src/traffic_via/tests/short b/src/traffic_via/tests/short index c0f098ce3bc..76d85bfb117 100644 --- a/src/traffic_via/tests/short +++ b/src/traffic_via/tests/short @@ -2,5 +2,5 @@ traffic_via: Invalid VIA header character: h traffic_via: Invalid VIA header character: o traffic_via: Invalid VIA header character: r traffic_via: Invalid VIA header character: t -Via header is short, Length is 5 +Via header is [short], Length is 5 Via Header Details: diff --git a/src/traffic_via/traffic_via.cc b/src/traffic_via/traffic_via.cc index b755f2c6916..cab50985205 100644 --- a/src/traffic_via/traffic_via.cc +++ b/src/traffic_via/traffic_via.cc @@ -224,7 +224,6 @@ decodeViaHeader(const char *str) memcpy(Via, str, viaHdrLength); Via[viaHdrLength] = '\0'; // null terminate - printf("Via header is %s, Length is %zu\n", Via, viaHdrLength); // Via header inside square brackets if (Via[0] == '[' && Via[viaHdrLength - 1] == ']') { @@ -233,6 +232,8 @@ decodeViaHeader(const char *str) Via[viaHdrLength] = '\0'; // null terminate the string after trimming } + printf("Via header is [%s], Length is %zu\n", Via, viaHdrLength); + if (viaHdrLength == 5) { Via = strcat(Via, " "); // Add one space character before decoding via header ++viaHdrLength; From bf688dec675294aa2742046f1d5b64723b7cb9ba Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 19 Mar 2019 09:21:35 +0900 Subject: [PATCH 371/526] Do not run clang-tidy on lib/yamlcpp/ --- lib/yamlcpp/Makefile.am | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/yamlcpp/Makefile.am b/lib/yamlcpp/Makefile.am index 05f441af24d..a451416b7c8 100644 --- a/lib/yamlcpp/Makefile.am +++ b/lib/yamlcpp/Makefile.am @@ -17,8 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -include $(top_srcdir)/build/tidy.mk - AM_CPPFLAGS += \ -I$(abs_top_srcdir)/lib/yamlcpp/include @@ -52,6 +50,3 @@ src/simplekey.cpp \ src/singledocparser.cpp \ src/stream.cpp \ src/tag.cpp - -clang-tidy-local: $(DIST_SOURCES) - $(CXX_Clang_Tidy) From e9ace8f9062c167911a9f19bf89da5f0e68bd073 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 19 Mar 2019 08:29:06 +0900 Subject: [PATCH 372/526] Exclude C header files from modernize-deprecated-headers --- include/ts/apidefs.h.in | 2 +- include/tscore/ink_defs.h | 6 +++--- include/tscore/ink_platform.h | 18 +++++++++--------- plugins/experimental/cookie_remap/hash.h | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index e546a562f77..688e313dbee 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -43,7 +43,7 @@ */ #include -#include +#include // NOLINT(modernize-deprecated-headers) #include #include diff --git a/include/tscore/ink_defs.h b/include/tscore/ink_defs.h index 20c80b53487..251fcbf626a 100644 --- a/include/tscore/ink_defs.h +++ b/include/tscore/ink_defs.h @@ -24,17 +24,17 @@ #pragma once #include "tscore/ink_config.h" -#include +#include // NOLINT(modernize-deprecated-headers) #include #ifdef HAVE_STDINT_H -#include +#include // NOLINT(modernize-deprecated-headers) #else // TODO: Add "standard" int types? #endif #ifdef HAVE_INTTYPES_H -#include +#include // NOLINT(modernize-deprecated-headers) #else // TODO: add PRI*64 stuff? #endif diff --git a/include/tscore/ink_platform.h b/include/tscore/ink_platform.h index 321bfdf2801..a7e0e8ae9b9 100644 --- a/include/tscore/ink_platform.h +++ b/include/tscore/ink_platform.h @@ -33,14 +33,14 @@ #endif #include -#include -#include -#include +#include // NOLINT(modernize-deprecated-headers) +#include // NOLINT(modernize-deprecated-headers) +#include // NOLINT(modernize-deprecated-headers) #include #include #include #include -#include +#include // NOLINT(modernize-deprecated-headers) #include #include #include @@ -61,11 +61,11 @@ struct ifafilt; #include #ifdef HAVE_STDLIB_H -#include +#include // NOLINT(modernize-deprecated-headers) #endif -#include +#include // NOLINT(modernize-deprecated-headers) #ifdef HAVE_STRING_H -#include +#include // NOLINT(modernize-deprecated-headers) #endif #ifdef HAVE_STRINGS_H #include @@ -107,7 +107,7 @@ struct ifafilt; #include #endif -#include +#include // NOLINT(modernize-deprecated-headers) #ifdef HAVE_SIGINFO_H #include #endif @@ -170,7 +170,7 @@ typedef unsigned int in_addr_t; #endif #ifdef HAVE_FLOAT_H -#include +#include // NOLINT(modernize-deprecated-headers) #endif #ifdef HAVE_SYS_SYSMACROS_H diff --git a/plugins/experimental/cookie_remap/hash.h b/plugins/experimental/cookie_remap/hash.h index daeffc2df30..6d6c94131bb 100644 --- a/plugins/experimental/cookie_remap/hash.h +++ b/plugins/experimental/cookie_remap/hash.h @@ -19,9 +19,9 @@ #ifndef _CKREMAP_HASH_H_ #define _CKREMAP_HASH_H_ -#include +#include // NOLINT(modernize-deprecated-headers) #include -#include +#include // NOLINT(modernize-deprecated-headers) #ifdef __cplusplus extern "C" { From 8e96e0bed137e6e566dee257cc79beb22864a931 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 19 Mar 2019 10:20:59 +0900 Subject: [PATCH 373/526] Suppress modernize-use-nullptr on ink_thread --- include/tscore/ink_thread.h | 3 +++ iocore/eventsystem/I_Thread.h | 1 + 2 files changed, 4 insertions(+) diff --git a/include/tscore/ink_thread.h b/include/tscore/ink_thread.h index 5382c9a2a5a..7a08f6205f1 100644 --- a/include/tscore/ink_thread.h +++ b/include/tscore/ink_thread.h @@ -192,6 +192,9 @@ ink_thread_self() static inline ink_thread ink_thread_null() { + // The implementation of ink_thread (the alias of pthread_t) is different on platforms + // - e.g. `struct pthread *` on Unix and `unsigned long int` on Linux + // NOLINTNEXTLINE(modernize-use-nullptr) return (ink_thread)0; } diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index 0545d6cde9a..e2c46b3391c 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -99,6 +99,7 @@ class Thread processors and you should not modify it directly. */ + // NOLINTNEXTLINE(modernize-use-nullptr) ink_thread tid = 0; /** From d77cd73166c1b27cde61ffb3c8a1545c437fad83 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 19 Mar 2019 10:58:35 +0900 Subject: [PATCH 374/526] Ran clang-tidy "proxy/logging/LogField.h" is changed by hand to follow changes made by performance-unnecessary-value-param to "proxy/logging/LogAccess.h". --- include/tscore/EventNotify.h | 10 ++-- include/tscore/Extendible.h | 4 +- include/tscore/Hash.h | 12 ++--- include/tscore/HashFNV.h | 16 +++---- include/tscore/HashMD5.h | 10 ++-- include/tscore/HashSip.h | 8 ++-- include/tscore/List.h | 2 +- include/tscore/MT_hashtable.h | 8 ++-- include/tscore/hugepages.h | 4 +- include/tscore/ink_error.h | 4 +- include/tscore/ink_llqueue.h | 2 +- include/tscore/ink_lockfile.h | 8 ++-- include/tscore/ink_memory.h | 2 +- include/tscpp/api/Async.h | 5 +- include/tscpp/api/Headers.h | 2 +- include/wccp/Wccp.h | 12 ++--- iocore/aio/I_AIO.h | 2 +- iocore/cache/P_CacheHosting.h | 2 +- iocore/cache/P_CacheInternal.h | 4 +- iocore/dns/P_SplitDNSProcessor.h | 8 ++-- iocore/hostdb/HostDB.cc | 2 +- iocore/hostdb/P_RefCountCache.h | 6 +-- iocore/hostdb/P_RefCountCacheSerializer.h | 7 +-- iocore/net/I_SessionAccept.h | 2 +- iocore/net/I_UDPConnection.h | 2 +- iocore/net/P_SNIActionPerformer.h | 2 +- iocore/net/P_SSLConfig.h | 2 +- iocore/net/P_SSLNetProcessor.h | 2 +- iocore/net/P_SSLNetVConnection.h | 4 +- iocore/net/P_SSLSNI.h | 4 +- iocore/net/P_SSLUtils.h | 2 +- iocore/net/P_UDPConnection.h | 4 +- iocore/net/P_UDPNet.h | 2 +- iocore/net/P_UDPPacket.h | 4 +- iocore/net/P_UnixPollDescriptor.h | 2 +- iocore/net/ProxyProtocol.h | 2 +- iocore/net/SSLSNIConfig.cc | 10 ++-- iocore/net/SSLSessionCache.h | 2 +- iocore/utils/Machine.cc | 4 +- lib/records/I_RecCore.h | 2 +- lib/records/I_RecProcess.h | 2 +- lib/tsconfig/Errata.h | 2 +- lib/tsconfig/TsValue.h | 3 +- mgmt/utils/MgmtSocket.h | 2 +- plugins/background_fetch/rules.cc | 2 +- plugins/cachekey/cachekey.cc | 3 +- plugins/cachekey/configs.cc | 4 +- plugins/cachekey/pattern.h | 2 +- plugins/esi/lib/HandlerManager.h | 2 +- .../access_control/access_control.h | 2 +- plugins/experimental/access_control/config.cc | 30 ++++++------ .../experimental/access_control/headers.cc | 4 +- plugins/experimental/access_control/pattern.h | 10 ++-- plugins/experimental/access_control/plugin.cc | 4 +- plugins/experimental/access_control/utils.cc | 6 +-- plugins/experimental/access_control/utils.h | 2 +- .../experimental/cookie_remap/cookie_remap.cc | 48 +++++++++---------- .../experimental/cookie_remap/cookiejar.cc | 2 +- plugins/experimental/cookie_remap/cookiejar.h | 2 +- .../cookie_remap/test_cookiejar.cc | 2 +- plugins/experimental/inliner/cache-handler.h | 12 ++--- plugins/experimental/inliner/cache.h | 4 +- plugins/experimental/inliner/chunk-decoder.h | 6 +-- plugins/experimental/inliner/fetcher.h | 6 +-- plugins/experimental/inliner/html-parser.h | 8 ++-- .../experimental/inliner/inliner-handler.h | 6 +-- plugins/experimental/inliner/png.h | 4 +- plugins/experimental/inliner/ts.h | 24 +++++----- .../ja3_fingerprint/ja3_fingerprint.cc | 7 +-- plugins/experimental/magick/magick.cc | 22 ++++----- plugins/experimental/memcache/tsmemcache.cc | 2 +- .../experimental/multiplexer/chunk-decoder.h | 4 +- plugins/experimental/multiplexer/fetcher.h | 6 +-- plugins/experimental/multiplexer/ts.h | 2 +- .../experimental/mysql_remap/lib/iniparser.h | 6 +-- .../experimental/mysql_remap/mysql_remap.cc | 2 +- plugins/experimental/prefetch/common.cc | 4 +- plugins/experimental/prefetch/configs.cc | 26 +++++----- plugins/experimental/prefetch/fetch.cc | 4 +- plugins/experimental/prefetch/fetch_policy.cc | 2 +- plugins/experimental/prefetch/fetch_policy.h | 2 +- .../experimental/prefetch/fetch_policy_lru.h | 14 +++--- .../prefetch/fetch_policy_simple.h | 14 +++--- plugins/experimental/prefetch/headers.cc | 4 +- plugins/experimental/prefetch/pattern.cc | 16 +++---- plugins/experimental/prefetch/pattern.h | 2 +- plugins/experimental/prefetch/plugin.cc | 6 +-- plugins/experimental/slice/Config.cc | 2 +- plugins/experimental/slice/Data.cc | 2 +- .../ssl_session_reuse/src/ats_ssl_plugin.cc | 2 +- .../ssl_session_reuse/src/config.cc | 4 +- .../ssl_session_reuse/src/openssl_utils.cc | 6 +-- .../ssl_session_reuse/src/publish.cc | 2 +- .../ssl_session_reuse/src/session_process.cc | 8 ++-- .../ssl_session_reuse/src/session_process.h | 4 +- .../ssl_session_reuse/src/ssl_init.cc | 2 +- .../ssl_session_reuse/src/ssl_key_utils.cc | 8 ++-- .../ssl_session_reuse/src/subscriber.cc | 2 +- plugins/experimental/sslheaders/util.cc | 6 +-- .../experimental/traffic_dump/traffic_dump.cc | 4 +- plugins/header_rewrite/condition.h | 2 +- plugins/header_rewrite/header_rewrite.cc | 10 ++-- plugins/header_rewrite/matcher.h | 2 +- plugins/header_rewrite/value.cc | 7 ++- plugins/header_rewrite/value.h | 6 +-- proxy/HostStatus.h | 2 +- proxy/RegressionSM.cc | 4 +- proxy/StatPages.h | 18 +++---- proxy/TransformInternal.h | 18 +++---- proxy/hdrs/HdrHeap.cc | 24 +++++----- proxy/http/HttpConfig.cc | 2 +- proxy/http/HttpTransact.h | 2 +- proxy/http/HttpTunnel.cc | 8 ++-- proxy/http/HttpUpdateSM.h | 8 ++-- proxy/http/remap/AclFiltering.h | 4 +- proxy/http/remap/UrlRewrite.cc | 8 ++-- proxy/http2/Http2DependencyTree.h | 2 +- proxy/http2/RegressionHPACK.cc | 2 +- proxy/logging/LogAccess.cc | 12 ++--- proxy/logging/LogAccess.h | 18 +++---- proxy/logging/LogBuffer.h | 4 +- proxy/logging/LogField.h | 4 +- src/traffic_cache_tool/CacheDefs.h | 2 +- src/traffic_cache_tool/CacheTool.cc | 6 +-- src/traffic_layout/file_system.cc | 8 ++-- src/traffic_logcat/logcat.cc | 8 ++-- src/traffic_logstats/logstats.cc | 2 +- src/traffic_server/CoreUtils.cc | 6 +-- src/traffic_server/CoreUtils.h | 8 ++-- src/traffic_server/InkAPI.cc | 36 +++++++------- src/traffic_server/InkAPITest.cc | 4 +- src/traffic_top/stats.h | 2 +- src/traffic_wccp/wccp_client.cc | 4 +- src/tscore/ArgParser.cc | 3 +- src/tscore/ink_hrtime.cc | 2 +- src/tscore/runroot.cc | 6 +-- .../unit_tests/test_IntrusiveHashMap.cc | 4 +- src/tscore/unit_tests/test_PriorityQueue.cc | 2 +- src/tscpp/api/TransformationPlugin.cc | 2 +- 139 files changed, 435 insertions(+), 430 deletions(-) diff --git a/include/tscore/EventNotify.h b/include/tscore/EventNotify.h index 2ac068923de..864c8093254 100644 --- a/include/tscore/EventNotify.h +++ b/include/tscore/EventNotify.h @@ -36,12 +36,12 @@ class EventNotify { public: EventNotify(); - void signal(void); - int wait(void); + void signal(); + int wait(); int timedwait(int timeout); // milliseconds - void lock(void); - bool trylock(void); - void unlock(void); + void lock(); + bool trylock(); + void unlock(); ~EventNotify(); private: diff --git a/include/tscore/Extendible.h b/include/tscore/Extendible.h index c6a1920d835..e4f46826dd7 100644 --- a/include/tscore/Extendible.h +++ b/include/tscore/Extendible.h @@ -30,7 +30,7 @@ */ #pragma once -#include "stdint.h" +#include #include #include #include @@ -60,7 +60,7 @@ enum AccessEnum { ATOMIC, BIT, STATIC, ACIDPTR, DIRECT, C_API, NUM_ACCESS_TYPES inline bool & areStaticsFrozen() { - static bool frozen = 0; + static bool frozen = false; return frozen; } diff --git a/include/tscore/Hash.h b/include/tscore/Hash.h index 68879a93951..08e438095a6 100644 --- a/include/tscore/Hash.h +++ b/include/tscore/Hash.h @@ -27,8 +27,8 @@ struct ATSHashBase { virtual void update(const void *, size_t) = 0; - virtual void final(void) = 0; - virtual void clear(void) = 0; + virtual void final() = 0; + virtual void clear() = 0; virtual ~ATSHashBase(); }; @@ -49,17 +49,17 @@ struct ATSHash : ATSHashBase { } }; - virtual const void *get(void) const = 0; - virtual size_t size(void) const = 0; + virtual const void *get() const = 0; + virtual size_t size() const = 0; virtual bool operator==(const ATSHash &) const; }; struct ATSHash32 : ATSHashBase { - virtual uint32_t get(void) const = 0; + virtual uint32_t get() const = 0; virtual bool operator==(const ATSHash32 &) const; }; struct ATSHash64 : ATSHashBase { - virtual uint64_t get(void) const = 0; + virtual uint64_t get() const = 0; virtual bool operator==(const ATSHash64 &) const; }; diff --git a/include/tscore/HashFNV.h b/include/tscore/HashFNV.h index e113d0cd012..ff92edcd7a0 100644 --- a/include/tscore/HashFNV.h +++ b/include/tscore/HashFNV.h @@ -31,7 +31,7 @@ #include struct ATSHash32FNV1a : ATSHash32 { - ATSHash32FNV1a(void); + ATSHash32FNV1a(); template void update(const void *data, size_t len, Transform xfrm); void @@ -40,9 +40,9 @@ struct ATSHash32FNV1a : ATSHash32 { update(data, len, ATSHash::nullxfrm()); } - void final(void) override; - uint32_t get(void) const override; - void clear(void) override; + void final() override; + uint32_t get() const override; + void clear() override; private: uint32_t hval; @@ -62,7 +62,7 @@ ATSHash32FNV1a::update(const void *data, size_t len, Transform xfrm) } struct ATSHash64FNV1a : ATSHash64 { - ATSHash64FNV1a(void); + ATSHash64FNV1a(); template void update(const void *data, size_t len, Transform xfrm); void @@ -71,9 +71,9 @@ struct ATSHash64FNV1a : ATSHash64 { update(data, len, ATSHash::nullxfrm()); } - void final(void) override; - uint64_t get(void) const override; - void clear(void) override; + void final() override; + uint64_t get() const override; + void clear() override; private: uint64_t hval; diff --git a/include/tscore/HashMD5.h b/include/tscore/HashMD5.h index bad5a90cc80..1447786dc47 100644 --- a/include/tscore/HashMD5.h +++ b/include/tscore/HashMD5.h @@ -25,12 +25,12 @@ #include struct ATSHashMD5 : ATSHash { - ATSHashMD5(void); + ATSHashMD5(); void update(const void *data, size_t len) override; - void final(void) override; - const void *get(void) const override; - size_t size(void) const override; - void clear(void) override; + void final() override; + const void *get() const override; + size_t size() const override; + void clear() override; ~ATSHashMD5() override; private: diff --git a/include/tscore/HashSip.h b/include/tscore/HashSip.h index 837289848dc..30d195b49f3 100644 --- a/include/tscore/HashSip.h +++ b/include/tscore/HashSip.h @@ -32,13 +32,13 @@ */ struct ATSHash64Sip24 : ATSHash64 { - ATSHash64Sip24(void); + ATSHash64Sip24(); ATSHash64Sip24(const unsigned char key[16]); ATSHash64Sip24(std::uint64_t key0, std::uint64_t key1); void update(const void *data, std::size_t len) override; - void final(void) override; - std::uint64_t get(void) const override; - void clear(void) override; + void final() override; + std::uint64_t get() const override; + void clear() override; private: unsigned char block_buffer[8] = {0}; diff --git a/include/tscore/List.h b/include/tscore/List.h index cc6278165d9..aeea32b295a 100644 --- a/include/tscore/List.h +++ b/include/tscore/List.h @@ -507,7 +507,7 @@ template struct SortableQueue : publi template struct CountQueue : public Queue { int size; - inline CountQueue(void) : size(0) {} + inline CountQueue() : size(0) {} inline void push(C *e); inline C *pop(); inline void enqueue(C *e); diff --git a/include/tscore/MT_hashtable.h b/include/tscore/MT_hashtable.h index 2c24225e586..f3b7c916a94 100644 --- a/include/tscore/MT_hashtable.h +++ b/include/tscore/MT_hashtable.h @@ -77,7 +77,7 @@ template class HashTableIteratorState template class IMTHashTable { public: - IMTHashTable(int size, bool (*gc_func)(data_t) = NULL, void (*pre_gc_func)(void) = nullptr) + IMTHashTable(int size, bool (*gc_func)(data_t) = NULL, void (*pre_gc_func)() = nullptr) { m_gc_func = gc_func; m_pre_gc_func = pre_gc_func; @@ -136,7 +136,7 @@ template class IMTHashTable data_t remove_entry(HashTableIteratorState *s); void - GC(void) + GC() { if (m_gc_func == NULL) { return; @@ -195,7 +195,7 @@ template class IMTHashTable int cur_size; int bucket_num; bool (*m_gc_func)(data_t); - void (*m_pre_gc_func)(void); + void (*m_pre_gc_func)(); private: IMTHashTable(); @@ -336,7 +336,7 @@ IMTHashTable::remove_entry(HashTableIteratorState template class MTHashTable { public: - MTHashTable(int size, bool (*gc_func)(data_t) = NULL, void (*pre_gc_func)(void) = nullptr) + MTHashTable(int size, bool (*gc_func)(data_t) = NULL, void (*pre_gc_func)() = nullptr) { for (int i = 0; i < MT_HASHTABLE_PARTITIONS; i++) { locks[i] = new_ProxyMutex(); diff --git a/include/tscore/hugepages.h b/include/tscore/hugepages.h index cb7e95b29b3..9735ee52f8a 100644 --- a/include/tscore/hugepages.h +++ b/include/tscore/hugepages.h @@ -22,8 +22,8 @@ #include -size_t ats_hugepage_size(void); -bool ats_hugepage_enabled(void); +size_t ats_hugepage_size(); +bool ats_hugepage_enabled(); void ats_hugepage_init(int); void *ats_alloc_hugepage(size_t); bool ats_free_hugepage(void *, size_t); diff --git a/include/tscore/ink_error.h b/include/tscore/ink_error.h index 39dbfacbc7c..0e4b4bbfab4 100644 --- a/include/tscore/ink_error.h +++ b/include/tscore/ink_error.h @@ -31,8 +31,8 @@ #pragma once -#include -#include +#include +#include #include "tscore/ink_platform.h" #include "tscore/ink_apidefs.h" diff --git a/include/tscore/ink_llqueue.h b/include/tscore/ink_llqueue.h index 4f86901f6c8..0241da521ba 100644 --- a/include/tscore/ink_llqueue.h +++ b/include/tscore/ink_llqueue.h @@ -43,7 +43,7 @@ typedef struct llq_s { ink_semaphore sema; } LLQ; -LLQ *create_queue(void); +LLQ *create_queue(); int enqueue(LLQ *q, void *data); void *dequeue(LLQ *q); bool queue_is_empty(LLQ *q); diff --git a/include/tscore/ink_lockfile.h b/include/tscore/ink_lockfile.h index dea4d97dae6..d3d6878f8e1 100644 --- a/include/tscore/ink_lockfile.h +++ b/include/tscore/ink_lockfile.h @@ -33,10 +33,10 @@ class Lockfile { public: - Lockfile(void) : fd(0) { fname[0] = '\0'; } + Lockfile() : fd(0) { fname[0] = '\0'; } // coverity[uninit_member] Lockfile(const char *filename) : fd(0) { ink_strlcpy(fname, filename, sizeof(fname)); } - ~Lockfile(void) {} + ~Lockfile() {} void SetLockfileName(const char *filename) { @@ -44,7 +44,7 @@ class Lockfile } const char * - GetLockfileName(void) + GetLockfileName() { return fname; } @@ -69,7 +69,7 @@ class Lockfile // Close() // // Closes the file handle on the opened Lockfile. - void Close(void); + void Close(); // Kill() // KillGroup() diff --git a/include/tscore/ink_memory.h b/include/tscore/ink_memory.h index 464c10eb90d..d02411139ea 100644 --- a/include/tscore/ink_memory.h +++ b/include/tscore/ink_memory.h @@ -109,7 +109,7 @@ void *ats_track_malloc(size_t size, uint64_t *stat); void *ats_track_realloc(void *ptr, size_t size, uint64_t *alloc_stat, uint64_t *free_stat); void ats_track_free(void *ptr, uint64_t *stat); -static inline size_t __attribute__((const)) ats_pagesize(void) +static inline size_t __attribute__((const)) ats_pagesize() { static size_t page_size; diff --git a/include/tscpp/api/Async.h b/include/tscpp/api/Async.h index b392f494eda..37bdd292666 100644 --- a/include/tscpp/api/Async.h +++ b/include/tscpp/api/Async.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "tscpp/api/noncopyable.h" @@ -105,7 +106,7 @@ class AsyncProvider void doRun(std::shared_ptr dispatch_controller) { - dispatch_controller_ = dispatch_controller; + dispatch_controller_ = std::move(dispatch_controller); run(); } friend class Async; @@ -157,7 +158,7 @@ class AsyncDispatchController : public AsyncDispatchControllerBase * @param mutex Mutex of the receiver that is locked during the dispatch */ AsyncDispatchController(AsyncEventReceiverType *event_receiver, AsyncProviderType *provider, std::shared_ptr mutex) - : event_receiver_(event_receiver), dispatch_mutex_(mutex), provider_(provider) + : event_receiver_(event_receiver), dispatch_mutex_(std::move(mutex)), provider_(provider) { } diff --git a/include/tscpp/api/Headers.h b/include/tscpp/api/Headers.h index ebc7ecff2dd..25e1ce18660 100644 --- a/include/tscpp/api/Headers.h +++ b/include/tscpp/api/Headers.h @@ -235,7 +235,7 @@ class HeaderField { private: header_field_iterator iter_; - HeaderField(header_field_iterator iter) : iter_(iter) {} + HeaderField(const header_field_iterator &iter) : iter_(iter) {} public: typedef unsigned int size_type; diff --git a/include/wccp/Wccp.h b/include/wccp/Wccp.h index ebb07639b16..7f58e922b83 100644 --- a/include/wccp/Wccp.h +++ b/include/wccp/Wccp.h @@ -292,7 +292,7 @@ class Cache : public EndPoint /// Default constructor. Cache(); /// Destructor - ~Cache(); + ~Cache() override; /// Define services from a configuration file. ts::Errata loadServicesFromFile(char const *path ///< Path to file. @@ -308,7 +308,7 @@ class Cache : public EndPoint - @c ServiceGroup::CONFLICT if the service doesn't match the existing service. */ Service defineServiceGroup(ServiceGroup const &svc, ///< Service group description. - ServiceGroup::Result *result = 0); + ServiceGroup::Result *result = nullptr); /** Add a seed router to the service group. @@ -335,7 +335,7 @@ class Cache : public EndPoint /// Get the current implementation instance cast to correct type. ImplType const *impl() const; /// Create a new implementation instance. - super::ImplType *make(); + super::ImplType *make() override; }; /** Hold a reference to a service group in this end point. @@ -387,7 +387,7 @@ class Router : public EndPoint /// Default constructor Router(); /// Destructor. - ~Router(); + ~Router() override; /// Transmit pending messages. int sendPendingMessages(); @@ -398,7 +398,7 @@ class Router : public EndPoint /// Get the current implementation instance cast to correct type. ImplType *impl(); /// Create a new implementation instance. - super::ImplType *make(); + super::ImplType *make() override; }; // ------------------------------------------------------ @@ -499,7 +499,7 @@ ServiceGroup::clearPorts() return *this; } -inline Cache::Service::Service() : m_group(0) {} +inline Cache::Service::Service() : m_group(nullptr) {} inline Cache::Service::Service(Cache const &cache, detail::cache::GroupData &group) : m_cache(cache), m_group(&group) {} diff --git a/iocore/aio/I_AIO.h b/iocore/aio/I_AIO.h index 298ca54b0a5..6b957b6a0fa 100644 --- a/iocore/aio/I_AIO.h +++ b/iocore/aio/I_AIO.h @@ -149,4 +149,4 @@ int ink_aio_write(AIOCallback *op, int fromAPI = 0); int ink_aio_readv(AIOCallback *op, int fromAPI = 0); // fromAPI is a boolean to indicate if this is from a API call such as upload proxy feature int ink_aio_writev(AIOCallback *op, int fromAPI = 0); -AIOCallback *new_AIOCallback(void); +AIOCallback *new_AIOCallback(); diff --git a/iocore/cache/P_CacheHosting.h b/iocore/cache/P_CacheHosting.h index 87afc045517..e78b2a1149a 100644 --- a/iocore/cache/P_CacheHosting.h +++ b/iocore/cache/P_CacheHosting.h @@ -194,7 +194,7 @@ struct ConfigVolumes { void BuildListFromString(char *config_file_path, char *file_buf); void - clear_all(void) + clear_all() { // remove all the volumes from the queue for (int i = 0; i < num_volumes; i++) { diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h index 8ed84757789..fb2fb71b974 100644 --- a/iocore/cache/P_CacheInternal.h +++ b/iocore/cache/P_CacheInternal.h @@ -955,8 +955,8 @@ CacheRemoveCont::event_handler(int event, void *data) return EVENT_DONE; } -int64_t cache_bytes_used(void); -int64_t cache_bytes_total(void); +int64_t cache_bytes_used(); +int64_t cache_bytes_total(); #ifdef DEBUG #define CACHE_DEBUG_INCREMENT_DYN_STAT(_x) CACHE_INCREMENT_DYN_STAT(_x) diff --git a/iocore/dns/P_SplitDNSProcessor.h b/iocore/dns/P_SplitDNSProcessor.h index 4e206e21103..d180aad8242 100644 --- a/iocore/dns/P_SplitDNSProcessor.h +++ b/iocore/dns/P_SplitDNSProcessor.h @@ -130,12 +130,12 @@ class DNSRequestData : public RequestData public: DNSRequestData(); - char *get_string(); + char *get_string() override; - const char *get_host(); + const char *get_host() override; - sockaddr const *get_ip(); // unused required virtual method. - sockaddr const *get_client_ip(); // unused required virtual method. + sockaddr const *get_ip() override; // unused required virtual method. + sockaddr const *get_client_ip() override; // unused required virtual method. const char *m_pHost; }; diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index ed7ba5faeea..91ea46af1aa 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -533,7 +533,7 @@ db_mark_for(IpAddr const &ip) } Ptr -probe(Ptr mutex, HostDBHash const &hash, bool ignore_timeout) +probe(const Ptr &mutex, HostDBHash const &hash, bool ignore_timeout) { // If hostdb is disabled, don't return anything if (!hostdb_enable) { diff --git a/iocore/hostdb/P_RefCountCache.h b/iocore/hostdb/P_RefCountCache.h index 9ca3764e807..1d734e46d05 100644 --- a/iocore/hostdb/P_RefCountCache.h +++ b/iocore/hostdb/P_RefCountCache.h @@ -396,7 +396,7 @@ template class RefCountCache public: // Constructor RefCountCache(unsigned int num_partitions, int size = -1, int items = -1, ts::VersionNumber object_version = ts::VersionNumber(), - std::string metrics_prefix = ""); + const std::string &metrics_prefix = ""); // Destructor ~RefCountCache(); @@ -427,7 +427,7 @@ template class RefCountCache template RefCountCache::RefCountCache(unsigned int num_partitions, int size, int items, ts::VersionNumber object_version, - std::string metrics_prefix) + const std::string &metrics_prefix) : header(RefCountCacheHeader(object_version)), rsb(nullptr) { this->max_size = size; @@ -568,7 +568,7 @@ RefCountCache::clear() // Errors are -1 template int -LoadRefCountCacheFromPath(RefCountCache &cache, std::string dirname, std::string filepath, +LoadRefCountCacheFromPath(RefCountCache &cache, const std::string &dirname, const std::string &filepath, CacheEntryType *(*load_func)(char *, unsigned int)) { // If we have no load method, then we can't load anything so lets just stop right here diff --git a/iocore/hostdb/P_RefCountCacheSerializer.h b/iocore/hostdb/P_RefCountCacheSerializer.h index c153e7f5639..41b80b5867f 100644 --- a/iocore/hostdb/P_RefCountCacheSerializer.h +++ b/iocore/hostdb/P_RefCountCacheSerializer.h @@ -25,6 +25,7 @@ #include "P_RefCountCache.h" +#include #include // This continuation is responsible for persisting RefCountCache to disk @@ -57,7 +58,7 @@ template class RefCountCacheSerializer : public Continuation int write_to_disk(const void *, size_t); RefCountCacheSerializer(Continuation *acont, RefCountCache *cc, int frequency, std::string dirname, std::string filename); - ~RefCountCacheSerializer(); + ~RefCountCacheSerializer() override; private: std::vector partition_items; @@ -85,8 +86,8 @@ RefCountCacheSerializer::RefCountCacheSerializer(Continuation *acont, RefCoun cache(cc), cont(acont), fd(-1), - dirname(dirname), - filename(filename), + dirname(std::move(dirname)), + filename(std::move(filename)), time_per_partition(HRTIME_SECONDS(frequency) / cc->partition_count()), start(Thread::get_hrtime()), total_items(0), diff --git a/iocore/net/I_SessionAccept.h b/iocore/net/I_SessionAccept.h index c728175c6ce..7d746ae533f 100644 --- a/iocore/net/I_SessionAccept.h +++ b/iocore/net/I_SessionAccept.h @@ -56,7 +56,7 @@ class SessionAccept : public Continuation { public: SessionAccept(ProxyMutex *amutex) : Continuation(amutex) { SET_HANDLER(&SessionAccept::mainEvent); } - ~SessionAccept() {} + ~SessionAccept() override {} /** Accept a new connection on this session. diff --git a/iocore/net/I_UDPConnection.h b/iocore/net/I_UDPConnection.h index f1be035cce0..d06920f52de 100644 --- a/iocore/net/I_UDPConnection.h +++ b/iocore/net/I_UDPConnection.h @@ -83,7 +83,7 @@ class UDPConnection : public Continuation void AddRef(); int GetRefCount(); - int getPortNum(void); + int getPortNum(); int GetSendGenerationNumber(); // const void SetLastSentPktTSSeqNum(int64_t sentSeqNum); diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index f940490e302..4c111f0e14b 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -81,7 +81,7 @@ class TunnelDestination : public ActionItem { public: TunnelDestination(const std::string_view &dest, bool decrypt) : destination(dest), tunnel_decrypt(decrypt) {} - ~TunnelDestination() {} + ~TunnelDestination() override {} int SNIAction(Continuation *cont) const override diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 39058b621b5..c0dc504f17d 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -130,7 +130,7 @@ struct SSLConfigParams : public ConfigInfo { mutable std::unordered_map top_level_ctx_map; mutable ink_mutex ctxMapLock; - SSL_CTX *getClientSSL_CTX(void) const; + SSL_CTX *getClientSSL_CTX() const; SSL_CTX *getCTX(const char *client_cert, const char *key_file, const char *ca_bundle_file, const char *ca_bundle_path) const; void cleanupCTXTable(); diff --git a/iocore/net/P_SSLNetProcessor.h b/iocore/net/P_SSLNetProcessor.h index 4b15f001424..92183ed82d7 100644 --- a/iocore/net/P_SSLNetProcessor.h +++ b/iocore/net/P_SSLNetProcessor.h @@ -55,7 +55,7 @@ struct SSLNetProcessor : public UnixNetProcessor { public: int start(int, size_t stacksize) override; - void cleanup(void); + void cleanup(); SSLNetProcessor(); ~SSLNetProcessor() override; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 25e84b57611..7f350107925 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -298,13 +298,13 @@ class SSLNetVConnection : public UnixNetVConnection } const char * - getSSLProtocol(void) const + getSSLProtocol() const { return ssl ? SSL_get_version(ssl) : nullptr; } const char * - getSSLCipherSuite(void) const + getSSLCipherSuite() const { return ssl ? SSL_get_cipher_name(ssl) : nullptr; } diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index 220de51736b..78d054eab53 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -59,12 +59,12 @@ struct namedElement { setGlobName(std::string name) { std::string::size_type pos = 0; - while ((pos = name.find(".", pos)) != std::string::npos) { + while ((pos = name.find('.', pos)) != std::string::npos) { name.replace(pos, 1, "\\."); pos += 2; } pos = 0; - while ((pos = name.find("*", pos)) != std::string::npos) { + while ((pos = name.find('*', pos)) != std::string::npos) { name.replace(pos, 1, ".{0,}"); } Debug("ssl_sni", "Regexed fqdn=%s", name.c_str()); diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index bc6fbf21219..7a8a9758fb1 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -171,7 +171,7 @@ namespace detail struct ats_wildcard_matcher { ats_wildcard_matcher() { - if (!regex.compile("^\\*\\.[^\\*.]+")) { + if (!regex.compile(R"(^\*\.[^\*.]+)")) { Fatal("failed to compile TLS wildcard matching regex"); } } diff --git a/iocore/net/P_UDPConnection.h b/iocore/net/P_UDPConnection.h index 011ccae2287..ed08d89df99 100644 --- a/iocore/net/P_UDPConnection.h +++ b/iocore/net/P_UDPConnection.h @@ -140,13 +140,13 @@ UDPConnection::GetSendGenerationNumber() } TS_INLINE int -UDPConnection::getPortNum(void) +UDPConnection::getPortNum() { return ats_ip_port_host_order(&static_cast(this)->binding); } TS_INLINE int64_t -UDPConnection::cancel(void) +UDPConnection::cancel() { UDPConnectionInternal *p = (UDPConnectionInternal *)this; diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index 9e7d869c42e..c23dc07a6e7 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -76,7 +76,7 @@ class PacketQueue int now_slot; void - init(void) + init() { now_slot = 0; ink_hrtime now = ink_get_hrtime_internal(); diff --git a/iocore/net/P_UDPPacket.h b/iocore/net/P_UDPPacket.h index 0b6363a2f5b..c76deee7f60 100644 --- a/iocore/net/P_UDPPacket.h +++ b/iocore/net/P_UDPPacket.h @@ -151,14 +151,14 @@ UDPPacket::setConnection(UDPConnection *c) } TS_INLINE IOBufferBlock * -UDPPacket::getIOBlockChain(void) +UDPPacket::getIOBlockChain() { ink_assert(dynamic_cast(this) != nullptr); return ((UDPPacketInternal *)this)->chain.get(); } TS_INLINE UDPConnection * -UDPPacket::getConnection(void) +UDPPacket::getConnection() { return ((UDPPacketInternal *)this)->conn; } diff --git a/iocore/net/P_UnixPollDescriptor.h b/iocore/net/P_UnixPollDescriptor.h index f377cbdcf48..471d9e7647c 100644 --- a/iocore/net/P_UnixPollDescriptor.h +++ b/iocore/net/P_UnixPollDescriptor.h @@ -112,7 +112,7 @@ struct PollDescriptor { } return &pfd[nfds++]; #else - return 0; + return nullptr; #endif } diff --git a/iocore/net/ProxyProtocol.h b/iocore/net/ProxyProtocol.h index b561205f05c..607dcdfbd30 100644 --- a/iocore/net/ProxyProtocol.h +++ b/iocore/net/ProxyProtocol.h @@ -40,7 +40,7 @@ extern bool proxy_protov1_parse(NetVConnection *, ts::TextView hdr); extern bool ssl_has_proxy_v1(NetVConnection *, char *, int64_t *); extern bool http_has_proxy_v1(IOBufferReader *, NetVConnection *); -const char *const PROXY_V1_CONNECTION_PREFACE = "\x50\x52\x4F\x58\x59"; +const char *const PROXY_V1_CONNECTION_PREFACE = R"(PROXY)"; const char *const PROXY_V2_CONNECTION_PREFACE = "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A\x02"; const size_t PROXY_V1_CONNECTION_PREFACE_LEN = strlen(PROXY_V1_CONNECTION_PREFACE); // 5 diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 006f56a193b..ea090c210d3 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -109,11 +109,11 @@ SNIConfigParams::SNIConfigParams() {} const actionVector * SNIConfigParams::get(const std::string &servername) const { - for (auto retval = sni_action_list.begin(); retval != sni_action_list.end(); ++retval) { - if (retval->match == nullptr && servername.length() == 0) { - return &retval->actions; - } else if (pcre_exec(retval->match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { - return &retval->actions; + for (const auto &retval : sni_action_list) { + if (retval.match == nullptr && servername.length() == 0) { + return &retval.actions; + } else if (pcre_exec(retval.match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { + return &retval.actions; } } return nullptr; diff --git a/iocore/net/SSLSessionCache.h b/iocore/net/SSLSessionCache.h index f1ccbe395b1..9c763a83434 100644 --- a/iocore/net/SSLSessionCache.h +++ b/iocore/net/SSLSessionCache.h @@ -116,7 +116,7 @@ class SSLSession Ptr asn1_data; /* this is the ASN1 representation of the SSL_CTX */ size_t len_asn1_data; - SSLSession(const SSLSessionID &id, Ptr ssl_asn1_data, size_t len_asn1) + SSLSession(const SSLSessionID &id, const Ptr &ssl_asn1_data, size_t len_asn1) : session_id(id), asn1_data(ssl_asn1_data), len_asn1_data(len_asn1) { } diff --git a/iocore/utils/Machine.cc b/iocore/utils/Machine.cc index b8bc01576e4..bcfc1efbe6d 100644 --- a/iocore/utils/Machine.cc +++ b/iocore/utils/Machine.cc @@ -231,8 +231,8 @@ Machine::Machine(char const *the_hostname, sockaddr const *addr) Machine::~Machine() { ats_free(hostname); - for (auto spot = machine_id_ipaddrs.begin(); spot != machine_id_ipaddrs.end(); spot++) { - delete spot->second; + for (auto &machine_id_ipaddr : machine_id_ipaddrs) { + delete machine_id_ipaddr.second; } } diff --git a/lib/records/I_RecCore.h b/lib/records/I_RecCore.h index 3bea5c5891e..188b33192da 100644 --- a/lib/records/I_RecCore.h +++ b/lib/records/I_RecCore.h @@ -46,7 +46,7 @@ int RecSetDiags(Diags *diags); typedef void (*RecConfigEntryCallback)(RecT rec_type, RecDataT data_type, const char *name, const char *value, RecSourceT source, bool inc_version); -void RecConfigFileInit(void); +void RecConfigFileInit(); int RecConfigFileParse(const char *path, RecConfigEntryCallback handler, bool inc_version); // Return a copy of the system's configuration directory. diff --git a/lib/records/I_RecProcess.h b/lib/records/I_RecProcess.h index eb7b98b51e6..c7a6faa2c89 100644 --- a/lib/records/I_RecProcess.h +++ b/lib/records/I_RecProcess.h @@ -31,7 +31,7 @@ //------------------------------------------------------------------------- int RecProcessInit(RecModeT mode_type, Diags *diags = nullptr); int RecProcessInitMessage(RecModeT mode_type); -int RecProcessStart(void); +int RecProcessStart(); //------------------------------------------------------------------------- // Setters for manipulating internal sleep intervals diff --git a/lib/tsconfig/Errata.h b/lib/tsconfig/Errata.h index 15b2373ce98..6d391ccbe77 100644 --- a/lib/tsconfig/Errata.h +++ b/lib/tsconfig/Errata.h @@ -827,7 +827,7 @@ inline size_t Errata::size() const { } inline bool Errata::isOK() const { - return 0 == m_data + return nullptr == m_data || 0 == m_data->size() || Message::Success_Test(this->top()) ; diff --git a/lib/tsconfig/TsValue.h b/lib/tsconfig/TsValue.h index fbbf473e1c6..8863eb8db6b 100644 --- a/lib/tsconfig/TsValue.h +++ b/lib/tsconfig/TsValue.h @@ -29,6 +29,7 @@ #include #include #include +#include #include namespace ts { namespace config { @@ -651,7 +652,7 @@ namespace detail { inline Value::~Value() { } inline Value::Value() : _vidx(detail::NULL_VALUE_INDEX) {} -inline Value::Value(Configuration cfg, detail::ValueIndex vidx) : _config(cfg), _vidx(vidx) { } +inline Value::Value(Configuration cfg, detail::ValueIndex vidx) : _config(std::move(cfg)), _vidx(vidx) { } inline bool Value::hasValue() const { return _config && _vidx != detail::NULL_VALUE_INDEX; } inline Value::operator detail::PseudoBool::Type () const { return this->hasValue() ? detail::PseudoBool::TRUE : detail::PseudoBool::FALSE; } inline bool Value::operator ! () const { return ! this->hasValue(); } diff --git a/mgmt/utils/MgmtSocket.h b/mgmt/utils/MgmtSocket.h index 99ecd88e952..46eb242b548 100644 --- a/mgmt/utils/MgmtSocket.h +++ b/mgmt/utils/MgmtSocket.h @@ -94,7 +94,7 @@ int mgmt_write_timeout(int fd, int sec, int usec); int mgmt_read_timeout(int fd, int sec, int usec); // Do we support passing Unix domain credentials on this platform? -bool mgmt_has_peereid(void); +bool mgmt_has_peereid(); // Get the Unix domain peer credentials. int mgmt_get_peereid(int fd, uid_t *euid, gid_t *egid); diff --git a/plugins/background_fetch/rules.cc b/plugins/background_fetch/rules.cc index 98ddd744da4..ddbd2bc0e64 100644 --- a/plugins/background_fetch/rules.cc +++ b/plugins/background_fetch/rules.cc @@ -24,7 +24,7 @@ #include #include -#include +#include #include "configs.h" #include "rules.h" diff --git a/plugins/cachekey/cachekey.cc b/plugins/cachekey/cachekey.cc index c89b65755b8..9cf4454210f 100644 --- a/plugins/cachekey/cachekey.cc +++ b/plugins/cachekey/cachekey.cc @@ -23,6 +23,7 @@ #include /* strlen() */ #include /* istringstream */ +#include #include "cachekey.h" static void @@ -195,7 +196,7 @@ getUri(TSMBuffer buf, TSMLoc url) * @param rri remap request info */ CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSRemapRequestInfo *rri) - : _txn(txn), _separator(separator), _uriType(uriType) + : _txn(txn), _separator(std::move(separator)), _uriType(uriType) { _key.reserve(512); diff --git a/plugins/cachekey/configs.cc b/plugins/cachekey/configs.cc index 2a13d7608f9..e391a5c4318 100644 --- a/plugins/cachekey/configs.cc +++ b/plugins/cachekey/configs.cc @@ -183,8 +183,8 @@ ConfigElements::noIncludeExcludeRules() const ConfigElements::~ConfigElements() { - for (auto it = _captures.begin(); it != _captures.end(); it++) { - delete it->second; + for (auto &_capture : _captures) { + delete _capture.second; } } diff --git a/plugins/cachekey/pattern.h b/plugins/cachekey/pattern.h index 876edf0a3db..47cd35cbb89 100644 --- a/plugins/cachekey/pattern.h +++ b/plugins/cachekey/pattern.h @@ -77,7 +77,7 @@ class Pattern class MultiPattern { public: - MultiPattern(const String name = "") : _name(name) {} + MultiPattern(const String &name = "") : _name(name) {} virtual ~MultiPattern(); bool empty() const; diff --git a/plugins/esi/lib/HandlerManager.h b/plugins/esi/lib/HandlerManager.h index 607027b654d..7b44d8651e1 100644 --- a/plugins/esi/lib/HandlerManager.h +++ b/plugins/esi/lib/HandlerManager.h @@ -45,7 +45,7 @@ class HandlerManager : protected ComponentBase SpecialIncludeHandler *getHandler(Variables &esi_vars, Expression &esi_expr, HttpDataFetcher &http_fetcher, const std::string &id) const; - ~HandlerManager(); + ~HandlerManager() override; private: typedef std::map FunctionHandleMap; diff --git a/plugins/experimental/access_control/access_control.h b/plugins/experimental/access_control/access_control.h index dbc9f6f9475..6569cb1ec65 100644 --- a/plugins/experimental/access_control/access_control.h +++ b/plugins/experimental/access_control/access_control.h @@ -216,7 +216,7 @@ class KvpAccessToken : public AccessToken { public: KvpAccessToken(const KvpAccessTokenConfig &tokenConfig, const StringMap &secretsMap, bool enableDebug = false); - AccessTokenStatus parse(const StringView token); + AccessTokenStatus parse(const StringView token) override; protected: const KvpAccessTokenConfig &_tokenConfig; /** @brief description of keys' names and delimiters */ diff --git a/plugins/experimental/access_control/config.cc b/plugins/experimental/access_control/config.cc index bf133c1f437..933225c4cd2 100644 --- a/plugins/experimental/access_control/config.cc +++ b/plugins/experimental/access_control/config.cc @@ -169,23 +169,23 @@ load(T &container, const String &filename) bool AccessControlConfig::init(int argc, char *argv[]) { - static const struct option longopt[] = {{const_cast("invalid-syntax-status-code"), optional_argument, 0, 'a'}, - {const_cast("invalid-signature-status-code"), optional_argument, 0, 'b'}, - {const_cast("invalid-timing-status-code"), optional_argument, 0, 'c'}, - {const_cast("invalid-scope-status-code"), optional_argument, 0, 'd'}, - {const_cast("invalid-origin-response"), optional_argument, 0, 'e'}, - {const_cast("internal-error-status-code"), optional_argument, 0, 'f'}, - {const_cast("check-cookie"), optional_argument, 0, 'g'}, - {const_cast("symmetric-keys-map"), optional_argument, 0, 'h'}, - {const_cast("reject-invalid-token-requests"), optional_argument, 0, 'i'}, - {const_cast("extract-subject-to-header"), optional_argument, 0, 'j'}, - {const_cast("extract-tokenid-to-header"), optional_argument, 0, 'k'}, - {const_cast("extract-status-to-header"), optional_argument, 0, 'l'}, - {const_cast("token-response-header"), optional_argument, 0, 'm'}, - {const_cast("use-redirects"), optional_argument, 0, 'n'}, + static const struct option longopt[] = {{const_cast("invalid-syntax-status-code"), optional_argument, nullptr, 'a'}, + {const_cast("invalid-signature-status-code"), optional_argument, nullptr, 'b'}, + {const_cast("invalid-timing-status-code"), optional_argument, nullptr, 'c'}, + {const_cast("invalid-scope-status-code"), optional_argument, nullptr, 'd'}, + {const_cast("invalid-origin-response"), optional_argument, nullptr, 'e'}, + {const_cast("internal-error-status-code"), optional_argument, nullptr, 'f'}, + {const_cast("check-cookie"), optional_argument, nullptr, 'g'}, + {const_cast("symmetric-keys-map"), optional_argument, nullptr, 'h'}, + {const_cast("reject-invalid-token-requests"), optional_argument, nullptr, 'i'}, + {const_cast("extract-subject-to-header"), optional_argument, nullptr, 'j'}, + {const_cast("extract-tokenid-to-header"), optional_argument, nullptr, 'k'}, + {const_cast("extract-status-to-header"), optional_argument, nullptr, 'l'}, + {const_cast("token-response-header"), optional_argument, nullptr, 'm'}, + {const_cast("use-redirects"), optional_argument, nullptr, 'n'}, {const_cast("include-uri-paths-file"), optional_argument, nullptr, 'o'}, {const_cast("exclude-uri-paths-file"), optional_argument, nullptr, 'p'}, - {0, 0, 0, 0}}; + {nullptr, 0, nullptr, 0}}; bool status = true; optind = 0; diff --git a/plugins/experimental/access_control/headers.cc b/plugins/experimental/access_control/headers.cc index fda3a74501d..ddc64e4c33e 100644 --- a/plugins/experimental/access_control/headers.cc +++ b/plugins/experimental/access_control/headers.cc @@ -21,8 +21,8 @@ * @brief HTTP headers manipulation. */ -#include -#include +#include +#include #include "headers.h" #include "common.h" diff --git a/plugins/experimental/access_control/pattern.h b/plugins/experimental/access_control/pattern.h index 57f2d9f601d..6f05487ee8f 100644 --- a/plugins/experimental/access_control/pattern.h +++ b/plugins/experimental/access_control/pattern.h @@ -76,7 +76,7 @@ class Pattern class MultiPattern { public: - MultiPattern(const String name = "") : _name(name) {} + MultiPattern(const String &name = "") : _name(name) {} virtual ~MultiPattern(); bool empty() const; @@ -106,14 +106,14 @@ class NonMatchingMultiPattern : public MultiPattern * @param subject subject string * @return return false if any of the patterns matches, true otherwise. */ - virtual bool - match(const String &subject) const + bool + match(const String &subject) const override { return !MultiPattern::match(subject); } - virtual bool - match(const String &subject, String &pattern) const + bool + match(const String &subject, String &pattern) const override { return !MultiPattern::match(subject, pattern); } diff --git a/plugins/experimental/access_control/plugin.cc b/plugins/experimental/access_control/plugin.cc index 5f003e39d41..4db3ac529dc 100644 --- a/plugins/experimental/access_control/plugin.cc +++ b/plugins/experimental/access_control/plugin.cc @@ -359,7 +359,7 @@ contHandleAccessControl(const TSCont contp, TSEvent event, void *edata) AccessToken *token = config->_tokenFactory->getAccessToken(); if (nullptr != token && - VALID == (data->_originState = token->validate(StringView(tokenHdrValue, tokenHdrValueLen), time(0)))) { + VALID == (data->_originState = token->validate(StringView(tokenHdrValue, tokenHdrValueLen), time(nullptr)))) { /* * From RFC 6265 "HTTP State Management Mechanism": * To maximize compatibility with user agents, servers that wish to @@ -520,7 +520,7 @@ enforceAccessControl(TSHttpTxn txnp, TSRemapRequestInfo *rri, AccessControlConfi if (0 < decryptedCookieSize) { AccessToken *token = config->_tokenFactory->getAccessToken(); if (nullptr != token) { - data->_vaState = token->validate(StringView(decodedCookie, decryptedCookieSize), time(0)); + data->_vaState = token->validate(StringView(decodedCookie, decryptedCookieSize), time(nullptr)); if (VALID != data->_vaState) { remapStatus = handleInvalidToken(txnp, data, reject, accessTokenStateToHttpStatus(data->_vaState, config), data->_vaState); diff --git a/plugins/experimental/access_control/utils.cc b/plugins/experimental/access_control/utils.cc index cfddac70e46..2915db76837 100644 --- a/plugins/experimental/access_control/utils.cc +++ b/plugins/experimental/access_control/utils.cc @@ -22,8 +22,8 @@ * @see utils.h */ -#include /* errno */ -#include /* LONG_MIN, LONG_MAX */ +#include /* errno */ +#include /* LONG_MIN, LONG_MAX */ #include /* BIO I/O abstraction */ #include /* buf_mem_st */ #include /* ERR_get_error() and ERR_error_string_n() */ @@ -279,7 +279,7 @@ cryptoMessageDigestGet(const char *digestType, const char *data, size_t dataLen, /* success */ result = len; - } while (0); + } while (false); EVP_PKEY_free(pkey); EVP_MD_CTX_destroy(ctx); diff --git a/plugins/experimental/access_control/utils.h b/plugins/experimental/access_control/utils.h index 02f027e5f35..6c821df692e 100644 --- a/plugins/experimental/access_control/utils.h +++ b/plugins/experimental/access_control/utils.h @@ -26,7 +26,7 @@ #include /* std:string_view */ #include /* EVP_* constants, structures and functions. */ -#include /* strlen, strncmp, strncpy, memset, size_t */ +#include /* strlen, strncmp, strncpy, memset, size_t */ #define MAX_MSGDIGEST_BUFFER_SIZE EVP_MAX_MD_SIZE diff --git a/plugins/experimental/cookie_remap/cookie_remap.cc b/plugins/experimental/cookie_remap/cookie_remap.cc index b5671888bed..d488944ad06 100644 --- a/plugins/experimental/cookie_remap/cookie_remap.cc +++ b/plugins/experimental/cookie_remap/cookie_remap.cc @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include #include @@ -300,7 +300,7 @@ class subop void setBucket(const std::string &s) { - int start_pos = s.find("/"); + int start_pos = s.find('/'); op_type = BUCKET; bucket = s; @@ -335,7 +335,7 @@ class subop return false; } regex_extra = pcre_study(regex, 0, &error_study); - if ((regex_extra == nullptr) && (error_study != 0)) { + if ((regex_extra == nullptr) && (error_study != nullptr)) { return false; } @@ -418,8 +418,8 @@ class op ~op() { TSDebug(MY_NAME, "op destructor called"); - for (SubOpQueue::iterator it = subops.begin(); it != subops.end(); ++it) { - delete *it; + for (auto &subop : subops) { + delete subop; } } @@ -469,8 +469,8 @@ class op TSDebug(MY_NAME, "sending to: %s", sendto.c_str()); TSDebug(MY_NAME, "if these operations match: "); - for (SubOpQueue::const_iterator it = subops.begin(); it != subops.end(); ++it) { - (*it)->printSubOp(); + for (auto subop : subops) { + subop->printSubOp(); } if (else_sendto.size() > 0) TSDebug(MY_NAME, "else: %s", else_sendto.c_str()); @@ -494,16 +494,16 @@ class op TSDebug(MY_NAME, "starting to process a new operation"); - for (SubOpQueue::const_iterator it = subops.begin(); it != subops.end(); ++it) { + for (auto subop : subops) { // subop* s = *it; - int subop_type = (*it)->getOpType(); - target_type target = (*it)->getTargetType(); + int subop_type = subop->getOpType(); + target_type target = subop->getTargetType(); - c = (*it)->getCookieName(); + c = subop->getCookieName(); if (c.length()) { TSDebug(MY_NAME, "processing cookie: %s", c.c_str()); - size_t period_pos = c.find_first_of("."); + size_t period_pos = c.find_first_of('.'); if (period_pos == std::string::npos) { // not a sublevel // cookie name @@ -616,7 +616,7 @@ class op // OPERATION::string matching if (subop_type == STRING) { - if (string_to_match == (*it)->getStringMatch()) { + if (string_to_match == subop->getStringMatch()) { TSDebug(MY_NAME, "string match succeeded"); continue; } else { @@ -629,7 +629,7 @@ class op // OPERATION::regex matching if (subop_type == REGEXP) { int ovector[OVECCOUNT]; - int ret = (*it)->regexMatch(string_to_match.c_str(), string_to_match.length(), ovector); + int ret = subop->regexMatch(string_to_match.c_str(), string_to_match.length(), ovector); if (ret >= 0) { std::string::size_type pos = sendto.find('$'); @@ -649,7 +649,7 @@ class op "successful regex " "match of: %s with %s " "rewriting string: %s", - string_to_match.c_str(), (*it)->getRegexString().c_str(), sendto.c_str()); + string_to_match.c_str(), subop->getRegexString().c_str(), sendto.c_str()); // replace the $(1-9) in the sendto url // as necessary @@ -659,7 +659,7 @@ class op if (isdigit(sendto[pos + 1])) { int ix = sendto[pos + 1] - '0'; - if (ix <= (*it)->getRegexCcount()) { // Just skip an illegal regex group + if (ix <= subop->getRegexCcount()) { // Just skip an illegal regex group dest += sendto.substr(ppos, pos - ppos); dest += string_to_match.substr(ovector[ix * 2], ovector[ix * 2 + 1] - ovector[ix * 2]); ppos = pos + 2; @@ -682,7 +682,7 @@ class op "could not match " "regular expression " "%s to %s", - (*it)->getRegexString().c_str(), string_to_match.c_str()); + subop->getRegexString().c_str(), string_to_match.c_str()); retval &= 0; break; } @@ -690,8 +690,8 @@ class op // OPERATION::bucket ranges if (subop_type == BUCKET) { - unsigned int taking = (*it)->bucketGetTaking(); - unsigned int out_of = (*it)->bucketOutOf(); + unsigned int taking = subop->bucketGetTaking(); + unsigned int out_of = subop->bucketOutOf(); uint32_t hash; @@ -1047,9 +1047,9 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) CookieJar jar; jar.create(temp_cookie); - for (OpsQueue::iterator it = ops->begin(); it != ops->end(); ++it) { + for (auto &op : *ops) { TSDebug(MY_NAME, ">>> processing new operation"); - if ((*it)->process(jar, rewrite_to, status, rri)) { + if (op->process(jar, rewrite_to, status, rri)) { cr_substitutions(rewrite_to, rri); size_t pos = 7; // 7 because we want to ignore the // in @@ -1062,7 +1062,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) } while (pos <= rewrite_to.length() && pos < tmp_pos); // Add Query Parameters if not already present - if (!client_req_query_params.empty() && rewrite_to.find("?") == std::string::npos) { + if (!client_req_query_params.empty() && rewrite_to.find('?') == std::string::npos) { rewrite_to.append(client_req_query_params); } @@ -1141,8 +1141,8 @@ TSRemapDeleteInstance(void *ih) OpsQueue *ops = (OpsQueue *)ih; TSDebug(MY_NAME, "deleting loaded operations"); - for (OpsQueue::iterator it = ops->begin(); it != ops->end(); ++it) { - delete *it; + for (auto &op : *ops) { + delete op; } delete ops; diff --git a/plugins/experimental/cookie_remap/cookiejar.cc b/plugins/experimental/cookie_remap/cookiejar.cc index be46e2c9439..8a8c67565a4 100644 --- a/plugins/experimental/cookie_remap/cookiejar.cc +++ b/plugins/experimental/cookie_remap/cookiejar.cc @@ -18,7 +18,7 @@ #include "cookiejar.h" #include "strip.h" -#include +#include /* allowed cookie-name definition from RFC * cookie-name = token diff --git a/plugins/experimental/cookie_remap/cookiejar.h b/plugins/experimental/cookie_remap/cookiejar.h index 65d1c9bd112..34e9907317b 100644 --- a/plugins/experimental/cookie_remap/cookiejar.h +++ b/plugins/experimental/cookie_remap/cookiejar.h @@ -52,7 +52,7 @@ class CookieJar public: unordered_map m_subelements; string m_val; - bool parts_inited = 0; + bool parts_inited = false; }; CookieVal *m_currentVal = nullptr; diff --git a/plugins/experimental/cookie_remap/test_cookiejar.cc b/plugins/experimental/cookie_remap/test_cookiejar.cc index cacf8575a15..73b94f349d5 100644 --- a/plugins/experimental/cookie_remap/test_cookiejar.cc +++ b/plugins/experimental/cookie_remap/test_cookiejar.cc @@ -19,7 +19,7 @@ #define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file #include #include -#include +#include #include "catch.hpp" #include "cookiejar.h" diff --git a/plugins/experimental/inliner/cache-handler.h b/plugins/experimental/inliner/cache-handler.h index 75126c526b7..88544639609 100644 --- a/plugins/experimental/inliner/cache-handler.h +++ b/plugins/experimental/inliner/cache-handler.h @@ -101,7 +101,7 @@ namespace inliner } void - done(void) + done() { if (GIF::verifySignature(content_)) { contentType_ = "image/gif"; @@ -165,13 +165,13 @@ namespace inliner } void - timeout(void) const + timeout() const { TSDebug(PLUGIN_TAG, "Fetch timeout"); } void - error(void) const + error() const { TSDebug(PLUGIN_TAG, "Fetch error"); } @@ -225,7 +225,7 @@ namespace inliner } template - CacheHandler(const std::string &s, const std::string &o, const std::string c, const std::string &i, T1 &&si, T2 &&si2) + CacheHandler(const std::string &s, const std::string &o, const std::string &c, const std::string &i, T1 &&si, T2 &&si2) : src_(s), original_(o), classes_(c), id_(i), sink_(std::forward(si)), sink2_(std::forward(si2)), reader_(nullptr) { assert(sink_ != nullptr); @@ -248,7 +248,7 @@ namespace inliner CacheHandler &operator=(const CacheHandler &) = delete; void - done(void) + done() { assert(reader_ != nullptr); assert(sink2_ != nullptr); @@ -291,7 +291,7 @@ namespace inliner } void - miss(void) + miss() { assert(sink_ != nullptr); *sink_ << original_; diff --git a/plugins/experimental/inliner/cache.h b/plugins/experimental/inliner/cache.h index 3e902b53285..37376462b52 100644 --- a/plugins/experimental/inliner/cache.h +++ b/plugins/experimental/inliner/cache.h @@ -40,7 +40,7 @@ namespace cache TSCacheKeyDestroy(key_); } - Key(void) : key_(TSCacheKeyCreate()) { assert(key_ != nullptr); } + Key() : key_(TSCacheKeyCreate()) { assert(key_ != nullptr); } Key(const Key &) = delete; Key &operator=(const Key &) = delete; @@ -51,7 +51,7 @@ namespace cache } TSCacheKey - key(void) const + key() const { return key_; } diff --git a/plugins/experimental/inliner/chunk-decoder.h b/plugins/experimental/inliner/chunk-decoder.h index 346fe53bf53..26318b90f7b 100644 --- a/plugins/experimental/inliner/chunk-decoder.h +++ b/plugins/experimental/inliner/chunk-decoder.h @@ -70,14 +70,14 @@ class ChunkDecoder int64_t size_; public: - ChunkDecoder(void) : state_(State::kSize), size_(0) {} + ChunkDecoder() : state_(State::kSize), size_(0) {} void parseSizeCharacter(const char); int parseSize(const char *, const int64_t); int decode(const TSIOBufferReader &); - bool isSizeState(void) const; + bool isSizeState() const; inline bool - isEnd(void) const + isEnd() const { return state_ == State::kEnd; } diff --git a/plugins/experimental/inliner/fetcher.h b/plugins/experimental/inliner/fetcher.h index 36e3fe496e4..a1cb37005fb 100644 --- a/plugins/experimental/inliner/fetcher.h +++ b/plugins/experimental/inliner/fetcher.h @@ -68,7 +68,7 @@ struct HttpParser { TSMBuffer buffer_; TSMLoc location_; - void destroyParser(void); + void destroyParser(); ~HttpParser() { @@ -77,7 +77,7 @@ struct HttpParser { destroyParser(); } - HttpParser(void) : parsed_(false), parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) + HttpParser() : parsed_(false), parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) { TSHttpHdrTypeSet(buffer_, location_, TS_HTTP_TYPE_RESPONSE); } @@ -85,7 +85,7 @@ struct HttpParser { bool parse(io::IO &); int - statusCode(void) const + statusCode() const { return static_cast(TSHttpHdrStatusGet(buffer_, location_)); } diff --git a/plugins/experimental/inliner/html-parser.h b/plugins/experimental/inliner/html-parser.h index e203fa940b0..38f4b84fbea 100644 --- a/plugins/experimental/inliner/html-parser.h +++ b/plugins/experimental/inliner/html-parser.h @@ -35,7 +35,7 @@ namespace inliner typedef std::vector AttributeVector; struct Attributes : AttributeVector { - operator std::string(void) const; + operator std::string() const; }; struct Tag { @@ -104,9 +104,9 @@ namespace inliner Attribute::ATTRIBUTES state_; Attributes attributes; - AttributeParser(void) : state_(Attribute::kPreName) {} + AttributeParser() : state_(Attribute::kPreName) {} void - reset(void) + reset() { state_ = Attribute::kPreName; attributes.clear(); @@ -133,7 +133,7 @@ namespace inliner Tag::TAGS tag_; AttributeParser attributeParser_; - HtmlParser(void) : state_(State::kUndefined), tag_(Tag::kUndefined) {} + HtmlParser() : state_(State::kUndefined), tag_(Tag::kUndefined) {} virtual ~HtmlParser() {} bool parseTag(const char); size_t parse(const char *, size_t, size_t o = 0); diff --git a/plugins/experimental/inliner/inliner-handler.h b/plugins/experimental/inliner/inliner-handler.h index af1fece6e81..c8733d3c838 100644 --- a/plugins/experimental/inliner/inliner-handler.h +++ b/plugins/experimental/inliner/inliner-handler.h @@ -57,14 +57,14 @@ namespace inliner Handler(const Handler &) = delete; Handler &operator=(const Handler &) = delete; - void parse(void); + void parse(); size_t bypass(const size_t, const size_t) override; void handleImage(const Attributes &) override; - std::string generateId(void); + std::string generateId(); - void abort(void); + void abort(); }; } // namespace inliner diff --git a/plugins/experimental/inliner/png.h b/plugins/experimental/inliner/png.h index 6c0db40d150..22a79ce1eb3 100644 --- a/plugins/experimental/inliner/png.h +++ b/plugins/experimental/inliner/png.h @@ -63,13 +63,13 @@ namespace inliner public: uint32_t - length(void) const + length() const { return (length_[0] << 24) | (length_[1] << 16) | (length_[2] << 8) | length_[3]; } std::string - type(void) const + type() const { return std::string(type_, 4); } diff --git a/plugins/experimental/inliner/ts.h b/plugins/experimental/inliner/ts.h index a09f1029a81..69cf2aa7ca3 100644 --- a/plugins/experimental/inliner/ts.h +++ b/plugins/experimental/inliner/ts.h @@ -58,7 +58,7 @@ namespace io TSIOBufferDestroy(buffer); } - IO(void) : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) {} + IO() : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) {} IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) { assert(buffer != nullptr); } static IO *read(TSVConn, TSCont, const int64_t); @@ -78,9 +78,9 @@ namespace io uint64_t copy(const std::string &) const; - int64_t consume(void) const; + int64_t consume() const; - int64_t done(void) const; + int64_t done() const; }; struct ReaderSize { @@ -131,7 +131,7 @@ namespace io } // noncopyable - Lock(void) : mutex_(nullptr) {} + Lock() : mutex_(nullptr) {} Lock(const Lock &) = delete; Lock(Lock &&l) : mutex_(l.mutex_) { const_cast(l.mutex_) = nullptr; } @@ -166,8 +166,8 @@ namespace io WriteOperation &operator<<(const std::string &); void process(const size_t b = 0); - void close(void); - void abort(void); + void close(); + void abort(); private: WriteOperation(const TSVConn, const TSMutex, const size_t); @@ -215,10 +215,10 @@ namespace io return IOSinkPointer(new IOSink(WriteOperation::Create(std::forward(a)...))); } - void process(void); - SinkPointer branch(void); - Lock lock(void); - void abort(void); + void process(); + SinkPointer branch(); + Lock lock(); + void abort(); private: IOSink(WriteOperationWeakPointer &&p) : operation_(std::move(p)) {} @@ -249,7 +249,7 @@ namespace io TSIOBufferDestroy(buffer_); } - BufferNode(void) : buffer_(TSIOBufferCreate()), reader_(TSIOBufferReaderAlloc(buffer_)) + BufferNode() : buffer_(TSIOBufferCreate()), reader_(TSIOBufferReaderAlloc(buffer_)) { assert(buffer_ != nullptr); assert(reader_ != nullptr); @@ -289,7 +289,7 @@ namespace io Sink(const Sink &) = delete; Sink &operator=(const Sink &) = delete; - SinkPointer branch(void); + SinkPointer branch(); Sink &operator<<(std::string &&); diff --git a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc index 16589c4b1b6..0c91b83dff5 100644 --- a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc +++ b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc @@ -16,12 +16,12 @@ limitations under the License. */ -#include +#include #include #include #include #include -#include +#include #include #include #include @@ -365,7 +365,8 @@ req_hdr_ja3_handler(TSCont contp, TSEvent event, void *edata) static bool read_config_option(int argc, const char *argv[], int &raw, int &log) { - static const struct option longopts[] = {{"ja3raw", no_argument, &raw, 1}, {"ja3log", no_argument, &log, 1}, {0, 0, 0, 0}}; + static const struct option longopts[] = { + {"ja3raw", no_argument, &raw, 1}, {"ja3log", no_argument, &log, 1}, {nullptr, 0, nullptr, 0}}; int opt = 0; while ((opt = getopt_long(argc, (char *const *)argv, "", longopts, nullptr)) >= 0) { diff --git a/plugins/experimental/magick/magick.cc b/plugins/experimental/magick/magick.cc index b4f3cb05d5f..abcadf82e35 100644 --- a/plugins/experimental/magick/magick.cc +++ b/plugins/experimental/magick/magick.cc @@ -169,7 +169,7 @@ struct EVPContext { EVP_MD_CTX_destroy(context); } - EVPContext(void) : context(EVP_MD_CTX_create()) { assert(nullptr != context); } + EVPContext() : context(EVP_MD_CTX_create()) { assert(nullptr != context); } }; struct EVPKey { @@ -181,7 +181,7 @@ struct EVPKey { EVP_PKEY_free(key); } - EVPKey(void) : key(EVP_PKEY_new()) { assert(nullptr != key); } + EVPKey() : key(EVP_PKEY_new()) { assert(nullptr != key); } bool assign(const char *const k) const @@ -250,7 +250,7 @@ struct Exception { info = DestroyExceptionInfo(info); } - Exception(void) : info(AcquireExceptionInfo()) { assert(nullptr != info); } + Exception() : info(AcquireExceptionInfo()) { assert(nullptr != info); } }; struct Image { @@ -262,7 +262,7 @@ struct Image { info = DestroyImageInfo(info); } - Image(void) : info(AcquireImageInfo()) { assert(nullptr != info); } + Image() : info(AcquireImageInfo()) { assert(nullptr != info); } }; struct Wand { @@ -278,17 +278,17 @@ struct Wand { } } - Wand(void) : wand(NewMagickWand()), blob(nullptr) { assert(nullptr != wand); } + Wand() : wand(NewMagickWand()), blob(nullptr) { assert(nullptr != wand); } void - clear(void) const + clear() const { assert(nullptr != wand); ClearMagickWand(wand); } std::string_view - get(void) + get() { assert(nullptr != wand); std::size_t length = 0; @@ -336,7 +336,7 @@ struct Wand { struct Core { ~Core() { MagickCoreTerminus(); } - Core(void) { MagickCoreGenesis("/tmp", MagickFalse); } + Core() { MagickCoreGenesis("/tmp", MagickFalse); } }; } // namespace magick @@ -362,7 +362,7 @@ struct QueryMap { } void - parse(void) + parse() { std::string_view key; std::size_t i = 0, j = 0; @@ -473,7 +473,7 @@ struct ImageTransform : TransformationPlugin { } void - handleInputComplete(void) override + handleInputComplete() override { TSDebug(PLUGIN_TAG, "handleInputComplete"); @@ -515,7 +515,7 @@ struct GlobalHookPlugin : GlobalPlugin { magick::EVPKey *key_; ThreadPool threadPool_; - ~GlobalHookPlugin() + ~GlobalHookPlugin() override { if (nullptr != key_) { delete key_; diff --git a/plugins/experimental/memcache/tsmemcache.cc b/plugins/experimental/memcache/tsmemcache.cc index a3cdebc1ae2..db03b23fc6b 100644 --- a/plugins/experimental/memcache/tsmemcache.cc +++ b/plugins/experimental/memcache/tsmemcache.cc @@ -962,7 +962,7 @@ MC::ascii_incr_decr_event(int event, void *data) } header.cas = ink_atomic_increment(&next_cas, 1); { - char *data = 0; + char *data = nullptr; int len = 0; // must be huge, why convert to a counter ?? if (cwvc->get_single_data((void **)&data, &len) < 0) { diff --git a/plugins/experimental/multiplexer/chunk-decoder.h b/plugins/experimental/multiplexer/chunk-decoder.h index 0cfb4644128..b0b5f2e0bce 100644 --- a/plugins/experimental/multiplexer/chunk-decoder.h +++ b/plugins/experimental/multiplexer/chunk-decoder.h @@ -55,10 +55,10 @@ class ChunkDecoder void parseSizeCharacter(const char); int parseSize(const char *, const int64_t); int decode(const TSIOBufferReader &); - bool isSizeState(void) const; + bool isSizeState() const; inline bool - isEnd(void) const + isEnd() const { return state_ == State::kEnd; } diff --git a/plugins/experimental/multiplexer/fetcher.h b/plugins/experimental/multiplexer/fetcher.h index a431d1e1529..c5abe54d71f 100644 --- a/plugins/experimental/multiplexer/fetcher.h +++ b/plugins/experimental/multiplexer/fetcher.h @@ -47,7 +47,7 @@ struct HttpParser { TSMBuffer buffer_; TSMLoc location_; - void destroyParser(void); + void destroyParser(); ~HttpParser() { @@ -56,7 +56,7 @@ struct HttpParser { destroyParser(); } - HttpParser(void) : parsed_(false), parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) + HttpParser() : parsed_(false), parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) { TSHttpHdrTypeSet(buffer_, location_, TS_HTTP_TYPE_RESPONSE); } @@ -64,7 +64,7 @@ struct HttpParser { bool parse(io::IO &); int - statusCode(void) const + statusCode() const { return static_cast(TSHttpHdrStatusGet(buffer_, location_)); } diff --git a/plugins/experimental/multiplexer/ts.h b/plugins/experimental/multiplexer/ts.h index 1e4f654a4ca..327c6b502f5 100644 --- a/plugins/experimental/multiplexer/ts.h +++ b/plugins/experimental/multiplexer/ts.h @@ -52,7 +52,7 @@ namespace io TSIOBufferDestroy(buffer); } - IO(void) : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) {} + IO() : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) {} IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) { assert(buffer != nullptr); } static IO *read(TSVConn, TSCont, const int64_t); diff --git a/plugins/experimental/mysql_remap/lib/iniparser.h b/plugins/experimental/mysql_remap/lib/iniparser.h index d9bb650f1ea..546a41fbea9 100644 --- a/plugins/experimental/mysql_remap/lib/iniparser.h +++ b/plugins/experimental/mysql_remap/lib/iniparser.h @@ -42,9 +42,9 @@ Includes ---------------------------------------------------------------------------*/ -#include -#include -#include +#include +#include +#include /* * The following #include is necessary on many Unixes but not Linux. diff --git a/plugins/experimental/mysql_remap/mysql_remap.cc b/plugins/experimental/mysql_remap/mysql_remap.cc index 6d31ecd798d..1cba55a4124 100644 --- a/plugins/experimental/mysql_remap/mysql_remap.cc +++ b/plugins/experimental/mysql_remap/mysql_remap.cc @@ -187,7 +187,7 @@ TSPluginInit(int argc, const char *argv[]) my_data *data = (my_data *)malloc(1 * sizeof(my_data)); TSPluginRegistrationInfo info; - bool reconnect = 1; + bool reconnect = true; info.plugin_name = const_cast(PLUGIN_NAME); info.vendor_name = const_cast("Apache Software Foundation"); diff --git a/plugins/experimental/prefetch/common.cc b/plugins/experimental/prefetch/common.cc index fa2c9cff815..4664eebc721 100644 --- a/plugins/experimental/prefetch/common.cc +++ b/plugins/experimental/prefetch/common.cc @@ -22,8 +22,8 @@ * @see common.h */ -#include -#include +#include +#include #include "common.h" diff --git a/plugins/experimental/prefetch/configs.cc b/plugins/experimental/prefetch/configs.cc index 721acd3721b..89792602d07 100644 --- a/plugins/experimental/prefetch/configs.cc +++ b/plugins/experimental/prefetch/configs.cc @@ -54,19 +54,19 @@ isTrue(const char *arg) bool PrefetchConfig::init(int argc, char *argv[]) { - static const struct option longopt[] = {{const_cast("front"), optional_argument, 0, 'f'}, - {const_cast("api-header"), optional_argument, 0, 'h'}, - {const_cast("next-header"), optional_argument, 0, 'n'}, - {const_cast("fetch-policy"), optional_argument, 0, 'p'}, - {const_cast("fetch-count"), optional_argument, 0, 'c'}, - {const_cast("fetch-path-pattern"), optional_argument, 0, 'e'}, - {const_cast("fetch-max"), optional_argument, 0, 'x'}, - {const_cast("replace-host"), optional_argument, 0, 'r'}, - {const_cast("name-space"), optional_argument, 0, 's'}, - {const_cast("metrics-prefix"), optional_argument, 0, 'm'}, - {const_cast("exact-match"), optional_argument, 0, 'y'}, - {const_cast("log-name"), optional_argument, 0, 'l'}, - {0, 0, 0, 0}}; + static const struct option longopt[] = {{const_cast("front"), optional_argument, nullptr, 'f'}, + {const_cast("api-header"), optional_argument, nullptr, 'h'}, + {const_cast("next-header"), optional_argument, nullptr, 'n'}, + {const_cast("fetch-policy"), optional_argument, nullptr, 'p'}, + {const_cast("fetch-count"), optional_argument, nullptr, 'c'}, + {const_cast("fetch-path-pattern"), optional_argument, nullptr, 'e'}, + {const_cast("fetch-max"), optional_argument, nullptr, 'x'}, + {const_cast("replace-host"), optional_argument, nullptr, 'r'}, + {const_cast("name-space"), optional_argument, nullptr, 's'}, + {const_cast("metrics-prefix"), optional_argument, nullptr, 'm'}, + {const_cast("exact-match"), optional_argument, nullptr, 'y'}, + {const_cast("log-name"), optional_argument, nullptr, 'l'}, + {nullptr, 0, nullptr, 0}}; bool status = true; optind = 0; diff --git a/plugins/experimental/prefetch/fetch.cc b/plugins/experimental/prefetch/fetch.cc index d63e3af704f..49445c3d713 100644 --- a/plugins/experimental/prefetch/fetch.cc +++ b/plugins/experimental/prefetch/fetch.cc @@ -25,9 +25,9 @@ #include #include #include -#include +#include #include -#include +#include #include "ts/ts.h" /* ATS API */ #include "fetch.h" diff --git a/plugins/experimental/prefetch/fetch_policy.cc b/plugins/experimental/prefetch/fetch_policy.cc index 083257f8fa3..14b98ff1d18 100644 --- a/plugins/experimental/prefetch/fetch_policy.cc +++ b/plugins/experimental/prefetch/fetch_policy.cc @@ -23,7 +23,7 @@ #include "fetch_policy.h" -#include +#include #include "common.h" #include "fetch_policy_lru.h" diff --git a/plugins/experimental/prefetch/fetch_policy.h b/plugins/experimental/prefetch/fetch_policy.h index 85947597629..cd91ec2570b 100644 --- a/plugins/experimental/prefetch/fetch_policy.h +++ b/plugins/experimental/prefetch/fetch_policy.h @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include diff --git a/plugins/experimental/prefetch/fetch_policy_lru.h b/plugins/experimental/prefetch/fetch_policy_lru.h index 26996475673..391023497c9 100644 --- a/plugins/experimental/prefetch/fetch_policy_lru.h +++ b/plugins/experimental/prefetch/fetch_policy_lru.h @@ -87,15 +87,15 @@ class FetchPolicyLru : public FetchPolicy public: /* Default size values are also considered minimum. TODO: find out if this works OK. */ FetchPolicyLru() : _maxSize(10), _size(0){}; - virtual ~FetchPolicyLru(){}; + ~FetchPolicyLru() override{}; /* Fetch policy interface methods */ - bool init(const char *parameters); - bool acquire(const std::string &url); - bool release(const std::string &url); - const char *name(); - size_t getMaxSize(); - size_t getSize(); + bool init(const char *parameters) override; + bool acquire(const std::string &url) override; + bool release(const std::string &url) override; + const char *name() override; + size_t getMaxSize() override; + size_t getSize() override; protected: LruMap _map; diff --git a/plugins/experimental/prefetch/fetch_policy_simple.h b/plugins/experimental/prefetch/fetch_policy_simple.h index be04d86dcb2..b07262ea9aa 100644 --- a/plugins/experimental/prefetch/fetch_policy_simple.h +++ b/plugins/experimental/prefetch/fetch_policy_simple.h @@ -33,13 +33,13 @@ class FetchPolicySimple : public FetchPolicy { public: FetchPolicySimple() {} - virtual ~FetchPolicySimple(){}; - bool init(const char *parameters); - bool acquire(const std::string &url); - bool release(const std::string &url); - const char *name(); - size_t getSize(); - size_t getMaxSize(); + ~FetchPolicySimple() override{}; + bool init(const char *parameters) override; + bool acquire(const std::string &url) override; + bool release(const std::string &url) override; + const char *name() override; + size_t getSize() override; + size_t getMaxSize() override; private: std::unordered_map _urls; diff --git a/plugins/experimental/prefetch/headers.cc b/plugins/experimental/prefetch/headers.cc index 8233dc54063..fbfe7329482 100644 --- a/plugins/experimental/prefetch/headers.cc +++ b/plugins/experimental/prefetch/headers.cc @@ -21,8 +21,8 @@ * @brief HTTP headers manipulation. */ -#include -#include +#include +#include #include "configs.h" #include "headers.h" diff --git a/plugins/experimental/prefetch/pattern.cc b/plugins/experimental/prefetch/pattern.cc index 20e3b64688c..25541a3195d 100644 --- a/plugins/experimental/prefetch/pattern.cc +++ b/plugins/experimental/prefetch/pattern.cc @@ -84,7 +84,7 @@ Pattern::init(const String &config) size_t next = 1; do { current = next + 1; - next = config.find_first_of("/", current); + next = config.find_first_of('/', current); } while (next != String::npos && '\\' == config[next - 1]); if (next != String::npos) { @@ -98,7 +98,7 @@ Pattern::init(const String &config) start = next + 1; do { current = next + 1; - next = config.find_first_of("/", current); + next = config.find_first_of('/', current); } while (next != String::npos && '\\' == config[next - 1]); if (next != String::npos) { @@ -393,8 +393,8 @@ Pattern::compile() */ MultiPattern::~MultiPattern() { - for (std::vector::iterator p = this->_list.begin(); p != this->_list.end(); ++p) { - delete (*p); + for (auto &p : this->_list) { + delete p; } } @@ -428,8 +428,8 @@ MultiPattern::add(Pattern *pattern) bool MultiPattern::match(const String &subject) const { - for (std::vector::const_iterator p = this->_list.begin(); p != this->_list.end(); ++p) { - if (nullptr != (*p) && (*p)->match(subject)) { + for (auto p : this->_list) { + if (nullptr != p && p->match(subject)) { return true; } } @@ -445,8 +445,8 @@ MultiPattern::match(const String &subject) const bool MultiPattern::replace(const String &subject, String &result) const { - for (std::vector::const_iterator p = this->_list.begin(); p != this->_list.end(); ++p) { - if (nullptr != (*p) && (*p)->replace(subject, result)) { + for (auto p : this->_list) { + if (nullptr != p && p->replace(subject, result)) { return true; } } diff --git a/plugins/experimental/prefetch/pattern.h b/plugins/experimental/prefetch/pattern.h index 2db76656c77..b9270c8c755 100644 --- a/plugins/experimental/prefetch/pattern.h +++ b/plugins/experimental/prefetch/pattern.h @@ -73,7 +73,7 @@ class Pattern class MultiPattern { public: - MultiPattern(const String name = "") : _name(name) {} + MultiPattern(const String &name = "") : _name(name) {} virtual ~MultiPattern(); bool empty() const; diff --git a/plugins/experimental/prefetch/plugin.cc b/plugins/experimental/prefetch/plugin.cc index e71d039695a..f111220050b 100644 --- a/plugins/experimental/prefetch/plugin.cc +++ b/plugins/experimental/prefetch/plugin.cc @@ -195,7 +195,7 @@ evaluate(const String &v) /* Find out if width is specified (hence leading zeros are required if the width is bigger then the result width) */ String stmt; size_t len = 0; - size_t pos = v.find_first_of(":"); + size_t pos = v.find_first_of(':'); if (String::npos != pos) { stmt.assign(v.substr(0, pos)); len = getValue(v.substr(pos + 1)); @@ -237,8 +237,8 @@ expand(String &s) { size_t cur = 0; while (String::npos != cur) { - size_t start = s.find_first_of("{", cur); - size_t stop = s.find_first_of("}", start); + size_t start = s.find_first_of('{', cur); + size_t stop = s.find_first_of('}', start); if (String::npos != start && String::npos != stop) { s.replace(start, stop - start + 1, evaluate(s.substr(start + 1, stop - start - 1))); diff --git a/plugins/experimental/slice/Config.cc b/plugins/experimental/slice/Config.cc index c8b51b0496e..27ec5aa89ab 100644 --- a/plugins/experimental/slice/Config.cc +++ b/plugins/experimental/slice/Config.cc @@ -76,7 +76,7 @@ Config::fromArgs(int const argc, char const *const argv[], char *const errbuf, i for (int index = 0; index < argc; ++index) { std::string const argstr = argv[index]; - std::size_t const spos = argstr.find_first_of(":"); + std::size_t const spos = argstr.find_first_of(':'); if (spos != std::string::npos) { std::string key = argstr.substr(0, spos); std::string val = argstr.substr(spos + 1); diff --git a/plugins/experimental/slice/Data.cc b/plugins/experimental/slice/Data.cc index a901867808b..aac6943d47b 100644 --- a/plugins/experimental/slice/Data.cc +++ b/plugins/experimental/slice/Data.cc @@ -36,7 +36,7 @@ monitor() { std::lock_guard guard(mutex); // while (0 < inplay) - while (1) { + while (true) { mutex.unlock(); std::this_thread::sleep_for(std::chrono::seconds(10)); std::cerr << "Inplay: " << inplay << std::endl; diff --git a/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc b/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc index 8523b097acd..9977657370b 100644 --- a/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc +++ b/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc @@ -22,7 +22,7 @@ */ -#include +#include #include #include #include diff --git a/plugins/experimental/ssl_session_reuse/src/config.cc b/plugins/experimental/ssl_session_reuse/src/config.cc index fd3dde1fb16..48e2982926c 100644 --- a/plugins/experimental/ssl_session_reuse/src/config.cc +++ b/plugins/experimental/ssl_session_reuse/src/config.cc @@ -23,7 +23,7 @@ */ #include -#include +#include #include #include #include @@ -105,7 +105,7 @@ Config::setLastConfigChange() bool Config::configHasChanged() { - time_t checkTime = time(0) / cCheckDivisor; + time_t checkTime = time(nullptr) / cCheckDivisor; if (0 == m_lastmtime || m_lastCheck != checkTime) { m_lastCheck = checkTime; diff --git a/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc b/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc index 18efc9b1fe1..2909e728591 100644 --- a/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc +++ b/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc @@ -26,10 +26,10 @@ #include #include #include -#include +#include #include -#include -#include +#include +#include #include #include #include diff --git a/plugins/experimental/ssl_session_reuse/src/publish.cc b/plugins/experimental/ssl_session_reuse/src/publish.cc index c0235b310c8..d4df6482574 100644 --- a/plugins/experimental/ssl_session_reuse/src/publish.cc +++ b/plugins/experimental/ssl_session_reuse/src/publish.cc @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include #include "common.h" diff --git a/plugins/experimental/ssl_session_reuse/src/session_process.cc b/plugins/experimental/ssl_session_reuse/src/session_process.cc index bd48aef95a8..440cd66bccb 100644 --- a/plugins/experimental/ssl_session_reuse/src/session_process.cc +++ b/plugins/experimental/ssl_session_reuse/src/session_process.cc @@ -22,9 +22,9 @@ */ -#include -#include -#include +#include +#include +#include #include #include #include @@ -191,7 +191,7 @@ decrypt_session(const std::string &encrypted_data, const unsigned char *key, int } int -decode_id(std::string encoded_id, char *decoded_data, int &decoded_data_len) +decode_id(const std::string &encoded_id, char *decoded_data, int &decoded_data_len) { size_t decode_len = 0; memset(decoded_data, 0, decoded_data_len); diff --git a/plugins/experimental/ssl_session_reuse/src/session_process.h b/plugins/experimental/ssl_session_reuse/src/session_process.h index 8bd9cc3346f..a676db2b3ab 100644 --- a/plugins/experimental/ssl_session_reuse/src/session_process.h +++ b/plugins/experimental/ssl_session_reuse/src/session_process.h @@ -24,7 +24,7 @@ #pragma once #include -#include +#include #define SSL_SESSION_MAX_DER 1024 * 10 @@ -42,6 +42,6 @@ int decrypt_session(const std::string &encrypted_data, const unsigned char *key, int32_t &session_len); int encode_id(const char *id, int idlen, std::string &decoded_data); -int decode_id(std::string encoded_id, char *decoded_data, int &decoded_data_len); +int decode_id(const std::string &encoded_id, char *decoded_data, int &decoded_data_len); int add_session(char *session_id, int session_id_len, const std::string &encrypted_session); diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc index 2333eb39129..0d87f5a9d0f 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc @@ -23,7 +23,7 @@ */ #include -#include +#include #include "ssl_utils.h" #include "Config.h" #include "common.h" diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc b/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc index 596c8dc7f0d..90e61d83d37 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc @@ -23,14 +23,14 @@ */ #include -#include +#include #include #include #include #include #include "ssl_utils.h" -#include "assert.h" +#include #include "redis_auth.h" #include "stek.h" #include "common.h" @@ -303,7 +303,7 @@ STEK_Update_Setter_Thread(void *arg) stek_master_setter_running = 1; TSDebug(PLUGIN, "Will now act as the STEK rotator for pod"); - while (1) { + while (true) { // Create new STEK, set it for me, and then send it to the POD. if ((!STEK_CreateNew(&newKey, 0, 1 /* entropy ensured */)) || (!STEK_Send_To_Network(&newKey))) { // Error occurred. We will retry after a short interval. @@ -381,7 +381,7 @@ STEK_Update_Checker_Thread(void *arg) lastChangeTime = lastWarningTime = time(¤tTime); // init to current to supress a startup warning. - while (1) { + while (true) { if (!stek_initialized && ssl_param.pub) { // Launch a request for the master to resend you the ticket key std::string redis_channel = ssl_param.cluster_name + "." + STEK_ID_RESEND; diff --git a/plugins/experimental/ssl_session_reuse/src/subscriber.cc b/plugins/experimental/ssl_session_reuse/src/subscriber.cc index 6f51709c694..5df7855a90c 100644 --- a/plugins/experimental/ssl_session_reuse/src/subscriber.cc +++ b/plugins/experimental/ssl_session_reuse/src/subscriber.cc @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include "common.h" #include "subscriber.h" diff --git a/plugins/experimental/sslheaders/util.cc b/plugins/experimental/sslheaders/util.cc index 0ccf954621b..14e215ffb1c 100644 --- a/plugins/experimental/sslheaders/util.cc +++ b/plugins/experimental/sslheaders/util.cc @@ -78,9 +78,9 @@ SslHdrParseExpansion(const char *spec, SslHdrExpansion &exp) // Push sep to point to the field selector. selector = sep + 1; - for (unsigned i = 0; i < fields.size(); ++i) { - if (strcmp(selector, fields[i].name) == 0) { - exp.field = fields[i].field; + for (auto field : fields) { + if (strcmp(selector, field.name) == 0) { + exp.field = field.field; return true; } } diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc b/plugins/experimental/traffic_dump/traffic_dump.cc index a66b90874c6..86159169f77 100644 --- a/plugins/experimental/traffic_dump/traffic_dump.cc +++ b/plugins/experimental/traffic_dump/traffic_dump.cc @@ -18,7 +18,7 @@ limitations under the License. */ -#include +#include #include #include @@ -28,7 +28,7 @@ #include #include -#include +#include #include #include diff --git a/plugins/header_rewrite/condition.h b/plugins/header_rewrite/condition.h index 081d04f62ce..88f9917dc7a 100644 --- a/plugins/header_rewrite/condition.h +++ b/plugins/header_rewrite/condition.h @@ -49,7 +49,7 @@ class Condition : public Statement public: Condition() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for Condition"); } - virtual ~Condition() + ~Condition() override { TSDebug(PLUGIN_NAME_DBG, "Calling DTOR for Condition"); delete _matcher; diff --git a/plugins/header_rewrite/header_rewrite.cc b/plugins/header_rewrite/header_rewrite.cc index 3afdcc86425..a385d913b3a 100644 --- a/plugins/header_rewrite/header_rewrite.cc +++ b/plugins/header_rewrite/header_rewrite.cc @@ -42,13 +42,13 @@ initGeoIP() { GeoIPDBTypes dbs[] = {GEOIP_COUNTRY_EDITION, GEOIP_COUNTRY_EDITION_V6, GEOIP_ASNUM_EDITION, GEOIP_ASNUM_EDITION_V6}; - for (unsigned i = 0; i < sizeof(dbs) / sizeof(dbs[0]); ++i) { - if (!gGeoIP[dbs[i]] && GeoIP_db_avail(dbs[i])) { + for (auto &db : dbs) { + if (!gGeoIP[db] && GeoIP_db_avail(db)) { // GEOIP_STANDARD seems to break threaded apps... - gGeoIP[dbs[i]] = GeoIP_open_type(dbs[i], GEOIP_MMAP_CACHE); + gGeoIP[db] = GeoIP_open_type(db, GEOIP_MMAP_CACHE); - char *db_info = GeoIP_database_info(gGeoIP[dbs[i]]); - TSDebug(PLUGIN_NAME, "initialized GeoIP-DB[%d] %s", dbs[i], db_info); + char *db_info = GeoIP_database_info(gGeoIP[db]); + TSDebug(PLUGIN_NAME, "initialized GeoIP-DB[%d] %s", db, db_info); free(db_info); } } diff --git a/plugins/header_rewrite/matcher.h b/plugins/header_rewrite/matcher.h index 20e99b62cbf..83bb459072c 100644 --- a/plugins/header_rewrite/matcher.h +++ b/plugins/header_rewrite/matcher.h @@ -67,7 +67,7 @@ template class Matchers : public Matcher }; void - setRegex(const std::string /* data ATS_UNUSED */) + setRegex(const std::string & /* data ATS_UNUSED */) { if (!helper.setRegexMatch(_data)) { std::stringstream ss; diff --git a/plugins/header_rewrite/value.cc b/plugins/header_rewrite/value.cc index 4ddfcce3adc..5059a009755 100644 --- a/plugins/header_rewrite/value.cc +++ b/plugins/header_rewrite/value.cc @@ -33,8 +33,8 @@ Value::~Value() { TSDebug(PLUGIN_NAME_DBG, "Calling DTOR for Value"); - for (auto it = _cond_vals.begin(); it != _cond_vals.end(); it++) { - delete *it; + for (auto &_cond_val : _cond_vals) { + delete _cond_val; } } @@ -47,8 +47,7 @@ Value::set_value(const std::string &val) SimpleTokenizer tokenizer(_value); auto tokens = tokenizer.get_tokens(); - for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string token = *it; + for (auto token : tokens) { Condition *tcond_val = nullptr; if (token.substr(0, 2) == "%{") { diff --git a/plugins/header_rewrite/value.h b/plugins/header_rewrite/value.h index 13821cd6d3b..c237abba4ff 100644 --- a/plugins/header_rewrite/value.h +++ b/plugins/header_rewrite/value.h @@ -41,7 +41,7 @@ class Value : Statement public: Value() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for Value"); } - virtual ~Value(); + ~Value() override; // noncopyable Value(const Value &) = delete; @@ -53,8 +53,8 @@ class Value : Statement append_value(std::string &s, const Resources &res) const { if (!_cond_vals.empty()) { - for (auto it = _cond_vals.begin(); it != _cond_vals.end(); it++) { - (*it)->append_value(s, res); + for (auto _cond_val : _cond_vals) { + _cond_val->append_value(s, res); } } else { s += _value; diff --git a/proxy/HostStatus.h b/proxy/HostStatus.h index c9e14431f5f..486b8f3c36a 100644 --- a/proxy/HostStatus.h +++ b/proxy/HostStatus.h @@ -30,7 +30,7 @@ #pragma once -#include +#include #include #include "tscore/ink_rwlock.h" #include "records/P_RecProcess.h" diff --git a/proxy/RegressionSM.cc b/proxy/RegressionSM.cc index ac83a697184..a6f28ebe88c 100644 --- a/proxy/RegressionSM.cc +++ b/proxy/RegressionSM.cc @@ -229,8 +229,8 @@ RegressionSM::RegressionSM(const RegressionSM &ao) : Continuation(ao) parent = &o; nwaiting = o.nwaiting; - for (unsigned i = 0; i < o.children.size(); i++) { - children.push_back(o.children[i]->clone()); + for (auto &i : o.children) { + children.push_back(i->clone()); } n = o.n; diff --git a/proxy/StatPages.h b/proxy/StatPages.h index e337cbb65ee..3d204f5f325 100644 --- a/proxy/StatPages.h +++ b/proxy/StatPages.h @@ -97,17 +97,17 @@ class BaseStatPagesHandler : public Continuation ~BaseStatPagesHandler() override { resp_clear(); }; protected: - inkcoreapi void resp_clear(void); + inkcoreapi void resp_clear(); inkcoreapi void resp_add(const char *fmt, ...); - inkcoreapi void resp_add_sep(void); + inkcoreapi void resp_add_sep(); inkcoreapi void resp_begin(const char *title); - inkcoreapi void resp_end(void); - void resp_begin_numbered(void); - void resp_end_numbered(void); - inkcoreapi void resp_begin_unnumbered(void); - inkcoreapi void resp_end_unnumbered(void); - inkcoreapi void resp_begin_item(void); - void resp_end_item(void); + inkcoreapi void resp_end(); + void resp_begin_numbered(); + void resp_end_numbered(); + inkcoreapi void resp_begin_unnumbered(); + inkcoreapi void resp_end_unnumbered(); + inkcoreapi void resp_begin_item(); + void resp_end_item(); inkcoreapi void resp_begin_table(int border, int columns, int percent); inkcoreapi void resp_end_table(); inkcoreapi void resp_begin_row(); diff --git a/proxy/TransformInternal.h b/proxy/TransformInternal.h index 754b241aa51..1fdbc706a86 100644 --- a/proxy/TransformInternal.h +++ b/proxy/TransformInternal.h @@ -55,21 +55,21 @@ class TransformVConnection : public TransformVCChain { public: TransformVConnection(Continuation *cont, APIHook *hooks); - ~TransformVConnection(); + ~TransformVConnection() override; int handle_event(int event, void *edata); - VIO *do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf); - VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner = false); - void do_io_close(int lerrno = -1); - void do_io_shutdown(ShutdownHowTo_t howto); + VIO *do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) override; + VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner = false) override; + void do_io_close(int lerrno = -1) override; + void do_io_shutdown(ShutdownHowTo_t howto) override; - void reenable(VIO *vio); + void reenable(VIO *vio) override; /** Compute the backlog. @return The actual backlog, or a value at least @a limit. */ - virtual uint64_t backlog(uint64_t limit = UINT64_MAX); + uint64_t backlog(uint64_t limit = UINT64_MAX) override; public: VConnection *m_transform; @@ -96,7 +96,7 @@ class NullTransform : public INKVConnInternal { public: NullTransform(ProxyMutex *mutex); - ~NullTransform(); + ~NullTransform() override; int handle_event(int event, void *edata); @@ -111,7 +111,7 @@ class RangeTransform : public INKVConnInternal public: RangeTransform(ProxyMutex *mutex, RangeRecord *ranges, int num_fields, HTTPHdr *transform_resp, const char *content_type, int content_type_len, int64_t content_length); - ~RangeTransform(); + ~RangeTransform() override; int handle_event(int event, void *edata); diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index edbc5781254..5dfd44d8051 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -707,25 +707,25 @@ HdrHeap::marshal(char *buf, int len) str_heaps++; } - for (unsigned i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { - if (m_ronly_heap[i].m_heap_start != nullptr) { - if (m_ronly_heap[i].m_heap_len > len) { + for (auto &i : m_ronly_heap) { + if (i.m_heap_start != nullptr) { + if (i.m_heap_len > len) { goto Failed; } - memcpy(b, m_ronly_heap[i].m_heap_start, m_ronly_heap[i].m_heap_len); + memcpy(b, i.m_heap_start, i.m_heap_len); // Add translation table entry for string heaps // FIX ME - possible offset overflow issues? - str_xlation[str_heaps].start = m_ronly_heap[i].m_heap_start; - str_xlation[str_heaps].end = m_ronly_heap[i].m_heap_start + m_ronly_heap[i].m_heap_len; + str_xlation[str_heaps].start = i.m_heap_start; + str_xlation[str_heaps].end = i.m_heap_start + i.m_heap_len; str_xlation[str_heaps].offset = str_xlation[str_heaps].start - (b - buf); ink_assert(str_xlation[str_heaps].start <= str_xlation[str_heaps].end); str_heaps++; - b += m_ronly_heap[i].m_heap_len; - len -= m_ronly_heap[i].m_heap_len; - str_size += m_ronly_heap[i].m_heap_len; + b += i.m_heap_len; + len -= i.m_heap_len; + str_size += i.m_heap_len; } } @@ -1024,10 +1024,10 @@ HdrHeap::inherit_string_heaps(const HdrHeap *inherit_from) free_slots--; inherit_str_size = inherit_from->m_read_write_heap->m_heap_size; } - for (unsigned index = 0; index < HDR_BUF_RONLY_HEAPS; ++index) { - if (inherit_from->m_ronly_heap[index].m_heap_start != nullptr) { + for (const auto &index : inherit_from->m_ronly_heap) { + if (index.m_heap_start != nullptr) { free_slots--; - inherit_str_size += inherit_from->m_ronly_heap[index].m_heap_len; + inherit_str_size += index.m_heap_len; } else { // Heaps are allocated from the front of the array, so if // we hit a NULL, we know we can stop diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index 1e892ef5826..2ba767c5746 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -921,7 +921,7 @@ set_negative_caching_list(const char *name, RecDataT dtype, RecData data, HttpCo } else if (n <= 0 || n >= HTTP_STATUS_NUMBER) { Error("Invalid status code '%.*s' for negative caching: out of range", static_cast(token.size()), token.data()); } else { - set[n] = 1; + set[n] = true; } } } diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 1f5375dc37f..9e2e5e43273 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -1112,4 +1112,4 @@ is_response_body_precluded(HTTPStatus status_code, int method) } } -inkcoreapi extern ink_time_t ink_local_time(void); +inkcoreapi extern ink_time_t ink_local_time(); diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 22199d17082..51901b83090 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -529,11 +529,11 @@ HttpTunnel::reset() { ink_assert(active == false); #ifdef DEBUG - for (int i = 0; i < MAX_PRODUCERS; ++i) { - ink_assert(producers[i].alive == false); + for (auto &producer : producers) { + ink_assert(producer.alive == false); } - for (int j = 0; j < MAX_CONSUMERS; ++j) { - ink_assert(consumers[j].alive == false); + for (auto &consumer : consumers) { + ink_assert(consumer.alive == false); } #endif diff --git a/proxy/http/HttpUpdateSM.h b/proxy/http/HttpUpdateSM.h index 67662937036..6c298225225 100644 --- a/proxy/http/HttpUpdateSM.h +++ b/proxy/http/HttpUpdateSM.h @@ -50,7 +50,7 @@ class HttpUpdateSM : public HttpSM HttpUpdateSM(); static HttpUpdateSM *allocate(); - void destroy(); + void destroy() override; Action *start_scheduled_update(Continuation *cont, HTTPHdr *req); @@ -61,9 +61,9 @@ class HttpUpdateSM : public HttpSM int cb_event; protected: - void handle_api_return(); - void set_next_state(); - int kill_this_async_hook(int event, void *data); + void handle_api_return() override; + void set_next_state() override; + int kill_this_async_hook(int event, void *data) override; }; inline HttpUpdateSM * diff --git a/proxy/http/remap/AclFiltering.h b/proxy/http/remap/AclFiltering.h index a4af7264fc0..b1ce7f824c1 100644 --- a/proxy/http/remap/AclFiltering.h +++ b/proxy/http/remap/AclFiltering.h @@ -64,7 +64,7 @@ struct src_ip_info_t { class acl_filter_rule { private: - void reset(void); + void reset(); public: acl_filter_rule *next; @@ -98,7 +98,7 @@ class acl_filter_rule ~acl_filter_rule(); void name(const char *_name = nullptr); int add_argv(int _argc, char *_argv[]); - void print(void); + void print(); static acl_filter_rule *find_byname(acl_filter_rule *list, const char *name); static void delete_byname(acl_filter_rule **list, const char *name); diff --git a/proxy/http/remap/UrlRewrite.cc b/proxy/http/remap/UrlRewrite.cc index 628c119fea0..6f2e83d1b95 100644 --- a/proxy/http/remap/UrlRewrite.cc +++ b/proxy/http/remap/UrlRewrite.cc @@ -144,8 +144,8 @@ void UrlRewrite::_destroyTable(std::unique_ptr &h_table) { if (h_table) { - for (auto it = h_table->begin(); it != h_table->end(); ++it) { - delete it->second; + for (auto &it : *h_table) { + delete it.second; } } } @@ -183,8 +183,8 @@ void UrlRewrite::PrintStore(MappingsStore &store) { if (store.hash_lookup) { - for (auto it = store.hash_lookup->begin(); it != store.hash_lookup->end(); ++it) { - it->second->Print(); + for (auto &it : *store.hash_lookup) { + it.second->Print(); } } diff --git a/proxy/http2/Http2DependencyTree.h b/proxy/http2/Http2DependencyTree.h index e7820d50341..60f2b59f5bb 100644 --- a/proxy/http2/Http2DependencyTree.h +++ b/proxy/http2/Http2DependencyTree.h @@ -177,7 +177,7 @@ template void Tree::_dump(Node *node, std::ostream &output) const { - output << "{ \"id\":\"" << node->id << "/" << node->weight << "/" << node->point << "/" << ((node->t != nullptr) ? "1" : "0") + output << R"({ "id":")" << node->id << "/" << node->weight << "/" << node->point << "/" << ((node->t != nullptr) ? "1" : "0") << "/" << ((node->active) ? "a" : "d") << "\","; // Dump the children output << " \"c\":["; diff --git a/proxy/http2/RegressionHPACK.cc b/proxy/http2/RegressionHPACK.cc index 7b2de58141a..e0cdadd6d50 100644 --- a/proxy/http2/RegressionHPACK.cc +++ b/proxy/http2/RegressionHPACK.cc @@ -47,7 +47,7 @@ const static struct { uint8_t *encoded_field; int encoded_field_len; int prefix; -} integer_test_case[] = {{10, (uint8_t *)"\x0A", 1, 5}, {1337, (uint8_t *)"\x1F\x9A\x0A", 3, 5}, {42, (uint8_t *)"\x2A", 1, 8}}; +} integer_test_case[] = {{10, (uint8_t *)"\x0A", 1, 5}, {1337, (uint8_t *)"\x1F\x9A\x0A", 3, 5}, {42, (uint8_t *)R"(*)", 1, 8}}; // Example: custom-key: custom-header const static struct { diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index ab1f4a43b06..a2df5f657c5 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -438,7 +438,7 @@ LogAccess::marshal_ip(char *dest, sockaddr const *ip) } inline int -LogAccess::unmarshal_with_map(int64_t code, char *dest, int len, Ptr map, const char *msg) +LogAccess::unmarshal_with_map(int64_t code, char *dest, int len, const Ptr &map, const char *msg) { long int codeStrLen = 0; @@ -951,7 +951,7 @@ LogAccess::unmarshal_ip_to_hex(char **buf, char *dest, int len) -------------------------------------------------------------------------*/ int -LogAccess::unmarshal_hierarchy(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_hierarchy(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -969,7 +969,7 @@ LogAccess::unmarshal_hierarchy(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_finish_status(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -987,7 +987,7 @@ LogAccess::unmarshal_finish_status(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_cache_code(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -1005,7 +1005,7 @@ LogAccess::unmarshal_cache_code(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_cache_hit_miss(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -1015,7 +1015,7 @@ LogAccess::unmarshal_cache_hit_miss(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_cache_write_code(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); diff --git a/proxy/logging/LogAccess.h b/proxy/logging/LogAccess.h index 2436178e665..1e052825fb7 100644 --- a/proxy/logging/LogAccess.h +++ b/proxy/logging/LogAccess.h @@ -300,14 +300,14 @@ class LogAccess static int unmarshal_ip(char **buf, IpEndpoint *dest); static int unmarshal_ip_to_str(char **buf, char *dest, int len); static int unmarshal_ip_to_hex(char **buf, char *dest, int len); - static int unmarshal_hierarchy(char **buf, char *dest, int len, Ptr map); - static int unmarshal_finish_status(char **buf, char *dest, int len, Ptr map); - static int unmarshal_cache_code(char **buf, char *dest, int len, Ptr map); - static int unmarshal_cache_hit_miss(char **buf, char *dest, int len, Ptr map); - static int unmarshal_cache_write_code(char **buf, char *dest, int len, Ptr map); + static int unmarshal_hierarchy(char **buf, char *dest, int len, const Ptr &map); + static int unmarshal_finish_status(char **buf, char *dest, int len, const Ptr &map); + static int unmarshal_cache_code(char **buf, char *dest, int len, const Ptr &map); + static int unmarshal_cache_hit_miss(char **buf, char *dest, int len, const Ptr &map); + static int unmarshal_cache_write_code(char **buf, char *dest, int len, const Ptr &map); static int unmarshal_client_protocol_stack(char **buf, char *dest, int len, Ptr map); - static int unmarshal_with_map(int64_t code, char *dest, int len, Ptr map, const char *msg = nullptr); + static int unmarshal_with_map(int64_t code, char *dest, int len, const Ptr &map, const char *msg = nullptr); static int unmarshal_record(char **buf, char *dest, int len); @@ -359,10 +359,10 @@ class LogAccess char *m_cache_lookup_url_canon_str; int m_cache_lookup_url_canon_len; - void validate_unmapped_url(void); - void validate_unmapped_url_path(void); + void validate_unmapped_url(); + void validate_unmapped_url_path(); - void validate_lookup_url(void); + void validate_lookup_url(); }; inline int diff --git a/proxy/logging/LogBuffer.h b/proxy/logging/LogBuffer.h index 3981c2907fc..1c64a18c4f6 100644 --- a/proxy/logging/LogBuffer.h +++ b/proxy/logging/LogBuffer.h @@ -261,9 +261,9 @@ class LogBufferList ~LogBufferList(); void add(LogBuffer *lb); - LogBuffer *get(void); + LogBuffer *get(); int - get_size(void) + get_size() { return m_size; } diff --git a/proxy/logging/LogField.h b/proxy/logging/LogField.h index 236258b141c..6ca2a64c18d 100644 --- a/proxy/logging/LogField.h +++ b/proxy/logging/LogField.h @@ -79,7 +79,7 @@ class LogField typedef int (LogAccess::*MarshalFunc)(char *buf); typedef int (*UnmarshalFunc)(char **buf, char *dest, int len); typedef int (*UnmarshalFuncWithSlice)(char **buf, char *dest, int len, LogSlice *slice); - typedef int (*UnmarshalFuncWithMap)(char **buf, char *dest, int len, Ptr map); + typedef int (*UnmarshalFuncWithMap)(char **buf, char *dest, int len, const Ptr &map); typedef void (LogAccess::*SetFunc)(char *buf, int len); enum Type { @@ -177,7 +177,7 @@ class LogField void set_aggregate_op(Aggregate agg_op); void update_aggregate(int64_t val); - static void init_milestone_container(void); + static void init_milestone_container(); static Container valid_container_name(char *name); static Aggregate valid_aggregate_name(char *name); static bool fieldlist_contains_aggregates(const char *fieldlist); diff --git a/src/traffic_cache_tool/CacheDefs.h b/src/traffic_cache_tool/CacheDefs.h index db5fd146483..1d42d481695 100644 --- a/src/traffic_cache_tool/CacheDefs.h +++ b/src/traffic_cache_tool/CacheDefs.h @@ -398,7 +398,7 @@ dir_from_offset(int64_t i, CacheDirEntry *seg) { #if DIR_DEPTH < 5 if (!i) - return 0; + return nullptr; return dir_in_seg(seg, i); #else i = i + ((i - 1) / (DIR_DEPTH - 1)); diff --git a/src/traffic_cache_tool/CacheTool.cc b/src/traffic_cache_tool/CacheTool.cc index 13973a51a3e..094538228f9 100644 --- a/src/traffic_cache_tool/CacheTool.cc +++ b/src/traffic_cache_tool/CacheTool.cc @@ -1219,7 +1219,7 @@ dir_check() } void -walk_bucket_chain(std::string devicePath) +walk_bucket_chain(const std::string &devicePath) { Cache cache; if ((err = cache.loadSpan(SpanFile))) { @@ -1237,7 +1237,7 @@ walk_bucket_chain(std::string devicePath) } void -Clear_Span(std::string devicePath) +Clear_Span(const std::string &devicePath) { Cache cache; if ((err = cache.loadSpan(SpanFile))) { @@ -1253,7 +1253,7 @@ Clear_Span(std::string devicePath) } void -Check_Freelist(std::string devicePath) +Check_Freelist(const std::string &devicePath) { Cache cache; if ((err = cache.loadSpan(SpanFile))) { diff --git a/src/traffic_layout/file_system.cc b/src/traffic_layout/file_system.cc index 644b68a90d8..cbfa14bcd7b 100644 --- a/src/traffic_layout/file_system.cc +++ b/src/traffic_layout/file_system.cc @@ -73,15 +73,15 @@ create_directory(const std::string &dir) int ret = 0, pos = 0, pos1 = 0; if ((s[0] == '.') || (s[0] == '/')) { - pos1 = s.find("/") + 1; + pos1 = s.find('/') + 1; } - pos = s.find("/", pos1); + pos = s.find('/', pos1); ret = mkdir(s.substr(0, pos).c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); pos1 = pos + 1; // create directory one layer by one layer while (true) { - pos = s.find("/", pos1); + pos = s.find('/', pos1); if ((size_t)pos == s.npos) { break; } @@ -192,7 +192,7 @@ filter_ts_files(const std::string &dir, const std::string &dst_path) // ----- filter traffic server related files ----- if (dir == LAYOUT_BINDIR || dir == LAYOUT_SBINDIR) { // check if executable is in the list of traffic server executables. If not, end the copying. - if (executables.find(dst_path.substr(dst_path.find_last_of("/") + 1)) == executables.end()) { + if (executables.find(dst_path.substr(dst_path.find_last_of('/') + 1)) == executables.end()) { return false; } } diff --git a/src/traffic_logcat/logcat.cc b/src/traffic_logcat/logcat.cc index 879938c47c0..b1cd96491fd 100644 --- a/src/traffic_logcat/logcat.cc +++ b/src/traffic_logcat/logcat.cc @@ -198,7 +198,7 @@ process_file(int in_fd, int out_fd) // see if there is an alternate format request from the command // line // - const char *alt_format = NULL; + const char *alt_format = nullptr; // convert the buffer to ascii entries and place onto stdout // if (header->fmt_fieldlist()) { @@ -218,7 +218,7 @@ open_output_file(char *output_file) if (access(output_file, F_OK)) { if (errno != ENOENT) { fprintf(stderr, "Error accessing output file %s: ", output_file); - perror(0); + perror(nullptr); file_desc = -1; } } else { @@ -235,7 +235,7 @@ open_output_file(char *output_file) if (file_desc < 0) { fprintf(stderr, "Error while opening output file %s: ", output_file); - perror(0); + perror(nullptr); } } @@ -303,7 +303,7 @@ main(int /* argc ATS_UNUSED */, const char *argv[]) int in_fd = open(file_arguments[i], O_RDONLY); if (in_fd < 0) { fprintf(stderr, "Error opening input file %s: ", file_arguments[i]); - perror(0); + perror(nullptr); error = DATA_PROCESSING_ERROR; } else { #if HAVE_POSIX_FADVISE diff --git a/src/traffic_logstats/logstats.cc b/src/traffic_logstats/logstats.cc index 263292d5061..52a5390ffd7 100644 --- a/src/traffic_logstats/logstats.cc +++ b/src/traffic_logstats/logstats.cc @@ -753,7 +753,7 @@ struct ExitStatus { } void - append(const std::string s) + append(const std::string &s) { ink_strlcat(notice, s.c_str(), sizeof(notice)); } diff --git a/src/traffic_server/CoreUtils.cc b/src/traffic_server/CoreUtils.cc index 15ddb38febc..0ae7ba39632 100644 --- a/src/traffic_server/CoreUtils.cc +++ b/src/traffic_server/CoreUtils.cc @@ -100,9 +100,9 @@ int program_counter = 0; #if defined(darwin) || defined(freebsd) || defined(solaris) || defined(openbsd) // FIXME: solaris x86 // TODO: Cleanup multiple includes -#include -#include -#include +#include +#include +#include #include "tscore/ink_platform.h" #include "CoreUtils.h" #endif /* darwin || freebsd || solaris */ diff --git a/src/traffic_server/CoreUtils.h b/src/traffic_server/CoreUtils.h index f3760d33722..b3d2a77135b 100644 --- a/src/traffic_server/CoreUtils.h +++ b/src/traffic_server/CoreUtils.h @@ -55,11 +55,11 @@ struct core_stack_state { #endif // linux check #if defined(darwin) || defined(freebsd) || defined(solaris) || defined(openbsd) // FIXME: solaris x86 -#include +#include #include -#include -#include -#include +#include +#include +#include #define NO_OF_ARGS \ 10 /* The argument depth up to which we would be looking into \ diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index c2ac86055b7..0454459159f 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -1777,28 +1777,28 @@ TShrtime() //////////////////////////////////////////////////////////////////// const char * -TSInstallDirGet(void) +TSInstallDirGet() { static std::string prefix = Layout::get()->prefix; return prefix.c_str(); } const char * -TSConfigDirGet(void) +TSConfigDirGet() { static std::string sysconfdir = RecConfigReadConfigDir(); return sysconfdir.c_str(); } const char * -TSRuntimeDirGet(void) +TSRuntimeDirGet() { static std::string runtimedir = RecConfigReadRuntimeDir(); return runtimedir.c_str(); } const char * -TSTrafficServerVersionGet(void) +TSTrafficServerVersionGet() { return traffic_server_version; } @@ -1820,7 +1820,7 @@ TSTrafficServerVersionGetPatch() } const char * -TSPluginDirGet(void) +TSPluginDirGet() { static std::string path = RecConfigReadPluginDir(); return path.c_str(); @@ -1962,7 +1962,7 @@ TSHandleMLocRelease(TSMBuffer bufp, TSMLoc parent, TSMLoc mloc) // TSMBuffer: pointers to HdrHeapSDKHandle objects TSMBuffer -TSMBufferCreate(void) +TSMBufferCreate() { TSMBuffer bufp; HdrHeapSDKHandle *new_heap = new HdrHeapSDKHandle; @@ -2456,7 +2456,7 @@ TSIpStringToAddr(const char *str, size_t str_len, sockaddr *addr) /**************/ TSMimeParser -TSMimeParserCreate(void) +TSMimeParserCreate() { TSMimeParser parser = reinterpret_cast(ats_malloc(sizeof(MIMEParser))); @@ -3547,7 +3547,7 @@ TSMimeHdrFieldValueDelete(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx) /* HttpParser */ /**************/ TSHttpParser -TSHttpParserCreate(void) +TSHttpParserCreate() { TSHttpParser parser = reinterpret_cast(ats_malloc(sizeof(HTTPParser))); http_parser_init((HTTPParser *)parser); @@ -4039,7 +4039,7 @@ sdk_sanity_check_cachekey(TSCacheKey key) } TSCacheKey -TSCacheKeyCreate(void) +TSCacheKeyCreate() { TSCacheKey key = (TSCacheKey) new CacheInfo(); @@ -4249,7 +4249,7 @@ TSCacheHttpInfoDestroy(TSCacheHttpInfo infop) } TSCacheHttpInfo -TSCacheHttpInfoCreate(void) +TSCacheHttpInfoCreate() { CacheHTTPInfo *info = new CacheHTTPInfo; info->create(); @@ -6367,7 +6367,7 @@ TSHttpTxnLookingUpTypeGet(TSHttpTxn txnp) } int -TSHttpCurrentClientConnectionsGet(void) +TSHttpCurrentClientConnectionsGet() { int64_t S; @@ -6376,7 +6376,7 @@ TSHttpCurrentClientConnectionsGet(void) } int -TSHttpCurrentActiveClientConnectionsGet(void) +TSHttpCurrentActiveClientConnectionsGet() { int64_t S; @@ -6385,7 +6385,7 @@ TSHttpCurrentActiveClientConnectionsGet(void) } int -TSHttpCurrentIdleClientConnectionsGet(void) +TSHttpCurrentIdleClientConnectionsGet() { int64_t total = 0; int64_t active = 0; @@ -6401,7 +6401,7 @@ TSHttpCurrentIdleClientConnectionsGet(void) } int -TSHttpCurrentCacheConnectionsGet(void) +TSHttpCurrentCacheConnectionsGet() { int64_t S; @@ -6410,7 +6410,7 @@ TSHttpCurrentCacheConnectionsGet(void) } int -TSHttpCurrentServerConnectionsGet(void) +TSHttpCurrentServerConnectionsGet() { int64_t S; @@ -7458,7 +7458,7 @@ TSMatcherExtractIPRange(char *match_str, sockaddr *addr1, sockaddr *addr2) } TSMatcherLine -TSMatcherLineCreate(void) +TSMatcherLineCreate() { return reinterpret_cast(ats_malloc(sizeof(matcher_line))); } @@ -9156,7 +9156,7 @@ TSSslSessionRemove(const TSSslSessionID *session_id) // APIs for managing and using UUIDs. TSUuid -TSUuidCreate(void) +TSUuidCreate() { ATSUuid *uuid = new ATSUuid(); return (TSUuid)uuid; @@ -9196,7 +9196,7 @@ TSUuidInitialize(TSUuid uuid, TSUuidVersion v) } TSUuid -TSProcessUuidGet(void) +TSProcessUuidGet() { Machine *machine = Machine::instance(); return (TSUuid)(&machine->uuid); diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index d0008386c78..a89b9c1b71a 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -160,7 +160,7 @@ static char *generate_response(const char *request); static int get_request_id(TSHttpTxn txnp); /* client side */ -static ClientTxn *synclient_txn_create(void); +static ClientTxn *synclient_txn_create(); static int synclient_txn_delete(ClientTxn *txn); static void synclient_txn_close(ClientTxn *txn); static int synclient_txn_send_request(ClientTxn *txn, char *request); @@ -503,7 +503,7 @@ get_response_id(TSHttpTxn txnp) ////////////////////////////////////////////////////////////////////////////// static ClientTxn * -synclient_txn_create(void) +synclient_txn_create() { const HttpProxyPort *proxy_port; diff --git a/src/traffic_top/stats.h b/src/traffic_top/stats.h index ffec58c0780..1789fd0ed02 100644 --- a/src/traffic_top/stats.h +++ b/src/traffic_top/stats.h @@ -73,7 +73,7 @@ class Stats // set the host size_t start = _url.find(":"); - size_t end = _url.find("/", start + 3); + size_t end = _url.find('/', start + 3); _host = _url.substr(start + 3, end - start - 3); end = _host.find(":"); if (end != string::npos) { diff --git a/src/traffic_wccp/wccp_client.cc b/src/traffic_wccp/wccp_client.cc index aca1bc94aad..95a16b4c172 100644 --- a/src/traffic_wccp/wccp_client.cc +++ b/src/traffic_wccp/wccp_client.cc @@ -20,9 +20,9 @@ limitations under the License. */ -#include +#include #include -#include +#include #include #include #include diff --git a/src/tscore/ArgParser.cc b/src/tscore/ArgParser.cc index 75f1f9fc217..b411f4c1e71 100644 --- a/src/tscore/ArgParser.cc +++ b/src/tscore/ArgParser.cc @@ -28,6 +28,7 @@ #include #include #include +#include #include std::string global_usage; @@ -199,7 +200,7 @@ ArgParser::require_commands() void ArgParser::set_error(std::string e) { - _error_msg = e; + _error_msg = std::move(e); } std::string diff --git a/src/tscore/ink_hrtime.cc b/src/tscore/ink_hrtime.cc index 3a328c21441..9d8eff8ef98 100644 --- a/src/tscore/ink_hrtime.cc +++ b/src/tscore/ink_hrtime.cc @@ -37,7 +37,7 @@ #include #include #endif -#include +#include #include char * diff --git a/src/tscore/runroot.cc b/src/tscore/runroot.cc index f444d99531d..7736a97629f 100644 --- a/src/tscore/runroot.cc +++ b/src/tscore/runroot.cc @@ -96,7 +96,7 @@ get_parent_yaml_path(const std::string &path) if (!yaml_file.empty()) { return yaml_file; } - whole_path = whole_path.substr(0, whole_path.find_last_of("/")); + whole_path = whole_path.substr(0, whole_path.find_last_of('/')); } return {}; } @@ -135,7 +135,7 @@ runroot_extra_handling(const char *executable, bool json) char RealBinPath[PATH_MAX] = {0}; if ((executable != nullptr) && realpath(executable, RealBinPath) != nullptr) { std::string bindir = RealBinPath; - bindir = bindir.substr(0, bindir.find_last_of("/")); // getting the bin dir not executable path + bindir = bindir.substr(0, bindir.find_last_of('/')); // getting the bin dir not executable path path = get_parent_yaml_path(bindir); if (!path.empty()) { runroot_file = path; @@ -242,7 +242,7 @@ runroot_map(const std::string &file) RunrootMapType map; try { YAML::Node yamlfile = YAML::LoadFile(file); - std::string prefix = file.substr(0, file.find_last_of("/")); + std::string prefix = file.substr(0, file.find_last_of('/')); for (const auto &it : yamlfile) { // key value pairs of dirs diff --git a/src/tscore/unit_tests/test_IntrusiveHashMap.cc b/src/tscore/unit_tests/test_IntrusiveHashMap.cc index 1675239b703..f8f5b882e23 100644 --- a/src/tscore/unit_tests/test_IntrusiveHashMap.cc +++ b/src/tscore/unit_tests/test_IntrusiveHashMap.cc @@ -112,9 +112,9 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") REQUIRE(map.bucket_count() > nb); for (auto &thing : map) { REQUIRE(0 == marks[thing._n]); - marks[thing._n] = 1; + marks[thing._n] = true; } - marks[0] = 1; + marks[0] = true; REQUIRE(marks.all()); map.insert(new Thing("dup"sv, 79)); map.insert(new Thing("dup"sv, 80)); diff --git a/src/tscore/unit_tests/test_PriorityQueue.cc b/src/tscore/unit_tests/test_PriorityQueue.cc index 81a1dd2ffa3..b5144044450 100644 --- a/src/tscore/unit_tests/test_PriorityQueue.cc +++ b/src/tscore/unit_tests/test_PriorityQueue.cc @@ -21,7 +21,7 @@ limitations under the License. */ -#include +#include #include "tscore/PriorityQueue.h" #include "catch.hpp" diff --git a/src/tscpp/api/TransformationPlugin.cc b/src/tscpp/api/TransformationPlugin.cc index 5d8507539be..524b07bae27 100644 --- a/src/tscpp/api/TransformationPlugin.cc +++ b/src/tscpp/api/TransformationPlugin.cc @@ -92,7 +92,7 @@ struct TransformationPluginState : noncopyable, public detail::ResumeAfterPauseC output_buffer_reader_ = TSIOBufferReaderAlloc(output_buffer_); }; - ~TransformationPluginState() + ~TransformationPluginState() override { if (output_buffer_reader_) { TSIOBufferReaderFree(output_buffer_reader_); From 031f321be814c71047a700b4b8c507382d51f709 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 19 Mar 2019 11:18:50 +0800 Subject: [PATCH 375/526] The response header of CONNECT should not have content-length or chunk-encoding --- proxy/http/HttpTransact.h | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 9e2e5e43273..af40b1530bd 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -1081,15 +1081,17 @@ class HttpTransact typedef void (*TransactEntryFunc_t)(HttpTransact::State *s); -//////////////////////////////////////////////////////// -// the spec says about message body the following: // -// All responses to the HEAD request method MUST NOT // -// include a message-body, even though the presence // -// of entity-header fields might lead one to believe // -// they do. All 1xx (informational), 204 (no content),// -// and 304 (not modified) responses MUST NOT include // -// a message-body. // -//////////////////////////////////////////////////////// +/* The spec says about message body the following: + * + * All responses to the HEAD and CONNECT request method + * MUST NOT include a message-body, even though the presence + * of entity-header fields might lead one to believe they do. + * + * All 1xx (informational), 204 (no content), and 304 (not modified) + * responses MUST NOT include a message-body. + * + * Refer : [https://tools.ietf.org/html/rfc7231#section-4.3.6] + */ inline bool is_response_body_precluded(HTTPStatus status_code) { @@ -1105,7 +1107,7 @@ is_response_body_precluded(HTTPStatus status_code) inline bool is_response_body_precluded(HTTPStatus status_code, int method) { - if ((method == HTTP_WKSIDX_HEAD) || is_response_body_precluded(status_code)) { + if ((method == HTTP_WKSIDX_HEAD) || (method == HTTP_WKSIDX_CONNECT) || is_response_body_precluded(status_code)) { return true; } else { return false; From 9b73fc965e209b7e846aa8699430cbeaf9ecfcad Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 20 Mar 2019 16:26:05 +0900 Subject: [PATCH 376/526] Exclude library headers from clang-tidy --- .clang-tidy | 1 + build/tidy.mk | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 1658bf3d379..a1ea4a36e28 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1 +1,2 @@ Checks: -*,modernize-loop-convert,modernize-use-bool-literals,modernize-deprecated-headers,performance-unnecessary-value-param,performance-faster-string-find,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-use-nullptr,modernize-use-override +HeaderFilterRegex: (?!(\/openssl\/|\/pcre\/|\/lib\/yaml\/)).* diff --git a/build/tidy.mk b/build/tidy.mk index 9829c0d1aac..26d82c14e02 100644 --- a/build/tidy.mk +++ b/build/tidy.mk @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -Clang_Tidy_Options = -fix -fix-errors -header-filter=.* +Clang_Tidy_Options = -fix -fix-errors # Sort the filenames to remove duplicates, then filter to retain # just the C and C++ sources so we don't pick up lex and yacc files From 8ba48435cef6ee862e7ff13658742a96450e0cf2 Mon Sep 17 00:00:00 2001 From: Valentin Gutierrez Date: Fri, 22 Mar 2019 13:14:55 +0100 Subject: [PATCH 377/526] fallback to curves list if available --- doc/admin-guide/files/records.config.en.rst | 4 ++-- iocore/net/SSLClientUtils.cc | 6 +++++- iocore/net/SSLUtils.cc | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index eb380e44545..d87e8425da5 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3209,7 +3209,7 @@ SSL Termination X25519:P-256:X448:P-521:P-384 - This configuration works with OpenSSL v1.1.1 and above. + This configuration works with OpenSSL v1.0.2 and above. .. ts:cv:: CONFIG proxy.config.ssl.client.groups_list STRING @@ -3219,7 +3219,7 @@ SSL Termination group NIDs or names, for example "P-521:P-384:P-256". For instructions, see "Groups" section of `TLS1.3 - OpenSSLWiki `_. - This configuration works with OpenSSL v1.1.1 and above. + This configuration works with OpenSSL v1.0.2 and above. .. ts:cv:: CONFIG proxy.config.ssl.TLSv1 INT 1 diff --git a/iocore/net/SSLClientUtils.cc b/iocore/net/SSLClientUtils.cc index bb048a3faee..2a0023f1f3a 100644 --- a/iocore/net/SSLClientUtils.cc +++ b/iocore/net/SSLClientUtils.cc @@ -169,9 +169,13 @@ SSLInitClientContext(const SSLConfigParams *params) } #endif -#ifdef SSL_CTX_set1_groups_list +#if defined(SSL_CTX_set1_groups_list) || defined(SSL_CTX_set1_curves_list) if (params->client_groups_list != nullptr) { +#ifdef SSL_CTX_set1_groups_list if (!SSL_CTX_set1_groups_list(client_ctx, params->client_groups_list)) { +#else + if (!SSL_CTX_set1_curves_list(client_ctx, params->client_groups_list)) { +#endif SSLError("invalid groups list for client in records.config"); goto fail; } diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 41411ab8c92..38009d5bc90 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -1333,9 +1333,13 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co } #endif -#ifdef SSL_CTX_set1_groups_list +#if defined(SSL_CTX_set1_groups_list) || defined(SSL_CTX_set1_curves_list) if (params->server_groups_list != nullptr) { +#ifdef SSL_CTX_set1_groups_list if (!SSL_CTX_set1_groups_list(ctx, params->server_groups_list)) { +#else + if (!SSL_CTX_set1_curves_list(ctx, params->server_groups_list)) { +#endif SSLError("invalid groups list for server in records.config"); goto fail; } From a7d4ca23ce42fc0ad0084d779601910be21ab597 Mon Sep 17 00:00:00 2001 From: Emanuele Rocca Date: Mon, 25 Mar 2019 10:39:27 +0100 Subject: [PATCH 378/526] Doc: fix copy-paste leftover in ts.stat_find --- doc/admin-guide/plugins/lua.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin-guide/plugins/lua.en.rst b/doc/admin-guide/plugins/lua.en.rst index 67fc36c574a..dd890e36d1e 100644 --- a/doc/admin-guide/plugins/lua.en.rst +++ b/doc/admin-guide/plugins/lua.en.rst @@ -3738,7 +3738,7 @@ Here is an example. ts.stat_find ------------ -**syntax:** *val = ts.stat_create(STAT_NAME)* +**syntax:** *val = ts.stat_find(STAT_NAME)* **context:** global From 841b9c699a779cc5528c6861aacf29f229f290db Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Mon, 18 Mar 2019 16:45:42 -0500 Subject: [PATCH 379/526] use pthread_cancel --- .../ssl_session_reuse/src/ats_ssl_plugin.cc | 15 +++++++++ .../ssl_session_reuse/src/publish.cc | 5 +++ .../ssl_session_reuse/src/ssl_key_utils.cc | 8 +++++ .../ssl_session_reuse/src/ssl_utils.h | 33 +++++++++++++++++++ .../ssl_session_reuse/src/subscriber.cc | 4 +++ 5 files changed, 65 insertions(+) diff --git a/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc b/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc index 9977657370b..1a8e171a02e 100644 --- a/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc +++ b/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc @@ -28,7 +28,20 @@ #include #include "ssl_utils.h" + +PluginThreads plugin_threads; + int SSL_session_callback(TSCont contp, TSEvent event, void *edata); + +static int +shutdown_handler(TSCont contp, TSEvent event, void *edata) +{ + if ((event == TS_EVENT_LIFECYCLE_SHUTDOWN)) { + plugin_threads.terminate(); + } + return 0; +} + void TSPluginInit(int argc, const char *argv[]) { @@ -38,6 +51,8 @@ TSPluginInit(int argc, const char *argv[]) info.vendor_name = (char *)("ats"); info.support_email = (char *)("ats-devel@oath.com"); + TSLifecycleHookAdd(TS_LIFECYCLE_SHUTDOWN_HOOK, TSContCreate(shutdown_handler, nullptr)); + #if (TS_VERSION_NUMBER >= 7000000) if (TSPluginRegister(&info) != TS_SUCCESS) { TSError("Plugin registration failed."); diff --git a/plugins/experimental/ssl_session_reuse/src/publish.cc b/plugins/experimental/ssl_session_reuse/src/publish.cc index d4df6482574..a74664c7369 100644 --- a/plugins/experimental/ssl_session_reuse/src/publish.cc +++ b/plugins/experimental/ssl_session_reuse/src/publish.cc @@ -33,10 +33,15 @@ #include "publisher.h" #include "Config.h" #include "redis_auth.h" +#include "ssl_utils.h" void * RedisPublisher::start_worker_thread(void *arg) { + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + RedisPublisher *publisher = static_cast(arg); publisher->runWorker(); return arg; diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc b/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc index 90e61d83d37..323fe0768f1 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc @@ -284,6 +284,10 @@ STEK_Send_To_Network(struct ssl_ticket_key_t *stekToSend) static void * STEK_Update_Setter_Thread(void *arg) { + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + int sleepInterval; struct ssl_ticket_key_t newKey; int startProblem = 0; // counter for start up and retry issues. @@ -368,6 +372,10 @@ STEK_update(const std::string &encrypted_stek) static void * STEK_Update_Checker_Thread(void *arg) { + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + time_t currentTime; time_t lastWarningTime; // last time we put out a warning diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_utils.h b/plugins/experimental/ssl_session_reuse/src/ssl_utils.h index 5711ff831b2..bbfbf0c88b6 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_utils.h +++ b/plugins/experimental/ssl_session_reuse/src/ssl_utils.h @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include "publisher.h" #include "subscriber.h" @@ -45,6 +47,35 @@ struct ssl_session_param { ~ssl_session_param(); }; +class PluginThreads +{ +public: + void + store(const pthread_t &th) + { + std::lock_guard lock(threads_mutex); + threads_queue.push_back(th); + } + + void + terminate() + { + std::lock_guard lock(threads_mutex); + for (pthread_t th : threads_queue) { + ::pthread_cancel(th); + } + while (!threads_queue.empty()) { + pthread_t th = threads_queue.front(); + ::pthread_join(th, nullptr); + threads_queue.pop_front(); + } + } + +private: + std::deque threads_queue; + std::mutex threads_mutex; +}; + int STEK_init_keys(); const char *get_key_ptr(); @@ -66,3 +97,5 @@ int init_subscriber(); int SSL_session_callback(TSCont contp, TSEvent event, void *edata); extern ssl_session_param ssl_param; // almost everything one needs is stored in here + +extern PluginThreads plugin_threads; diff --git a/plugins/experimental/ssl_session_reuse/src/subscriber.cc b/plugins/experimental/ssl_session_reuse/src/subscriber.cc index 5df7855a90c..7dfc5560350 100644 --- a/plugins/experimental/ssl_session_reuse/src/subscriber.cc +++ b/plugins/experimental/ssl_session_reuse/src/subscriber.cc @@ -39,6 +39,10 @@ void * setup_subscriber(void *arg) { + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + RedisSubscriber *me = static_cast(arg); me->run(); return (void *)1; From 176cb8e552139d879c70711a47911b49af4565cd Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Thu, 21 Mar 2019 13:43:47 -0500 Subject: [PATCH 380/526] check return value --- .../experimental/ssl_session_reuse/src/config.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/experimental/ssl_session_reuse/src/config.cc b/plugins/experimental/ssl_session_reuse/src/config.cc index 48e2982926c..a2d94571c4a 100644 --- a/plugins/experimental/ssl_session_reuse/src/config.cc +++ b/plugins/experimental/ssl_session_reuse/src/config.cc @@ -62,7 +62,10 @@ Config::loadConfig(const std::string &filename) size_t n = info.st_size; std::string config_data; config_data.resize(n); - read(fd, const_cast(config_data.data()), n); + if (read(fd, const_cast(config_data.data()), n) < 0) { + close(fd); + return success; + } ts::TextView content(config_data); while (content) { @@ -77,10 +80,13 @@ Config::loadConfig(const std::string &filename) m_config[std::string(field.data(), field.size())] = std::string(line.data(), line.size()); } } + + close(fd); + + m_noConfig = false; + success = true; + m_alreadyLoaded = true; } - m_noConfig = false; - success = true; - m_alreadyLoaded = true; return success; } From 1c8688176b239334853dd8695788a8405758dcaa Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 18 Mar 2019 22:15:58 +0000 Subject: [PATCH 381/526] Regex name checks on ssl_server_name should be anchored. --- iocore/net/P_SSLSNI.h | 2 +- tests/gold_tests/tls/tls_client_cert2.test.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index 78d054eab53..14c23941251 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -77,7 +77,7 @@ struct namedElement { const char *err_ptr; int err_offset = 0; if (!regexName.empty()) { - match = pcre_compile(regexName.c_str(), 0, &err_ptr, &err_offset, nullptr); + match = pcre_compile(regexName.c_str(), PCRE_ANCHORED, &err_ptr, &err_offset, nullptr); } else { match = nullptr; } diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py index 05ee893b28e..125e9690418 100644 --- a/tests/gold_tests/tls/tls_client_cert2.test.py +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -96,6 +96,9 @@ '- fqdn: "*bar.com"', ' client_cert: {0}/signed2-bar.pem'.format(ts.Variables.SSLDir), ' client_key: {0}/signed-bar.key'.format(ts.Variables.SSLDir), + '- fqdn: "foo.com"', + ' client_cert: {0}/signed2-foo.pem'.format(ts.Variables.SSLDir), + ' client_key: {0}/signed-foo.key'.format(ts.Variables.SSLDir), ]) @@ -156,3 +159,21 @@ trfail.Processes.Default.ReturnCode = 0 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") +# Should fail +tr = Test.AddTestRun("random.foo.com to server 2") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:random.foo.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + +#Should fail +trfail = Test.AddTestRun("random.foo.com to server 1") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:random.foo.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + From 94ccc447d79bf8f61923df6f2536b1a24a6226e3 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Sat, 23 Mar 2019 19:45:18 +0800 Subject: [PATCH 382/526] Replace EThread::has_event_loop with EThread::tt == REGULAR --- iocore/eventsystem/I_EThread.h | 2 -- iocore/eventsystem/P_UnixEThread.h | 2 +- iocore/eventsystem/UnixEThread.cc | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index f80456e6981..8ce926ab1c8 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -325,8 +325,6 @@ class EThread : public Thread bool is_event_type(EventType et); void set_event_type(EventType et); - bool has_event_loop = false; - // Private Interface void execute() override; diff --git a/iocore/eventsystem/P_UnixEThread.h b/iocore/eventsystem/P_UnixEThread.h index cccda6bc6b6..834a6c923fe 100644 --- a/iocore/eventsystem/P_UnixEThread.h +++ b/iocore/eventsystem/P_UnixEThread.h @@ -183,7 +183,7 @@ TS_INLINE EThread * this_event_thread() { EThread *ethread = this_ethread(); - if (ethread != nullptr && ethread->has_event_loop) { + if (ethread != nullptr && ethread->tt == REGULAR) { return ethread; } else { return nullptr; diff --git a/iocore/eventsystem/UnixEThread.cc b/iocore/eventsystem/UnixEThread.cc index e8586fa73b3..06920dbe5b7 100644 --- a/iocore/eventsystem/UnixEThread.cc +++ b/iocore/eventsystem/UnixEThread.cc @@ -208,7 +208,6 @@ EThread::execute_regular() static EventMetrics METRIC_INIT; // give priority to immediate events - has_event_loop = true; for (;;) { if (unlikely(shutdown_event_system == true)) { return; From a9f01a8dbc860d40905acd843db19af790cc18fd Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Mon, 25 Mar 2019 09:33:05 -0700 Subject: [PATCH 383/526] Doc: Changing volume.config invalidates the cache --- doc/admin-guide/files/volume.config.en.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/admin-guide/files/volume.config.en.rst b/doc/admin-guide/files/volume.config.en.rst index ebae6d4e12a..7fdfc8e3381 100644 --- a/doc/admin-guide/files/volume.config.en.rst +++ b/doc/admin-guide/files/volume.config.en.rst @@ -52,6 +52,11 @@ do not allocate all the disk space in the cache, then the extra disk space is not used. You can use the extra space later to create new volumes without deleting and clearing the existing volumes. +.. important:: + + Changing this file to add, remove or modify volumes effectively invalidates + the cache. + Examples ======== From c9c59a833489fbe0373c558608f83449bdb250ba Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Mon, 25 Mar 2019 09:23:55 -0700 Subject: [PATCH 384/526] Moved unit test suppression file and updated it --- ci/asan_leak_suppression/unit_tests.txt | 6 ++++++ src/tscore/Makefile.am | 2 +- src/tscore/suppression.txt | 3 --- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 ci/asan_leak_suppression/unit_tests.txt delete mode 100644 src/tscore/suppression.txt diff --git a/ci/asan_leak_suppression/unit_tests.txt b/ci/asan_leak_suppression/unit_tests.txt new file mode 100644 index 00000000000..b75dec9a9c4 --- /dev/null +++ b/ci/asan_leak_suppression/unit_tests.txt @@ -0,0 +1,6 @@ +# leaks in test_X509HostnameValidator +leak:libcrypto.so.1.1 +# for OpenSSL 1.0.2: +leak:CRYPTO_malloc +leak:CRYPTO_realloc +leak:ConsCell diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index cd089720dad..1edb2ceb1bb 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -21,7 +21,7 @@ include $(top_srcdir)/build/tidy.mk noinst_PROGRAMS = mkdfa CompileParseRules check_PROGRAMS = test_atomic test_freelist test_geometry test_X509HostnameValidator test_tscore -TESTS_ENVIRONMENT = LSAN_OPTIONS=suppressions=suppression.txt +TESTS_ENVIRONMENT = LSAN_OPTIONS=suppressions=$(abs_top_srcdir)/ci/asan_leak_suppression/unit_tests.txt TESTS = $(check_PROGRAMS) diff --git a/src/tscore/suppression.txt b/src/tscore/suppression.txt deleted file mode 100644 index 7e7e361b85d..00000000000 --- a/src/tscore/suppression.txt +++ /dev/null @@ -1,3 +0,0 @@ -leak:CRYPTO_malloc -leak:CRYPTO_realloc -leak:ConsCell From 6e9cf2f88002fc73802c4b05c61749b81a518582 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 20 Mar 2019 16:30:18 +0900 Subject: [PATCH 385/526] Suppress false positive of clang-tidy on macOS --- plugins/experimental/remap_stats/remap_stats.c | 3 +++ plugins/lua/ts_lua_client_request.c | 1 + plugins/lua/ts_lua_client_response.c | 1 + plugins/lua/ts_lua_server_response.c | 1 + 4 files changed, 6 insertions(+) diff --git a/plugins/experimental/remap_stats/remap_stats.c b/plugins/experimental/remap_stats/remap_stats.c index 240b2cff4f3..b5a98928ec2 100644 --- a/plugins/experimental/remap_stats/remap_stats.c +++ b/plugins/experimental/remap_stats/remap_stats.c @@ -51,6 +51,7 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex c static __thread bool hash_init = false; if (unlikely(!hash_init)) { + // NOLINTNEXTLINE hcreate_r(TS_MAX_API_STATS << 1, &stat_cache); hash_init = true; TSDebug(DEBUG_TAG, "stat cache hash init"); @@ -58,6 +59,7 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex c search.key = name; search.data = 0; + // NOLINTNEXTLINE hsearch_r(search, FIND, &result, &stat_cache); if (unlikely(result == NULL)) { @@ -78,6 +80,7 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex c if (stat_id >= 0) { search.key = TSstrdup(name); search.data = (void *)((intptr_t)stat_id); + // NOLINTNEXTLINE hsearch_r(search, ENTER, &result, &stat_cache); TSDebug(DEBUG_TAG, "Cached stat_name: %s stat_id: %d", name, stat_id); } diff --git a/plugins/lua/ts_lua_client_request.c b/plugins/lua/ts_lua_client_request.c index aef49cac10e..ae4356d4853 100644 --- a/plugins/lua/ts_lua_client_request.c +++ b/plugins/lua/ts_lua_client_request.c @@ -483,6 +483,7 @@ ts_lua_client_request_set_url_port(lua_State *L) GET_HTTP_CONTEXT(http_ctx, L); + // NOLINTNEXTLINE port = luaL_checkint(L, 1); TSUrlPortSet(http_ctx->client_request_bufp, http_ctx->client_request_url, port); diff --git a/plugins/lua/ts_lua_client_response.c b/plugins/lua/ts_lua_client_response.c index d5c656c16f4..dfeebccb201 100644 --- a/plugins/lua/ts_lua_client_response.c +++ b/plugins/lua/ts_lua_client_response.c @@ -316,6 +316,7 @@ ts_lua_client_response_set_status(lua_State *L) TS_LUA_CHECK_CLIENT_RESPONSE_HDR(http_ctx); + // NOLINTNEXTLINE status = luaL_checkint(L, 1); reason = TSHttpHdrReasonLookup(status); diff --git a/plugins/lua/ts_lua_server_response.c b/plugins/lua/ts_lua_server_response.c index 107c9b4b616..3d991d2ed93 100644 --- a/plugins/lua/ts_lua_server_response.c +++ b/plugins/lua/ts_lua_server_response.c @@ -299,6 +299,7 @@ ts_lua_server_response_set_status(lua_State *L) TS_LUA_CHECK_SERVER_RESPONSE_HDR(http_ctx); + // NOLINTNEXTLINE status = luaL_checkint(L, 1); reason = TSHttpHdrReasonLookup(status); From 7651e269d638f1d83882558b86e3064282d21976 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 19 Mar 2019 15:44:19 +0900 Subject: [PATCH 386/526] Ran clang-tidy with modernize-use-default-member-init 1. Remove unmatched #endif and set `_location` nullptr. - iocore/eventsystem/I_IOBuffer.h - iocore/eventsystem/P_IOBuffer.h 2. Include "ts/apidefs.h" - include/tscpp/api/Stat.h 3. Rollback changes made with -fix-error option (clang-diagnostic-implicit-function-declaration) - plugins/lua/ts_lua_client_request.c - plugins/lua/ts_lua_client_response.c - plugins/lua/ts_lua_server_response.c - plugins/experimental/remap_stats/remap_stats.c --- .clang-tidy | 5 +- example/cppapi/websocket/WSBuffer.cc | 2 +- example/cppapi/websocket/WSBuffer.h | 2 +- example/intercept/intercept.cc | 12 +- example/passthru/passthru.cc | 8 +- include/tscore/Arena.h | 4 +- include/tscore/HashMD5.h | 4 +- include/tscore/IpMap.h | 4 +- include/tscore/List.h | 4 +- include/tscore/MT_hashtable.h | 4 +- include/tscore/PriorityQueue.h | 6 +- include/tscore/Ptr.h | 6 +- include/tscore/RbTree.h | 14 +- include/tscore/SimpleTokenizer.h | 13 +- include/tscore/TsBuffer.h | 12 +- include/tscore/ink_cap.h | 2 +- include/tscore/ink_inet.h | 4 +- include/tscore/ink_lockfile.h | 6 +- include/tscore/ink_string++.h | 10 +- include/tscore/ink_uuid.h | 4 +- include/tscpp/api/Continuation.h | 4 +- include/tscpp/api/Stat.h | 4 +- include/tscpp/util/MemSpan.h | 2 +- include/wccp/Wccp.h | 6 +- iocore/cache/Cache.cc | 2 +- iocore/cache/CacheHttp.cc | 2 +- iocore/cache/I_Cache.h | 9 +- iocore/cache/I_Store.h | 32 +- iocore/cache/P_CacheDir.h | 27 +- iocore/cache/P_CacheDisk.h | 6 +- iocore/cache/P_CacheHosting.h | 34 +- iocore/cache/P_CacheHttp.h | 4 +- iocore/cache/P_CacheInternal.h | 30 +- iocore/cache/P_CacheTest.h | 14 +- iocore/cache/P_CacheVol.h | 16 +- iocore/cache/RamCacheCLFUS.cc | 38 +- iocore/cache/Store.cc | 2 +- iocore/dns/DNS.cc | 2 +- iocore/dns/DNSConnection.cc | 3 +- iocore/dns/I_DNSProcessor.h | 12 +- iocore/dns/P_DNSConnection.h | 26 +- iocore/dns/P_DNSProcessor.h | 32 +- iocore/dns/P_SplitDNSProcessor.h | 28 +- iocore/dns/SplitDNS.cc | 6 +- iocore/eventsystem/I_EThread.h | 22 +- iocore/eventsystem/I_IOBuffer.h | 53 +-- iocore/eventsystem/I_ProxyAllocator.h | 6 +- iocore/eventsystem/I_VIO.h | 10 +- iocore/eventsystem/P_IOBuffer.h | 7 - iocore/eventsystem/P_VIO.h | 4 +- iocore/hostdb/HostDB.cc | 6 +- iocore/hostdb/I_HostDBProcessor.h | 22 +- iocore/hostdb/P_HostDBProcessor.h | 30 +- iocore/hostdb/P_RefCountCache.h | 6 +- iocore/hostdb/RefCountCache.cc | 3 +- iocore/net/Connection.cc | 2 +- iocore/net/I_NetVConnection.h | 31 +- iocore/net/I_Socks.h | 4 +- iocore/net/NetVCTest.cc | 34 +- iocore/net/P_Connection.h | 14 +- iocore/net/P_NetVCTest.h | 60 +-- iocore/net/P_SNIActionPerformer.h | 4 +- iocore/net/P_SSLCertLookup.h | 16 +- iocore/net/P_SSLNextProtocolSet.h | 4 +- iocore/net/P_SSLUtils.h | 7 +- iocore/net/P_Socks.h | 34 +- iocore/net/P_UDPConnection.h | 13 +- iocore/net/P_UDPIOEvent.h | 14 +- iocore/net/P_UDPNet.h | 6 +- iocore/net/P_UDPPacket.h | 16 +- iocore/net/P_UnixNetState.h | 8 +- iocore/net/P_UnixNetVConnection.h | 24 +- iocore/net/SSLCertLookup.cc | 6 +- iocore/net/SSLNextProtocolSet.cc | 2 +- iocore/net/SSLSessionCache.cc | 2 +- iocore/net/SSLSessionCache.h | 2 +- iocore/net/UnixNetVConnection.cc | 16 +- iocore/net/UnixUDPNet.cc | 9 +- iocore/utils/I_OneWayTunnel.h | 26 +- iocore/utils/OneWayTunnel.cc | 17 +- lib/records/I_RecHttp.h | 18 +- lib/records/RecHttp.cc | 12 +- lib/tsconfig/Errata.h | 4 +- lib/tsconfig/TsBuilder.h | 6 +- lib/tsconfig/TsValue.h | 12 +- plugins/authproxy/authproxy.cc | 40 +- plugins/cache_promote/cache_promote.cc | 16 +- plugins/cachekey/configs.h | 8 +- plugins/cachekey/pattern.cc | 2 +- plugins/cachekey/pattern.h | 10 +- plugins/conf_remap/conf_remap.cc | 4 +- plugins/escalate/escalate.cc | 4 +- plugins/esi/combo_handler.cc | 16 +- plugins/esi/fetcher/HttpDataFetcherImpl.h | 16 +- plugins/esi/serverIntercept.cc | 8 +- plugins/esi/test/TestHttpDataFetcher.h | 6 +- .../experimental/access_control/pattern.cc | 2 +- plugins/experimental/access_control/pattern.h | 10 +- plugins/experimental/balancer/roundrobin.cc | 4 +- .../experimental/cookie_remap/cookie_remap.cc | 26 +- plugins/experimental/geoip_acl/acl.h | 10 +- plugins/experimental/hipes/hipes.cc | 28 +- plugins/experimental/inliner/chunk-decoder.h | 6 +- plugins/experimental/inliner/fetcher.h | 4 +- plugins/experimental/inliner/html-parser.h | 10 +- plugins/experimental/inliner/ts.h | 10 +- plugins/experimental/magick/magick.cc | 8 +- plugins/experimental/memcache/tsmemcache.h | 6 +- plugins/experimental/mp4/mp4_common.h | 8 +- plugins/experimental/mp4/mp4_meta.h | 126 ++---- plugins/experimental/multiplexer/fetcher.h | 4 +- plugins/experimental/multiplexer/ts.h | 6 +- plugins/experimental/prefetch/configs.h | 15 +- plugins/experimental/prefetch/fetch.cc | 2 +- plugins/experimental/prefetch/fetch.h | 14 +- .../experimental/prefetch/fetch_policy_lru.h | 6 +- plugins/experimental/prefetch/pattern.cc | 2 +- plugins/experimental/prefetch/pattern.h | 6 +- plugins/experimental/prefetch/plugin.cc | 4 +- plugins/experimental/slice/ContentRange.h | 8 +- plugins/experimental/slice/Range.h | 6 +- .../ssl_session_reuse/src/publish.cc | 6 +- .../ssl_session_reuse/src/publisher.h | 4 +- .../ssl_session_reuse/src/ssl_init.cc | 2 +- .../ssl_session_reuse/src/ssl_utils.h | 2 +- plugins/experimental/sslheaders/sslheaders.cc | 3 +- plugins/experimental/sslheaders/sslheaders.h | 8 +- .../stream_editor/stream_editor.cc | 16 +- plugins/generator/generator.cc | 14 +- plugins/regex_remap/regex_remap.cc | 74 ++-- plugins/s3_auth/aws_auth_v4_wrap.h | 4 +- proxy/CacheControl.h | 48 +-- proxy/ControlBase.h | 4 +- proxy/ControlMatcher.h | 28 +- proxy/InkAPIInternal.h | 4 +- proxy/Plugin.cc | 5 +- proxy/Plugin.h | 12 +- proxy/PluginVC.cc | 6 +- proxy/PluginVC.h | 36 +- proxy/ProxyClientTransaction.cc | 10 +- proxy/ProxyClientTransaction.h | 10 +- proxy/StatPages.h | 12 +- proxy/Transform.cc | 3 +- proxy/Transform.h | 8 +- proxy/TransformInternal.h | 6 +- proxy/hdrs/HTTP.cc | 15 +- proxy/hdrs/HTTP.h | 24 +- proxy/hdrs/HdrHeap.h | 12 +- proxy/hdrs/HdrTest.h | 4 +- proxy/hdrs/MIME.h | 6 +- proxy/hdrs/URL.h | 4 +- proxy/http/Http1ClientSession.cc | 19 +- proxy/http/Http1ClientSession.h | 30 +- proxy/http/HttpCacheSM.cc | 24 +- proxy/http/HttpCacheSM.h | 38 +- proxy/http/HttpConfig.h | 379 ++++++------------ proxy/http/HttpProxyServerMain.cc | 4 +- proxy/http/HttpSM.h | 6 +- proxy/http/HttpSessionAccept.h | 10 +- proxy/http/HttpSessionManager.h | 4 +- proxy/http/HttpTunnel.cc | 66 +-- proxy/http/HttpTunnel.h | 110 ++--- proxy/http/HttpUpdateSM.cc | 2 +- proxy/http/HttpUpdateSM.h | 4 +- proxy/http/remap/AclFiltering.cc | 3 +- proxy/http/remap/AclFiltering.h | 10 +- proxy/http/remap/RemapConfig.cc | 2 +- proxy/http/remap/RemapConfig.h | 14 +- proxy/http/remap/RemapProcessor.h | 6 +- proxy/http/remap/UrlMapping.h | 18 +- proxy/http2/HPACK.h | 8 +- proxy/http2/HTTP2.h | 23 +- proxy/http2/Http2ClientSession.h | 4 +- proxy/logging/LogConfig.cc | 14 +- proxy/logging/LogConfig.h | 20 +- proxy/logging/LogField.cc | 2 +- proxy/logging/LogField.h | 2 +- proxy/logging/LogFieldAliasMap.h | 18 +- proxy/logging/LogFilter.cc | 2 +- proxy/logging/LogFilter.h | 2 +- proxy/logging/LogObject.h | 4 +- proxy/logging/LogSock.cc | 2 +- proxy/logging/LogSock.h | 4 +- src/traffic_logstats/logstats.cc | 52 +-- src/traffic_server/traffic_server.cc | 55 +-- src/tscore/BaseLogFile.cc | 10 +- src/tscore/HashMD5.cc | 2 +- src/tscore/ink_cap.cc | 3 +- src/tscore/ink_resource.cc | 10 +- src/tscpp/api/Headers.cc | 10 +- src/tscpp/api/InterceptPlugin.cc | 8 +- src/tscpp/api/Logger.cc | 23 +- src/tscpp/api/Request.cc | 22 +- src/tscpp/api/Response.cc | 6 +- src/tscpp/api/Stat.cc | 2 +- 195 files changed, 1180 insertions(+), 1787 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index a1ea4a36e28..996a4881cad 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,2 +1,5 @@ -Checks: -*,modernize-loop-convert,modernize-use-bool-literals,modernize-deprecated-headers,performance-unnecessary-value-param,performance-faster-string-find,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-use-nullptr,modernize-use-override +Checks: -*,modernize-use-default-member-init,modernize-loop-convert,modernize-use-bool-literals,modernize-deprecated-headers,performance-unnecessary-value-param,performance-faster-string-find,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-use-nullptr,modernize-use-override +CheckOptions: + - key: modernize-use-default-member-init.UseAssignment + value: '1' HeaderFilterRegex: (?!(\/openssl\/|\/pcre\/|\/lib\/yaml\/)).* diff --git a/example/cppapi/websocket/WSBuffer.cc b/example/cppapi/websocket/WSBuffer.cc index 66e4f8079b0..f6e68999d80 100644 --- a/example/cppapi/websocket/WSBuffer.cc +++ b/example/cppapi/websocket/WSBuffer.cc @@ -56,7 +56,7 @@ static const std::string magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; -WSBuffer::WSBuffer() : frame_(0) {} +WSBuffer::WSBuffer() {} void WSBuffer::buffer(std::string const &data) diff --git a/example/cppapi/websocket/WSBuffer.h b/example/cppapi/websocket/WSBuffer.h index 4b12f168cde..cf7bdc54b0a 100644 --- a/example/cppapi/websocket/WSBuffer.h +++ b/example/cppapi/websocket/WSBuffer.h @@ -84,6 +84,6 @@ class WSBuffer private: std::string ws_buf_; // incoming data. - int frame_; // frame type of current message + int frame_ = 0; // frame type of current message std::string msg_buf_; // decoded message data }; diff --git a/example/intercept/intercept.cc b/example/intercept/intercept.cc index 8369667c540..c793e19b96f 100644 --- a/example/intercept/intercept.cc +++ b/example/intercept/intercept.cc @@ -70,11 +70,11 @@ static int InterceptTxnHook(TSCont contp, TSEvent event, void *edata); // a write). We need two of these for each TSVConn; one to push // data into the TSVConn and one to pull data out. struct InterceptIOChannel { - TSVIO vio; - TSIOBuffer iobuf; - TSIOBufferReader reader; + TSVIO vio = nullptr; + TSIOBuffer iobuf = nullptr; + TSIOBufferReader reader = nullptr; - InterceptIOChannel() : vio(nullptr), iobuf(nullptr), reader(nullptr) {} + InterceptIOChannel() {} ~InterceptIOChannel() { if (this->reader) { @@ -130,12 +130,12 @@ struct InterceptIO { // are intercepting is the server. Hence the "client" and // "server" nomenclature here. struct InterceptState { - TSHttpTxn txn; // The transaction on whose behalf we are intercepting. + TSHttpTxn txn = nullptr; // The transaction on whose behalf we are intercepting. InterceptIO client; // Server intercept VC state. InterceptIO server; // Intercept origin VC state. - InterceptState() : txn(nullptr) {} + InterceptState() {} ~InterceptState() {} }; diff --git a/example/passthru/passthru.cc b/example/passthru/passthru.cc index 09f386f0b8a..587f35a0c6c 100644 --- a/example/passthru/passthru.cc +++ b/example/passthru/passthru.cc @@ -53,11 +53,11 @@ union EventArgument { }; struct PassthruIO { - TSVIO vio; - TSIOBuffer iobuf; - TSIOBufferReader reader; + TSVIO vio = nullptr; + TSIOBuffer iobuf = nullptr; + TSIOBufferReader reader = nullptr; - PassthruIO() : vio(nullptr), iobuf(nullptr), reader(nullptr) {} + PassthruIO() {} ~PassthruIO() { clear(); } void clear() diff --git a/include/tscore/Arena.h b/include/tscore/Arena.h index c885c980212..6d359bd7af8 100644 --- a/include/tscore/Arena.h +++ b/include/tscore/Arena.h @@ -37,7 +37,7 @@ struct ArenaBlock { class Arena { public: - Arena() : m_blocks(nullptr) {} + Arena() {} ~Arena() { reset(); } inkcoreapi void *alloc(size_t size, size_t alignment = sizeof(double)); void free(void *mem, size_t size); @@ -49,7 +49,7 @@ class Arena inkcoreapi void reset(); private: - ArenaBlock *m_blocks; + ArenaBlock *m_blocks = nullptr; }; /*------------------------------------------------------------------------- diff --git a/include/tscore/HashMD5.h b/include/tscore/HashMD5.h index 1447786dc47..09c7bd67c63 100644 --- a/include/tscore/HashMD5.h +++ b/include/tscore/HashMD5.h @@ -36,6 +36,6 @@ struct ATSHashMD5 : ATSHash { private: EVP_MD_CTX *ctx; unsigned char md_value[EVP_MAX_MD_SIZE]; - unsigned int md_len; - bool finalized; + unsigned int md_len = 0; + bool finalized = false; }; diff --git a/include/tscore/IpMap.h b/include/tscore/IpMap.h index 754f50630b8..568374ebfc1 100644 --- a/include/tscore/IpMap.h +++ b/include/tscore/IpMap.h @@ -123,7 +123,7 @@ class IpMap public: using self_type = Node; ///< Self reference type. /// Default constructor. - Node() : _data(nullptr) {} + Node() {} /// Construct with @a data. Node(void *data) : _data(data) {} /// @return Client data for the node. @@ -146,7 +146,7 @@ class IpMap virtual sockaddr const *max() const = 0; protected: - void *_data; ///< Client data. + void *_data = nullptr; ///< Client data. }; /** Iterator over nodes / intervals. diff --git a/include/tscore/List.h b/include/tscore/List.h index aeea32b295a..25bfac5a498 100644 --- a/include/tscore/List.h +++ b/include/tscore/List.h @@ -506,8 +506,8 @@ template struct SortableQueue : publi // template struct CountQueue : public Queue { - int size; - inline CountQueue() : size(0) {} + int size = 0; + inline CountQueue() {} inline void push(C *e); inline C *pop(); inline void enqueue(C *e); diff --git a/include/tscore/MT_hashtable.h b/include/tscore/MT_hashtable.h index f3b7c916a94..cb6a80d34ee 100644 --- a/include/tscore/MT_hashtable.h +++ b/include/tscore/MT_hashtable.h @@ -69,8 +69,8 @@ struct MT_ListEntry{ template class HashTableIteratorState { public: - HashTableIteratorState() : cur_buck(-1), ppcur(NULL) {} - int cur_buck; + HashTableIteratorState() : ppcur(NULL) {} + int cur_buck = -1; HashTableEntry **ppcur; }; diff --git a/include/tscore/PriorityQueue.h b/include/tscore/PriorityQueue.h index 3d2c0d3dc82..e1111da0f03 100644 --- a/include/tscore/PriorityQueue.h +++ b/include/tscore/PriorityQueue.h @@ -28,9 +28,9 @@ #include template struct PriorityQueueEntry { - PriorityQueueEntry(T n) : index(0), node(n){}; - PriorityQueueEntry() : index(0), node(NULL){}; - uint32_t index; + PriorityQueueEntry(T n) : node(n){}; + PriorityQueueEntry() : node(NULL){}; + uint32_t index = 0; T node; }; diff --git a/include/tscore/Ptr.h b/include/tscore/Ptr.h index d067604a9ef..559f346feb3 100644 --- a/include/tscore/Ptr.h +++ b/include/tscore/Ptr.h @@ -44,8 +44,8 @@ struct ForceVFPTToTop { class RefCountObj : public ForceVFPTToTop { public: - RefCountObj() : m_refcount(0) {} - RefCountObj(const RefCountObj &s) : m_refcount(0) + RefCountObj() {} + RefCountObj(const RefCountObj &s) { (void)s; return; @@ -86,7 +86,7 @@ class RefCountObj : public ForceVFPTToTop } private: - int m_refcount; + int m_refcount = 0; }; //////////////////////////////////////////////////////////////////////// diff --git a/include/tscore/RbTree.h b/include/tscore/RbTree.h index 4d936d9dcfd..dcb52f58ec6 100644 --- a/include/tscore/RbTree.h +++ b/include/tscore/RbTree.h @@ -108,7 +108,7 @@ namespace detail int validate(); /// Default constructor. - RBNode() : _color(RED), _parent(nullptr), _left(nullptr), _right(nullptr), _next(nullptr), _prev(nullptr) {} + RBNode() {} /// Destructor (force virtual). virtual ~RBNode() {} /** Rotate the subtree rooted at this node. @@ -206,12 +206,12 @@ namespace detail //! Invoke @c structure_fixup() on this node and all of its ancestors. self *rippleStructureFixup(); - Color _color; ///< node color - self *_parent; ///< parent node (needed for rotations) - self *_left; ///< left child - self *_right; ///< right child - self *_next; ///< Next node. - self *_prev; ///< Previous node. + Color _color = RED; ///< node color + self *_parent = nullptr; ///< parent node (needed for rotations) + self *_left = nullptr; ///< left child + self *_right = nullptr; ///< right child + self *_next = nullptr; ///< Next node. + self *_prev = nullptr; ///< Previous node. }; } /* namespace detail */ diff --git a/include/tscore/SimpleTokenizer.h b/include/tscore/SimpleTokenizer.h index 3f0e7fad3d0..15471ef66d9 100644 --- a/include/tscore/SimpleTokenizer.h +++ b/include/tscore/SimpleTokenizer.h @@ -123,14 +123,13 @@ class SimpleTokenizer OVERWRITE_INPUT_STRING = 8 }; - SimpleTokenizer(char delimiter = ' ', unsigned mode = 0, char escape = '\\') - : _data(nullptr), _delimiter(delimiter), _mode(mode), _escape(escape), _start(0), _length(0) + SimpleTokenizer(char delimiter = ' ', unsigned mode = 0, char escape = '\\') : _delimiter(delimiter), _mode(mode), _escape(escape) { } // NOTE: The input strring 's' is overwritten for mode OVERWRITE_INPUT_STRING. SimpleTokenizer(const char *s, char delimiter = ' ', unsigned mode = 0, char escape = '\\') - : _data(nullptr), _delimiter(delimiter), _mode(mode), _escape(escape) + : _delimiter(delimiter), _mode(mode), _escape(escape) { setString(s); } @@ -188,15 +187,15 @@ class SimpleTokenizer } private: - char *_data; // a pointer to the input data itself, + char *_data = nullptr; // a pointer to the input data itself, // or to a copy of it char _delimiter; // the token delimiter unsigned _mode; // flags that determine the // mode of operation - char _escape; // the escape character - size_t _start; // pointer to the start of the next + char _escape; // the escape character + size_t _start = 0; // pointer to the start of the next // token - size_t _length; // the length of _data + size_t _length = 0; // the length of _data void _clearData() diff --git a/include/tscore/TsBuffer.h b/include/tscore/TsBuffer.h index 2ccb9a498b4..39e55b8a98b 100644 --- a/include/tscore/TsBuffer.h +++ b/include/tscore/TsBuffer.h @@ -49,8 +49,8 @@ struct Buffer { typedef Buffer self; ///< Self reference type. typedef bool (self::*pseudo_bool)() const; - char *_ptr; ///< Pointer to base of memory chunk. - size_t _size; ///< Size of memory chunk. + char *_ptr = nullptr; ///< Pointer to base of memory chunk. + size_t _size = 0; ///< Size of memory chunk. /// Default constructor (empty buffer). Buffer(); @@ -129,8 +129,8 @@ struct ConstBuffer { typedef ConstBuffer self; ///< Self reference type. typedef bool (self::*pseudo_bool)() const; - char const *_ptr; ///< Pointer to base of memory chunk. - size_t _size; ///< Size of memory chunk. + char const *_ptr = nullptr; ///< Pointer to base of memory chunk. + size_t _size = 0; ///< Size of memory chunk. /// Default constructor (empty buffer). ConstBuffer(); @@ -286,7 +286,7 @@ struct ConstBuffer { // ---------------------------------------------------------- // Inline implementations. -inline Buffer::Buffer() : _ptr(nullptr), _size(0) {} +inline Buffer::Buffer() {} inline Buffer::Buffer(char *ptr, size_t n) : _ptr(ptr), _size(n) {} inline Buffer & Buffer::set(char *ptr, size_t n) @@ -353,7 +353,7 @@ Buffer::size() const return _size; } -inline ConstBuffer::ConstBuffer() : _ptr(nullptr), _size(0) {} +inline ConstBuffer::ConstBuffer() {} inline ConstBuffer::ConstBuffer(char const *ptr, size_t n) : _ptr(ptr), _size(n) {} inline ConstBuffer::ConstBuffer(char const *start, char const *end) : _ptr(start), _size(end - start) {} inline ConstBuffer::ConstBuffer(Buffer const &that) : _ptr(that._ptr), _size(that._size) {} diff --git a/include/tscore/ink_cap.h b/include/tscore/ink_cap.h index c5ed681ef57..4a2c7d57ca1 100644 --- a/include/tscore/ink_cap.h +++ b/include/tscore/ink_cap.h @@ -90,7 +90,7 @@ class ElevateAccess void demote(); private: - bool elevated; + bool elevated = false; uid_t saved_uid; unsigned level; diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index 3de20baff58..39403374375 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -1149,7 +1149,7 @@ struct IpAddr { typedef IpAddr self; ///< Self reference type. /// Default construct (invalid address). - IpAddr() : _family(AF_UNSPEC) {} + IpAddr() {} /** Construct from IPv4 address. * @@ -1284,7 +1284,7 @@ struct IpAddr { /// Test for any addr bool isAnyAddr() const; - uint16_t _family; ///< Protocol family. + uint16_t _family = AF_UNSPEC; ///< Protocol family. /// Address data. union Addr { in_addr_t _ip4; ///< IPv4 address storage. diff --git a/include/tscore/ink_lockfile.h b/include/tscore/ink_lockfile.h index d3d6878f8e1..8f5aa2ef6f7 100644 --- a/include/tscore/ink_lockfile.h +++ b/include/tscore/ink_lockfile.h @@ -33,9 +33,9 @@ class Lockfile { public: - Lockfile() : fd(0) { fname[0] = '\0'; } + Lockfile() { fname[0] = '\0'; } // coverity[uninit_member] - Lockfile(const char *filename) : fd(0) { ink_strlcpy(fname, filename, sizeof(fname)); } + Lockfile(const char *filename) { ink_strlcpy(fname, filename, sizeof(fname)); } ~Lockfile() {} void SetLockfileName(const char *filename) @@ -86,5 +86,5 @@ class Lockfile private: char fname[PATH_NAME_MAX]; - int fd; + int fd = 0; }; diff --git a/include/tscore/ink_string++.h b/include/tscore/ink_string++.h index 126ca25fddb..fc9d86c343e 100644 --- a/include/tscore/ink_string++.h +++ b/include/tscore/ink_string++.h @@ -41,12 +41,12 @@ ***********************************************************************/ struct Str { - const char *str; // string pointer - size_t len; // length of string (not counting NUL) - struct Str *next; // next in list - struct Str *prev; // prev in list + const char *str = nullptr; // string pointer + size_t len = 0; // length of string (not counting NUL) + struct Str *next = nullptr; // next in list + struct Str *prev = nullptr; // prev in list - Str() : str(nullptr), len(0), next(nullptr), prev(nullptr) {} + Str() {} Str(char *s) { str = s; diff --git a/include/tscore/ink_uuid.h b/include/tscore/ink_uuid.h index fe6f3a9e8eb..32127dcfc3a 100644 --- a/include/tscore/ink_uuid.h +++ b/include/tscore/ink_uuid.h @@ -33,7 +33,7 @@ class ATSUuid { public: // Constructors - ATSUuid() : _version(TS_UUID_UNDEFINED) {} + ATSUuid() {} ATSUuid &operator=(const ATSUuid other); // Initialize the UUID from a string @@ -122,7 +122,7 @@ class ATSUuid } _uuid; // This is the typically used visible portion of the UUID - TSUuidVersion _version; + TSUuidVersion _version = TS_UUID_UNDEFINED; char _string[TS_UUID_STRING_LEN + 1]; bool diff --git a/include/tscpp/api/Continuation.h b/include/tscpp/api/Continuation.h index 62e40a13af4..ef25dabc9bb 100644 --- a/include/tscpp/api/Continuation.h +++ b/include/tscpp/api/Continuation.h @@ -43,7 +43,7 @@ class Continuation // Create "empty" continuation, can only be populated by move assignement. // - Continuation() : _cont(nullptr) {} + Continuation() {} TSCont asTSCont() const @@ -139,7 +139,7 @@ class Continuation // static int _generalEventFunc(TSCont cont, TSEvent event, void *edata); - TSCont _cont; + TSCont _cont = nullptr; }; } // end namespace atscppapi diff --git a/include/tscpp/api/Stat.h b/include/tscpp/api/Stat.h index 25d633ddbdd..a5a844097d3 100644 --- a/include/tscpp/api/Stat.h +++ b/include/tscpp/api/Stat.h @@ -21,7 +21,9 @@ #pragma once +#include "ts/apidefs.h" #include "tscpp/api/noncopyable.h" + #include #include @@ -97,7 +99,7 @@ class Stat : noncopyable void set(int64_t value); private: - int stat_id_; /**< The internal stat ID */ + int stat_id_ = TS_ERROR; /**< The internal stat ID */ }; } // namespace atscppapi diff --git a/include/tscpp/util/MemSpan.h b/include/tscpp/util/MemSpan.h index 5a4d63141b7..f0a2890b3d0 100644 --- a/include/tscpp/util/MemSpan.h +++ b/include/tscpp/util/MemSpan.h @@ -317,7 +317,7 @@ inline MemSpan::MemSpan(void *start, void *end) : _data(start), _size(static_cas template MemSpan::MemSpan(T (&a)[N]) : _data(a), _size(N * sizeof(T)) {} -inline constexpr MemSpan::MemSpan(std::nullptr_t) : _data(nullptr), _size(0) {} +inline constexpr MemSpan::MemSpan(std::nullptr_t) {} inline ptrdiff_t MemSpan::distance(void const *lhs, void const *rhs) diff --git a/include/wccp/Wccp.h b/include/wccp/Wccp.h index 7f58e922b83..40dff8fd4b6 100644 --- a/include/wccp/Wccp.h +++ b/include/wccp/Wccp.h @@ -372,8 +372,8 @@ class Cache::Service : public ServiceConstants private: Service(Cache const &cache, detail::cache::GroupData &group); - Cache m_cache; ///< Parent cache. - detail::cache::GroupData *m_group; ///< Service Group data. + Cache m_cache; ///< Parent cache. + detail::cache::GroupData *m_group = nullptr; ///< Service Group data. friend class Cache; }; @@ -499,7 +499,7 @@ ServiceGroup::clearPorts() return *this; } -inline Cache::Service::Service() : m_group(nullptr) {} +inline Cache::Service::Service() {} inline Cache::Service::Service(Cache const &cache, detail::cache::GroupData &group) : m_cache(cache), m_group(&group) {} diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index dd408c9e9e5..9d986076c83 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -285,7 +285,7 @@ update_cache_config(const char * /* name ATS_UNUSED */, RecDataT /* data_type AT return 0; } -CacheVC::CacheVC() : alternate_index(CACHE_ALT_INDEX_DEFAULT) +CacheVC::CacheVC() { size_to_init = sizeof(CacheVC) - (size_t) & ((CacheVC *)nullptr)->vio; memset((void *)&vio, 0, size_to_init); diff --git a/iocore/cache/CacheHttp.cc b/iocore/cache/CacheHttp.cc index e78845f71fb..b4ff5d0919d 100644 --- a/iocore/cache/CacheHttp.cc +++ b/iocore/cache/CacheHttp.cc @@ -32,7 +32,7 @@ static vec_info default_vec_info; static CacheHTTPInfo default_http_info; -CacheHTTPInfoVector::CacheHTTPInfoVector() : magic(nullptr), data(&default_vec_info, 4), xcount(0) {} +CacheHTTPInfoVector::CacheHTTPInfoVector() : data(&default_vec_info, 4) {} /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/iocore/cache/I_Cache.h b/iocore/cache/I_Cache.h index 57f574a84dd..8d4447b6f6e 100644 --- a/iocore/cache/I_Cache.h +++ b/iocore/cache/I_Cache.h @@ -62,9 +62,8 @@ typedef HTTPInfo CacheHTTPInfo; struct CacheProcessor : public Processor { CacheProcessor() : min_stripe_version(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION), - max_stripe_version(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION), - cb_after_init(nullptr), - wait_for_cache(0) + max_stripe_version(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION) + { } @@ -159,8 +158,8 @@ struct CacheProcessor : public Processor { ts::VersionNumber min_stripe_version; ts::VersionNumber max_stripe_version; - CALLBACK_FUNC cb_after_init; - int wait_for_cache; + CALLBACK_FUNC cb_after_init = nullptr; + int wait_for_cache = 0; }; inline void diff --git a/iocore/cache/I_Store.h b/iocore/cache/I_Store.h index fbb48b87bfb..901ae9fd75a 100644 --- a/iocore/cache/I_Store.h +++ b/iocore/cache/I_Store.h @@ -70,17 +70,17 @@ struct span_diskid_t { // Those on the same disk should be in a linked list. // struct Span { - int64_t blocks; // in STORE_BLOCK_SIZE blocks - int64_t offset; // used only if (file == true); in bytes - unsigned hw_sector_size; - unsigned alignment; + int64_t blocks = 0; // in STORE_BLOCK_SIZE blocks + int64_t offset = 0; // used only if (file == true); in bytes + unsigned hw_sector_size = DEFAULT_HW_SECTOR_SIZE; + unsigned alignment = 0; span_diskid_t disk_id; - int forced_volume_num; ///< Force span in to specific volume. + int forced_volume_num = -1; ///< Force span in to specific volume. private: - bool is_mmapable_internal; + bool is_mmapable_internal = false; public: - bool file_pathname; // the pathname is a file + bool file_pathname = false; // the pathname is a file // v- used as a magic location for copy constructor. // we memcpy everything before this member and do explicit assignment for the rest. ats_scoped_str pathname; @@ -157,17 +157,7 @@ struct Span { /// Set the volume number. void volume_number_set(int n); - Span() - : blocks(0), - offset(0), - hw_sector_size(DEFAULT_HW_SECTOR_SIZE), - alignment(0), - forced_volume_num(-1), - is_mmapable_internal(false), - file_pathname(false) - { - disk_id[0] = disk_id[1] = 0; - } + Span() { disk_id[0] = disk_id[1] = 0; } /// Copy constructor. /// @internal Prior to this implementation handling the char* pointers was done manual @@ -259,10 +249,10 @@ struct Store { ~Store(); // The number of disks/paths defined in storage.config - unsigned n_disks_in_config; + unsigned n_disks_in_config = 0; // The number of disks/paths we could actually read and parse. - unsigned n_disks; - Span **disk; + unsigned n_disks = 0; + Span **disk = nullptr; Result read_config(); diff --git a/iocore/cache/P_CacheDir.h b/iocore/cache/P_CacheDir.h index 27a31b4acaf..576224f167f 100644 --- a/iocore/cache/P_CacheDir.h +++ b/iocore/cache/P_CacheDir.h @@ -258,29 +258,18 @@ struct OpenDir : public Continuation { }; struct CacheSync : public Continuation { - int vol_idx; - char *buf; - size_t buflen; - bool buf_huge; - off_t writepos; + int vol_idx = 0; + char *buf = nullptr; + size_t buflen = 0; + bool buf_huge = false; + off_t writepos = 0; AIOCallbackInternal io; - Event *trigger; - ink_hrtime start_time; + Event *trigger = nullptr; + ink_hrtime start_time = 0; int mainEvent(int event, Event *e); void aio_write(int fd, char *b, int n, off_t o); - CacheSync() - : Continuation(new_ProxyMutex()), - vol_idx(0), - buf(nullptr), - buflen(0), - buf_huge(false), - writepos(0), - trigger(nullptr), - start_time(0) - { - SET_HANDLER(&CacheSync::mainEvent); - } + CacheSync() : Continuation(new_ProxyMutex()) { SET_HANDLER(&CacheSync::mainEvent); } }; // Global Functions diff --git a/iocore/cache/P_CacheDisk.h b/iocore/cache/P_CacheDisk.h index 51f2462bf40..18379de9d6e 100644 --- a/iocore/cache/P_CacheDisk.h +++ b/iocore/cache/P_CacheDisk.h @@ -53,11 +53,11 @@ struct DiskVolBlock { }; struct DiskVolBlockQueue { - DiskVolBlock *b; - int new_block; /* whether an existing vol or a new one */ + DiskVolBlock *b = nullptr; + int new_block = 0; /* whether an existing vol or a new one */ LINK(DiskVolBlockQueue, link); - DiskVolBlockQueue() : b(nullptr), new_block(0) {} + DiskVolBlockQueue() {} }; struct DiskVol { diff --git a/iocore/cache/P_CacheHosting.h b/iocore/cache/P_CacheHosting.h index e78b2a1149a..765c695ec3f 100644 --- a/iocore/cache/P_CacheHosting.h +++ b/iocore/cache/P_CacheHosting.h @@ -46,34 +46,24 @@ struct CacheHostRecord { ats_free(cp); } - CacheType type; - Vol **vols; - int good_num_vols; - int num_vols; - int num_initialized; - unsigned short *vol_hash_table; - CacheVol **cp; - int num_cachevols; - - CacheHostRecord() - : type(CACHE_NONE_TYPE), - vols(nullptr), - good_num_vols(0), - num_vols(0), - num_initialized(0), - vol_hash_table(nullptr), - cp(nullptr), - num_cachevols(0) - { - } + CacheType type = CACHE_NONE_TYPE; + Vol **vols = nullptr; + int good_num_vols = 0; + int num_vols = 0; + int num_initialized = 0; + unsigned short *vol_hash_table = nullptr; + CacheVol **cp = nullptr; + int num_cachevols = 0; + + CacheHostRecord() {} }; void build_vol_hash_table(CacheHostRecord *cp); struct CacheHostResult { - CacheHostRecord *record; + CacheHostRecord *record = nullptr; - CacheHostResult() : record(nullptr) {} + CacheHostResult() {} }; class CacheHostMatcher diff --git a/iocore/cache/P_CacheHttp.h b/iocore/cache/P_CacheHttp.h index 24ddabe7acb..c9932a8124a 100644 --- a/iocore/cache/P_CacheHttp.h +++ b/iocore/cache/P_CacheHttp.h @@ -43,7 +43,7 @@ struct vec_info { }; struct CacheHTTPInfoVector { - void *magic; + void *magic = nullptr; CacheHTTPInfoVector(); ~CacheHTTPInfoVector(); @@ -72,7 +72,7 @@ struct CacheHTTPInfoVector { int unmarshal(const char *buf, int length, RefCountObj *block_ptr); CacheArray data; - int xcount; + int xcount = 0; Ptr vector_buf; }; diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h index fb2fb71b974..54d531485b8 100644 --- a/iocore/cache/P_CacheInternal.h +++ b/iocore/cache/P_CacheInternal.h @@ -425,7 +425,7 @@ struct CacheVC : public CacheVConnection { OpenDirEntry *od; AIOCallbackInternal io; - int alternate_index; // preferred position in vector + int alternate_index = CACHE_ALT_INDEX_DEFAULT; // preferred position in vector LINK(CacheVC, opendir_link); #ifdef CACHE_STAT_PAGES LINK(CacheVC, stat_link); @@ -971,14 +971,14 @@ struct Vol; class CacheHostTable; struct Cache { - int cache_read_done; - int total_good_nvol; - int total_nvol; - int ready; - int64_t cache_size; // in store block size - CacheHostTable *hosttable; - int total_initialized_vol; - CacheType scheme; + int cache_read_done = 0; + int total_good_nvol = 0; + int total_nvol = 0; + int ready = CACHE_INITIALIZING; + int64_t cache_size = 0; // in store block size + CacheHostTable *hosttable = nullptr; + int total_initialized_vol = 0; + CacheType scheme = CACHE_NONE_TYPE; int open(bool reconfigure, bool fix); int close(); @@ -1009,17 +1009,7 @@ struct Cache { Vol *key_to_vol(const CacheKey *key, const char *hostname, int host_len); - Cache() - : cache_read_done(0), - total_good_nvol(0), - total_nvol(0), - ready(CACHE_INITIALIZING), - cache_size(0), // in store block size - hosttable(nullptr), - total_initialized_vol(0), - scheme(CACHE_NONE_TYPE) - { - } + Cache() {} }; extern Cache *theCache; diff --git a/iocore/cache/P_CacheTest.h b/iocore/cache/P_CacheTest.h index 3b369fd3c6e..ba9814e72ae 100644 --- a/iocore/cache/P_CacheTest.h +++ b/iocore/cache/P_CacheTest.h @@ -48,17 +48,17 @@ struct PinnedDocTable : public Continuation { }; struct CacheTestHost { - char *name; - unsigned int xlast_cachable_id; - double xprev_host_prob; - double xnext_host_prob; + char *name = nullptr; + unsigned int xlast_cachable_id = 0; + double xprev_host_prob = 0; + double xnext_host_prob = 0; - CacheTestHost() : name(nullptr), xlast_cachable_id(0), xprev_host_prob(0), xnext_host_prob(0) {} + CacheTestHost() {} }; struct CacheTestHeader { - CacheTestHeader() : serial(0) {} - uint64_t serial; + CacheTestHeader() {} + uint64_t serial = 0; }; struct CacheTestSM : public RegressionSM { diff --git a/iocore/cache/P_CacheVol.h b/iocore/cache/P_CacheVol.h index a927eb46113..5b3254ea7bf 100644 --- a/iocore/cache/P_CacheVol.h +++ b/iocore/cache/P_CacheVol.h @@ -279,17 +279,17 @@ struct AIO_Callback_handler : public Continuation { }; struct CacheVol { - int vol_number; - int scheme; - off_t size; - int num_vols; - Vol **vols; - DiskVol **disk_vols; + int vol_number = -1; + int scheme = 0; + off_t size = 0; + int num_vols = 0; + Vol **vols = nullptr; + DiskVol **disk_vols = nullptr; LINK(CacheVol, link); // per volume stats - RecRawStatBlock *vol_rsb; + RecRawStatBlock *vol_rsb = nullptr; - CacheVol() : vol_number(-1), scheme(0), size(0), num_vols(0), vols(nullptr), disk_vols(nullptr), vol_rsb(nullptr) {} + CacheVol() {} }; // Note : hdr() needs to be 8 byte aligned. diff --git a/iocore/cache/RamCacheCLFUS.cc b/iocore/cache/RamCacheCLFUS.cc index 9cb4c44d9b1..3fe4af0a416 100644 --- a/iocore/cache/RamCacheCLFUS.cc +++ b/iocore/cache/RamCacheCLFUS.cc @@ -71,9 +71,9 @@ struct RamCacheCLFUSEntry { }; struct RamCacheCLFUS : public RamCache { - int64_t max_bytes; - int64_t bytes; - int64_t objects; + int64_t max_bytes = 0; + int64_t bytes = 0; + int64_t objects = 0; // returns 1 on found/stored, 0 on not found/stored, if provided auxkey1 and auxkey2 must match int get(CryptoHash *key, Ptr *ret_data, uint32_t auxkey1 = 0, uint32_t auxkey2 = 0) override; @@ -85,16 +85,16 @@ struct RamCacheCLFUS : public RamCache { void init(int64_t max_bytes, Vol *vol) override; // private - Vol *vol; // for stats - double average_value; - int64_t history; - int ibuckets; - int nbuckets; + Vol *vol = nullptr; // for stats + double average_value = 0; + int64_t history = 0; + int ibuckets = 0; + int nbuckets = 0; DList(RamCacheCLFUSEntry, hash_link) * bucket; Que(RamCacheCLFUSEntry, lru_link) lru[2]; - uint16_t *seen; - int ncompressed; - RamCacheCLFUSEntry *compressed; // first uncompressed lru[0] entry + uint16_t *seen = nullptr; + int ncompressed = 0; + RamCacheCLFUSEntry *compressed = nullptr; // first uncompressed lru[0] entry void compress_entries(EThread *thread, int do_at_most = INT_MAX); void resize_hashtable(); void victimize(RamCacheCLFUSEntry *e); @@ -102,21 +102,7 @@ struct RamCacheCLFUS : public RamCache { RamCacheCLFUSEntry *destroy(RamCacheCLFUSEntry *e); void requeue_victims(Que(RamCacheCLFUSEntry, lru_link) & victims); void tick(); // move CLOCK on history - RamCacheCLFUS() - : max_bytes(0), - bytes(0), - objects(0), - vol(nullptr), - average_value(0), - history(0), - ibuckets(0), - nbuckets(0), - bucket(nullptr), - seen(nullptr), - ncompressed(0), - compressed(nullptr) - { - } + RamCacheCLFUS() : bucket(nullptr) {} }; int64_t diff --git a/iocore/cache/Store.cc b/iocore/cache/Store.cc index bcafbd4b270..38deecd286d 100644 --- a/iocore/cache/Store.cc +++ b/iocore/cache/Store.cc @@ -72,7 +72,7 @@ span_file_typename(mode_t st_mode) } Ptr tmp_p; -Store::Store() : n_disks_in_config(0), n_disks(0), disk(nullptr) {} +Store::Store() {} void Store::add(Span *ds) diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index aae4c6bf100..38e1e2efedc 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -386,7 +386,7 @@ ink_dn_expand(const u_char *msg, const u_char *eom, const u_char *comp_dn, u_cha return ::dn_expand((unsigned char *)msg, (unsigned char *)eom, (unsigned char *)comp_dn, (char *)exp_dn, length); } -DNSProcessor::DNSProcessor() : thread(nullptr), handler(nullptr) +DNSProcessor::DNSProcessor() { ink_zero(l_res); ink_zero(local_ipv6); diff --git a/iocore/dns/DNSConnection.cc b/iocore/dns/DNSConnection.cc index 311ff9369f1..fc4cb32ff75 100644 --- a/iocore/dns/DNSConnection.cc +++ b/iocore/dns/DNSConnection.cc @@ -48,8 +48,7 @@ DNSConnection::Options const DNSConnection::DEFAULT_OPTIONS; // Functions // -DNSConnection::DNSConnection() - : fd(NO_FD), num(0), generator((uint32_t)((uintptr_t)time(nullptr) ^ (uintptr_t)this)), handler(nullptr) +DNSConnection::DNSConnection() : fd(NO_FD), generator((uint32_t)((uintptr_t)time(nullptr) ^ (uintptr_t)this)) { memset(&ip, 0, sizeof(ip)); } diff --git a/iocore/dns/I_DNSProcessor.h b/iocore/dns/I_DNSProcessor.h index 15af75049ce..a0ec65115eb 100644 --- a/iocore/dns/I_DNSProcessor.h +++ b/iocore/dns/I_DNSProcessor.h @@ -68,13 +68,13 @@ struct DNSProcessor : public Processor { /// Query handler to use. /// Default: single threaded handler. - DNSHandler *handler; + DNSHandler *handler = nullptr; /// Query timeout value. /// Default: @c DEFAULT_DNS_TIMEOUT (or as set in records.config) - int timeout; ///< Timeout value for request. + int timeout = 0; ///< Timeout value for request. /// Host resolution style. /// Default: IPv4, IPv6 ( @c HOST_RES_IPV4 ) - HostResStyle host_res_style; + HostResStyle host_res_style = HOST_RES_IPV4; /// Default constructor. Options(); @@ -121,8 +121,8 @@ struct DNSProcessor : public Processor { // private: // - EThread *thread; - DNSHandler *handler; + EThread *thread = nullptr; + DNSHandler *handler = nullptr; ts_imp_res_state l_res; IpEndpoint local_ipv6; IpEndpoint local_ipv4; @@ -172,7 +172,7 @@ DNSProcessor::gethostbyaddr(Continuation *cont, IpAddr const *addr, Options cons return getby(reinterpret_cast(addr), 0, T_PTR, cont, opt); } -inline DNSProcessor::Options::Options() : handler(nullptr), timeout(0), host_res_style(HOST_RES_IPV4) {} +inline DNSProcessor::Options::Options() {} inline DNSProcessor::Options & DNSProcessor::Options::setHandler(DNSHandler *h) diff --git a/iocore/dns/P_DNSConnection.h b/iocore/dns/P_DNSConnection.h index d4a3e5f27ec..b31c39b4c86 100644 --- a/iocore/dns/P_DNSConnection.h +++ b/iocore/dns/P_DNSConnection.h @@ -46,22 +46,22 @@ struct DNSConnection { /// Connection is done non-blocking. /// Default: @c true. - bool _non_blocking_connect; + bool _non_blocking_connect = true; /// Set socket to have non-blocking I/O. /// Default: @c true. - bool _non_blocking_io; + bool _non_blocking_io = true; /// Use TCP if @c true, use UDP if @c false. /// Default: @c false. - bool _use_tcp; + bool _use_tcp = false; /// Bind to a random port. /// Default: @c true. - bool _bind_random_port; + bool _bind_random_port = true; /// Bind to this local address when using IPv6. /// Default: unset, bind to IN6ADDR_ANY. - sockaddr const *_local_ipv6; + sockaddr const *_local_ipv6 = nullptr; /// Bind to this local address when using IPv4. /// Default: unset, bind to INADDRY_ANY. - sockaddr const *_local_ipv4; + sockaddr const *_local_ipv4 = nullptr; Options(); @@ -75,12 +75,12 @@ struct DNSConnection { int fd; IpEndpoint ip; - int num; + int num = 0; Options opt; LINK(DNSConnection, link); EventIO eio; InkRand generator; - DNSHandler *handler; + DNSHandler *handler = nullptr; /// TCPData structure is to track the reading progress of a TCP connection struct TCPData { @@ -110,15 +110,7 @@ struct DNSConnection { static Options const DEFAULT_OPTIONS; }; -inline DNSConnection::Options::Options() - : _non_blocking_connect(true), - _non_blocking_io(true), - _use_tcp(false), - _bind_random_port(true), - _local_ipv6(nullptr), - _local_ipv4(nullptr) -{ -} +inline DNSConnection::Options::Options() {} inline DNSConnection::Options & DNSConnection::Options::setNonBlockingIo(bool p) diff --git a/iocore/dns/P_DNSProcessor.h b/iocore/dns/P_DNSProcessor.h index 391f53463de..4eb5045a011 100644 --- a/iocore/dns/P_DNSProcessor.h +++ b/iocore/dns/P_DNSProcessor.h @@ -187,25 +187,25 @@ struct DNSHandler : public Continuation { IpEndpoint local_ipv6; ///< Local V6 address if set. IpEndpoint local_ipv4; ///< Local V4 address if set. int ifd[MAX_NAMED]; - int n_con; + int n_con = 0; DNSConnection tcpcon[MAX_NAMED]; DNSConnection udpcon[MAX_NAMED]; Queue entries; Queue triggered; - int in_flight; - int name_server; - int in_write_dns; - HostEnt *hostent_cache; + int in_flight = 0; + int name_server = 0; + int in_write_dns = 0; + HostEnt *hostent_cache = nullptr; int ns_down[MAX_NAMED]; int failover_number[MAX_NAMED]; int failover_soon_number[MAX_NAMED]; ink_hrtime crossed_failover_number[MAX_NAMED]; - ink_hrtime last_primary_retry; - ink_hrtime last_primary_reopen; + ink_hrtime last_primary_retry = 0; + ink_hrtime last_primary_reopen = 0; - ink_res_state m_res; - int txn_lookup_timeout; + ink_res_state m_res = nullptr; + int txn_lookup_timeout = 0; InkRand generator; // bitmap of query ids in use @@ -298,9 +298,9 @@ struct DNSServer { char x_def_domain[MAXDNAME]; char x_domain_srch_list[MAXDNAME]; - DNSHandler *x_dnsH; + DNSHandler *x_dnsH = nullptr; - DNSServer() : x_dnsH(nullptr) + DNSServer() { memset(x_server_ip, 0, sizeof(x_server_ip)); @@ -313,15 +313,7 @@ struct DNSServer { TS_INLINE DNSHandler::DNSHandler() : Continuation(nullptr), - n_con(0), - in_flight(0), - name_server(0), - in_write_dns(0), - hostent_cache(nullptr), - last_primary_retry(0), - last_primary_reopen(0), - m_res(nullptr), - txn_lookup_timeout(0), + generator((uint32_t)((uintptr_t)time(nullptr) ^ (uintptr_t)this)) { ats_ip_invalidate(&ip); diff --git a/iocore/dns/P_SplitDNSProcessor.h b/iocore/dns/P_SplitDNSProcessor.h index d180aad8242..41d0fb92172 100644 --- a/iocore/dns/P_SplitDNSProcessor.h +++ b/iocore/dns/P_SplitDNSProcessor.h @@ -70,7 +70,7 @@ struct SplitDNSResult { /* ------------ public ------------ */ - DNSResultType r; + DNSResultType r = DNS_SRVR_UNDEFINED; DNSServer *get_dns_record(); int get_dns_srvr_count(); @@ -78,10 +78,10 @@ struct SplitDNSResult { /* ------------ private ------------ */ - int m_line_number; + int m_line_number = 0; - SplitDNSRecord *m_rec; - bool m_wrap_around; + SplitDNSRecord *m_rec = nullptr; + bool m_wrap_around = false; }; /* -------------------------------------------------------------- @@ -94,17 +94,17 @@ struct SplitDNS : public ConfigInfo { void *getDNSRecord(const char *hostname); void findServer(RequestData *rdata, SplitDNSResult *result); - DNS_table *m_DNSSrvrTable; + DNS_table *m_DNSSrvrTable = nullptr; - int32_t m_SplitDNSlEnable; + int32_t m_SplitDNSlEnable = 0; /* ---------------------------- required by the alleged fast path ---------------------------- */ - bool m_bEnableFastPath; - void *m_pxLeafArray; - int m_numEle; + bool m_bEnableFastPath = false; + void *m_pxLeafArray = nullptr; + int m_numEle = 0; }; /* -------------------------------------------------------------- @@ -137,14 +137,14 @@ class DNSRequestData : public RequestData sockaddr const *get_ip() override; // unused required virtual method. sockaddr const *get_client_ip() override; // unused required virtual method. - const char *m_pHost; + const char *m_pHost = nullptr; }; /* -------------------------------------------------------------- DNSRequestData::get_string() -------------------------------------------------------------- */ TS_INLINE -DNSRequestData::DNSRequestData() : m_pHost(nullptr) {} +DNSRequestData::DNSRequestData() {} /* -------------------------------------------------------------- DNSRequestData::get_string() @@ -203,15 +203,15 @@ class SplitDNSRecord : public ControlBase void Print(); DNSServer m_servers; - int m_dnsSrvr_cnt; - int m_domain_srch_list; + int m_dnsSrvr_cnt = 0; + int m_domain_srch_list = 0; }; /* -------------------------------------------------------------- SplitDNSRecord::SplitDNSRecord() -------------------------------------------------------------- */ TS_INLINE -SplitDNSRecord::SplitDNSRecord() : m_dnsSrvr_cnt(0), m_domain_srch_list(0) {} +SplitDNSRecord::SplitDNSRecord() {} /* -------------------------------------------------------------- SplitDNSRecord::~SplitDNSRecord() diff --git a/iocore/dns/SplitDNS.cc b/iocore/dns/SplitDNS.cc index 998ef74de6d..11bef367de9 100644 --- a/iocore/dns/SplitDNS.cc +++ b/iocore/dns/SplitDNS.cc @@ -74,14 +74,12 @@ Ptr SplitDNSConfig::dnsHandler_mutex; /* -------------------------------------------------------------- SplitDNSResult::SplitDNSResult() -------------------------------------------------------------- */ -inline SplitDNSResult::SplitDNSResult() : r(DNS_SRVR_UNDEFINED), m_line_number(0), m_rec(nullptr), m_wrap_around(false) {} +inline SplitDNSResult::SplitDNSResult() {} /* -------------------------------------------------------------- SplitDNS::SplitDNS() -------------------------------------------------------------- */ -SplitDNS::SplitDNS() : m_DNSSrvrTable(nullptr), m_SplitDNSlEnable(0), m_bEnableFastPath(false), m_pxLeafArray(nullptr), m_numEle(0) -{ -} +SplitDNS::SplitDNS() {} SplitDNS::~SplitDNS() { diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index 8ce926ab1c8..9a1bbcd0511 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -388,27 +388,27 @@ class EThread : public Thread struct EventMetrics { /// Time the loop was active, not including wait time but including event dispatch time. struct LoopTimes { - ink_hrtime _start; ///< The time of the first loop for this sample. Used to mark valid entries. - ink_hrtime _min; ///< Shortest loop time. - ink_hrtime _max; ///< Longest loop time. - LoopTimes() : _start(0), _min(INT64_MAX), _max(0) {} + ink_hrtime _start = 0; ///< The time of the first loop for this sample. Used to mark valid entries. + ink_hrtime _min = INT64_MAX; ///< Shortest loop time. + ink_hrtime _max = 0; ///< Longest loop time. + LoopTimes() {} } _loop_time; struct Events { - int _min; - int _max; - int _total; - Events() : _min(INT_MAX), _max(0), _total(0) {} + int _min = INT_MAX; + int _max = 0; + int _total = 0; + Events() {} } _events; - int _count; ///< # of times the loop executed. - int _wait; ///< # of timed wait for events + int _count = 0; ///< # of times the loop executed. + int _wait = 0; ///< # of timed wait for events /// Add @a that to @a this data. /// This embodies the custom logic per member concerning whether each is a sum, min, or max. EventMetrics &operator+=(EventMetrics const &that); - EventMetrics() : _count(0), _wait(0) {} + EventMetrics() {} }; /** The number of metric blocks kept. diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h index b57d28fa435..88a54cf9d35 100644 --- a/iocore/eventsystem/I_IOBuffer.h +++ b/iocore/eventsystem/I_IOBuffer.h @@ -239,7 +239,7 @@ class IOBufferData : public RefCountObj alloc or dealloc methods. */ - AllocType _mem_type; + AllocType _mem_type = NO_ALLOC; /** Points to the allocated memory. This member stores the address of @@ -247,10 +247,10 @@ class IOBufferData : public RefCountObj instead use the alloc or dealloc methods. */ - char *_data; + char *_data = nullptr; #ifdef TRACK_BUFFER_USER - const char *_location; + const char *_location = nullptr; #endif /** @@ -258,16 +258,7 @@ class IOBufferData : public RefCountObj this method. Use one of the functions with the 'new_' prefix instead. */ - IOBufferData() - : _size_index(BUFFER_SIZE_NOT_ALLOCATED), - _mem_type(NO_ALLOC), - _data(nullptr) -#ifdef TRACK_BUFFER_USER - , - _location(nullptr) -#endif - { - } + IOBufferData() : _size_index(BUFFER_SIZE_NOT_ALLOCATED) {} // noncopyable, declaration only IOBufferData(const IOBufferData &) = delete; @@ -489,12 +480,12 @@ class IOBufferBlock : public RefCountObj */ void free() override; - char *_start; - char *_end; - char *_buf_end; + char *_start = nullptr; + char *_end = nullptr; + char *_buf_end = nullptr; #ifdef TRACK_BUFFER_USER - const char *_location; + const char *_location = nullptr; #endif /** @@ -893,14 +884,14 @@ class IOBufferReader return mbuf; } - MIOBufferAccessor *accessor; // pointer back to the accessor + MIOBufferAccessor *accessor = nullptr; // pointer back to the accessor /** Back pointer to this object's MIOBuffer. A pointer back to the MIOBuffer this reader is allocated from. */ - MIOBuffer *mbuf; + MIOBuffer *mbuf = nullptr; Ptr block; /** @@ -909,10 +900,10 @@ class IOBufferReader of the available data. */ - int64_t start_offset; - int64_t size_limit; + int64_t start_offset = 0; + int64_t size_limit = INT64_MAX; - IOBufferReader() : accessor(nullptr), mbuf(nullptr), start_offset(0), size_limit(INT64_MAX) {} + IOBufferReader() {} }; /** @@ -1303,7 +1294,7 @@ class MIOBuffer IOBufferReader readers[MAX_MIOBUFFER_READERS]; #ifdef TRACK_BUFFER_USER - const char *_location; + const char *_location = nullptr; #endif MIOBuffer(void *b, int64_t bufsize, int64_t aWater_mark); @@ -1352,20 +1343,12 @@ struct MIOBufferAccessor { entry = nullptr; } - MIOBufferAccessor() - : -#ifdef DEBUG - name(nullptr), -#endif - mbuf(nullptr), - entry(nullptr) - { - } + MIOBufferAccessor() {} ~MIOBufferAccessor(); #ifdef DEBUG - const char *name; + const char *name = nullptr; #endif // noncopyable @@ -1373,8 +1356,8 @@ struct MIOBufferAccessor { MIOBufferAccessor &operator=(const MIOBufferAccessor &) = delete; private: - MIOBuffer *mbuf; - IOBufferReader *entry; + MIOBuffer *mbuf = nullptr; + IOBufferReader *entry = nullptr; }; extern MIOBuffer *new_MIOBuffer_internal( diff --git a/iocore/eventsystem/I_ProxyAllocator.h b/iocore/eventsystem/I_ProxyAllocator.h index 4cf8eaeb3cf..a5edd125fd0 100644 --- a/iocore/eventsystem/I_ProxyAllocator.h +++ b/iocore/eventsystem/I_ProxyAllocator.h @@ -39,10 +39,10 @@ extern int thread_freelist_low_watermark; extern int cmd_disable_pfreelist; struct ProxyAllocator { - int allocated; - void *freelist; + int allocated = 0; + void *freelist = nullptr; - ProxyAllocator() : allocated(0), freelist(nullptr) {} + ProxyAllocator() {} }; template diff --git a/iocore/eventsystem/I_VIO.h b/iocore/eventsystem/I_VIO.h index 6e7748c6640..40d8fa444a6 100644 --- a/iocore/eventsystem/I_VIO.h +++ b/iocore/eventsystem/I_VIO.h @@ -165,7 +165,7 @@ class VIO call with events for this operation. */ - Continuation *cont; + Continuation *cont = nullptr; /** Number of bytes to be done for this operation. @@ -173,7 +173,7 @@ class VIO The total number of bytes this operation must complete. */ - int64_t nbytes; + int64_t nbytes = 0; /** Number of bytes already completed. @@ -183,7 +183,7 @@ class VIO the lock. */ - int64_t ndone; + int64_t ndone = 0; /** Type of operation. @@ -191,7 +191,7 @@ class VIO The type of operation that this VIO represents. */ - int op; + int op = VIO::NONE; /** Provides access to the reader or writer for this operation. @@ -207,7 +207,7 @@ class VIO functions. */ - VConnection *vc_server; + VConnection *vc_server = nullptr; /** Reference to the state machine's mutex. diff --git a/iocore/eventsystem/P_IOBuffer.h b/iocore/eventsystem/P_IOBuffer.h index d6a4bb49454..4968ff90e39 100644 --- a/iocore/eventsystem/P_IOBuffer.h +++ b/iocore/eventsystem/P_IOBuffer.h @@ -382,13 +382,6 @@ new_IOBufferBlock_internal( TS_INLINE IOBufferBlock::IOBufferBlock() - : _start(nullptr), - _end(nullptr), - _buf_end(nullptr) -#ifdef TRACK_BUFFER_USER - , - _location(nullptr) -#endif { return; } diff --git a/iocore/eventsystem/P_VIO.h b/iocore/eventsystem/P_VIO.h index d8fcb05451b..296296c07b8 100644 --- a/iocore/eventsystem/P_VIO.h +++ b/iocore/eventsystem/P_VIO.h @@ -25,7 +25,7 @@ #include "I_VIO.h" TS_INLINE -VIO::VIO(int aop) : cont(nullptr), nbytes(0), ndone(0), op(aop), buffer(), vc_server(nullptr), mutex(nullptr) {} +VIO::VIO(int aop) : op(aop), buffer(), mutex(nullptr) {} ///////////////////////////////////////////////////////////// // @@ -33,7 +33,7 @@ VIO::VIO(int aop) : cont(nullptr), nbytes(0), ndone(0), op(aop), buffer(), vc_se // ///////////////////////////////////////////////////////////// TS_INLINE -VIO::VIO() : cont(nullptr), nbytes(0), ndone(0), op(VIO::NONE), buffer(), vc_server(nullptr), mutex(nullptr) {} +VIO::VIO() : buffer(), mutex(nullptr) {} TS_INLINE Continuation * VIO::get_continuation() diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 91ea46af1aa..e4a1427be68 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -217,9 +217,7 @@ HostDBHash::refresh() ctx.finalize(hash); } -HostDBHash::HostDBHash() : host_name(nullptr), host_len(0), port(0), dns_server(nullptr), pSD(nullptr), db_mark(HOSTDB_MARK_GENERIC) -{ -} +HostDBHash::HostDBHash() {} HostDBHash::~HostDBHash() { @@ -228,7 +226,7 @@ HostDBHash::~HostDBHash() } } -HostDBCache::HostDBCache() : refcountcache(nullptr), pending_dns(nullptr), remoteHostDBQueue(nullptr) +HostDBCache::HostDBCache() { hosts_file_ptr = new RefCountedHostsFileMap(); } diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h index 347f1a94832..25871315f97 100644 --- a/iocore/hostdb/I_HostDBProcessor.h +++ b/iocore/hostdb/I_HostDBProcessor.h @@ -349,13 +349,13 @@ struct HostDBInfo : public RefCountObj { struct HostDBRoundRobin { /** Total number (to compute space used). */ - short rrcount; + short rrcount = 0; /** Number which have not failed a connect. */ - short good; + short good = 0; - unsigned short current; - ink_time_t timed_rr_ctime; + unsigned short current = 0; + ink_time_t timed_rr_ctime = 0; // This is the equivalent of a variable length array, we can't use a VLA because // HostDBInfo is a non-POD type-- so this is the best we can do. @@ -389,7 +389,7 @@ struct HostDBRoundRobin { HostDBInfo *select_next(sockaddr const *addr); HostDBInfo *select_best_http(sockaddr const *client_ip, ink_time_t now, int32_t fail_window); HostDBInfo *select_best_srv(char *target, InkRand *rand, ink_time_t now, int32_t fail_window); - HostDBRoundRobin() : rrcount(0), good(0), current(0), timed_rr_ctime(0) {} + HostDBRoundRobin() {} }; struct HostDBCache; @@ -424,13 +424,13 @@ struct HostDBProcessor : public Processor { /// Optional parameters for getby... struct Options { - typedef Options self; ///< Self reference type. - int port; ///< Target service port (default 0 -> don't care) - int flags; ///< Processing flags (default HOSTDB_DO_NOT_FORCE_DNS) - int timeout; ///< Timeout value (default 0 -> default timeout) - HostResStyle host_res_style; ///< How to query host (default HOST_RES_IPV4) + typedef Options self; ///< Self reference type. + int port = 0; ///< Target service port (default 0 -> don't care) + int flags = HOSTDB_DO_NOT_FORCE_DNS; ///< Processing flags (default HOSTDB_DO_NOT_FORCE_DNS) + int timeout = 0; ///< Timeout value (default 0 -> default timeout) + HostResStyle host_res_style = HOST_RES_IPV4; ///< How to query host (default HOST_RES_IPV4) - Options() : port(0), flags(HOSTDB_DO_NOT_FORCE_DNS), timeout(0), host_res_style(HOST_RES_IPV4) {} + Options() {} /// Set the flags. self & setFlags(int f) diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h index 15d1c47f704..00b8731a36b 100644 --- a/iocore/hostdb/P_HostDBProcessor.h +++ b/iocore/hostdb/P_HostDBProcessor.h @@ -194,12 +194,12 @@ struct HostDBCache { // Map to contain all of the host file overrides, initialize it to empty Ptr hosts_file_ptr; // TODO: make ATS call a close() method or something on shutdown (it does nothing of the sort today) - RefCountCache *refcountcache; + RefCountCache *refcountcache = nullptr; // TODO configurable number of items in the cache - Queue *pending_dns; + Queue *pending_dns = nullptr; Queue &pending_dns_for_hash(const CryptoHash &hash); - Queue *remoteHostDBQueue; + Queue *remoteHostDBQueue = nullptr; HostDBCache(); bool is_pending_dns_for_hash(const CryptoHash &hash); }; @@ -399,16 +399,16 @@ struct HostDBHash { CryptoHash hash; ///< The hash value. - const char *host_name; ///< Host name. - int host_len; ///< Length of @a _host_name - IpAddr ip; ///< IP address. - in_port_t port; ///< IP port (host order). + const char *host_name = nullptr; ///< Host name. + int host_len = 0; ///< Length of @a _host_name + IpAddr ip; ///< IP address. + in_port_t port = 0; ///< IP port (host order). /// DNS server. Not strictly part of the hash data but /// it's both used by @c HostDBContinuation and provides access to /// hash data. It's just handier to store it here for both uses. - DNSServer *dns_server; - SplitDNS *pSD; ///< Hold the container for @a dns_server. - HostDBMark db_mark; ///< Mark / type of record. + DNSServer *dns_server = nullptr; + SplitDNS *pSD = nullptr; ///< Hold the container for @a dns_server. + HostDBMark db_mark = HOSTDB_MARK_GENERIC; ///< Mark / type of record. /// Default constructor. HostDBHash(); @@ -490,12 +490,12 @@ struct HostDBContinuation : public Continuation { struct Options { typedef Options self; ///< Self reference type. - int timeout; ///< Timeout value. Default 0 - HostResStyle host_res_style; ///< IP address family fallback. Default @c HOST_RES_NONE - bool force_dns; ///< Force DNS lookup. Default @c false - Continuation *cont; ///< Continuation / action. Default @c nullptr (none) + int timeout = 0; ///< Timeout value. Default 0 + HostResStyle host_res_style = HOST_RES_NONE; ///< IP address family fallback. Default @c HOST_RES_NONE + bool force_dns = false; ///< Force DNS lookup. Default @c false + Continuation *cont = nullptr; ///< Continuation / action. Default @c nullptr (none) - Options() : timeout(0), host_res_style(HOST_RES_NONE), force_dns(false), cont(nullptr) {} + Options() {} }; static const Options DEFAULT_OPTIONS; ///< Default defaults. void init(HostDBHash const &hash, Options const &opt = DEFAULT_OPTIONS); diff --git a/iocore/hostdb/P_RefCountCache.h b/iocore/hostdb/P_RefCountCache.h index 1d734e46d05..60feab110ea 100644 --- a/iocore/hostdb/P_RefCountCache.h +++ b/iocore/hostdb/P_RefCountCache.h @@ -77,11 +77,11 @@ class RefCountCacheHashEntry Ptr item; RefCountCacheHashEntry *_next{nullptr}; RefCountCacheHashEntry *_prev{nullptr}; - PriorityQueueEntry *expiry_entry; + PriorityQueueEntry *expiry_entry = nullptr; RefCountCacheItemMeta meta; // Need a no-argument constructor to use the classAllocator - RefCountCacheHashEntry() : item(Ptr()), expiry_entry(nullptr), meta(0, 0) {} + RefCountCacheHashEntry() : item(Ptr()), meta(0, 0) {} void set(RefCountObj *i, uint64_t key, unsigned int size, int expire_time) { @@ -368,7 +368,7 @@ RefCountCachePartition::get_map() class RefCountCacheHeader { public: - unsigned int magic; + unsigned int magic = REFCOUNTCACHE_MAGIC_NUMBER; ts::VersionNumber version{REFCOUNTCACHE_VERSION}; ts::VersionNumber object_version; // version passed in of whatever it is we are caching diff --git a/iocore/hostdb/RefCountCache.cc b/iocore/hostdb/RefCountCache.cc index 832a880c648..b7cecae96c7 100644 --- a/iocore/hostdb/RefCountCache.cc +++ b/iocore/hostdb/RefCountCache.cc @@ -38,8 +38,7 @@ RefCountCacheHashEntry::dealloc(RefCountCacheHashEntry *e) return refCountCacheHashingValueAllocator.free(e); } -RefCountCacheHeader::RefCountCacheHeader(ts::VersionNumber object_version) - : magic(REFCOUNTCACHE_MAGIC_NUMBER), object_version(object_version){}; +RefCountCacheHeader::RefCountCacheHeader(ts::VersionNumber object_version) : object_version(object_version){}; bool RefCountCacheHeader::operator==(const RefCountCacheHeader other) const diff --git a/iocore/net/Connection.cc b/iocore/net/Connection.cc index 1595bb9791a..a8370a65d3a 100644 --- a/iocore/net/Connection.cc +++ b/iocore/net/Connection.cc @@ -57,7 +57,7 @@ NetVCOptions::toString(addr_bind_style s) return ANY_ADDR == s ? "any" : INTF_ADDR == s ? "interface" : "foreign"; } -Connection::Connection() : fd(NO_FD), is_bound(false), is_connected(false), sock_type(0) +Connection::Connection() : fd(NO_FD) { memset(&addr, 0, sizeof(addr)); } diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index 786c7afe708..e9065680549 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -612,8 +612,8 @@ class NetVConnection : public AnnotatedVConnection // is enabled by SocksProxy SocksAddrType socks_addr; - unsigned int attributes; - EThread *thread; + unsigned int attributes = 0; + EThread *thread = nullptr; /// PRIVATE: The public interface is VIO::reenable() void reenable(VIO *vio) override = 0; @@ -815,31 +815,22 @@ class NetVConnection : public AnnotatedVConnection IpEndpoint local_addr; IpEndpoint remote_addr; - bool got_local_addr; - bool got_remote_addr; + bool got_local_addr = false; + bool got_remote_addr = false; - bool is_internal_request; + bool is_internal_request = false; /// Set if this connection is transparent. - bool is_transparent; + bool is_transparent = false; /// Set if proxy protocol is enabled - bool is_proxy_protocol; + bool is_proxy_protocol = false; /// Set if the next write IO that empties the write buffer should generate an event. - int write_buffer_empty_event; + int write_buffer_empty_event = 0; /// NetVConnection Context. - NetVConnectionContext_t netvc_context; + NetVConnectionContext_t netvc_context = NET_VCONNECTION_UNSET; }; -inline NetVConnection::NetVConnection() - : AnnotatedVConnection(nullptr), - attributes(0), - thread(nullptr), - got_local_addr(false), - got_remote_addr(false), - is_internal_request(false), - is_transparent(false), - is_proxy_protocol(false), - write_buffer_empty_event(0), - netvc_context(NET_VCONNECTION_UNSET) +inline NetVConnection::NetVConnection() : AnnotatedVConnection(nullptr) + { ink_zero(local_addr); ink_zero(remote_addr); diff --git a/iocore/net/I_Socks.h b/iocore/net/I_Socks.h index 0c2b5c61d6a..b1e7e5c75ba 100644 --- a/iocore/net/I_Socks.h +++ b/iocore/net/I_Socks.h @@ -56,7 +56,7 @@ enum { }; struct SocksAddrType { - unsigned char type; + unsigned char type = SOCKS_ATYPE_NONE; union { // mostly it is ipv4. in other cases we will xalloc(). unsigned char ipv4[4]; @@ -64,6 +64,6 @@ struct SocksAddrType { } addr; void reset(); - SocksAddrType() : type(SOCKS_ATYPE_NONE) { addr.buf = nullptr; } + SocksAddrType() { addr.buf = nullptr; } ~SocksAddrType() { reset(); } }; diff --git a/iocore/net/NetVCTest.cc b/iocore/net/NetVCTest.cc index 7889cd13bf6..bda22bd7390 100644 --- a/iocore/net/NetVCTest.cc +++ b/iocore/net/NetVCTest.cc @@ -80,37 +80,7 @@ NVC_test_def netvc_tests_def[] = { }; const unsigned num_netvc_tests = countof(netvc_tests_def); -NetVCTest::NetVCTest() - : Continuation(nullptr), - test_cont_type(NET_VC_TEST_ACTIVE), - test_vc(nullptr), - regress(nullptr), - driver(nullptr), - read_vio(nullptr), - write_vio(nullptr), - read_buffer(nullptr), - write_buffer(nullptr), - reader_for_rbuf(nullptr), - reader_for_wbuf(nullptr), - write_bytes_to_add_per(0), - timeout(0), - actual_bytes_read(0), - actual_bytes_sent(0), - write_done(false), - read_done(false), - read_seed(0), - write_seed(0), - bytes_to_send(0), - bytes_to_read(0), - nbytes_read(0), - nbytes_write(0), - expected_read_term(0), - expected_write_term(0), - test_name(nullptr), - module_name(nullptr), - debug_tag(nullptr) -{ -} +NetVCTest::NetVCTest() : Continuation(nullptr) {} NetVCTest::~NetVCTest() { @@ -395,6 +365,6 @@ NetVCTest::main_handler(int event, void *data) return 0; } -NetTestDriver::NetTestDriver() : Continuation(nullptr), errors(0), r(nullptr), pstatus(nullptr) {} +NetTestDriver::NetTestDriver() : Continuation(nullptr) {} NetTestDriver::~NetTestDriver() {} diff --git a/iocore/net/P_Connection.h b/iocore/net/P_Connection.h index f7d4d687515..9741fd7c37d 100644 --- a/iocore/net/P_Connection.h +++ b/iocore/net/P_Connection.h @@ -78,11 +78,11 @@ struct NetVCOptions; // /////////////////////////////////////////////////////////////////////// struct Connection { - SOCKET fd; ///< Socket for connection. - IpEndpoint addr; ///< Associated address. - bool is_bound; ///< Flag for already bound to a local address. - bool is_connected; ///< Flag for already connected. - int sock_type; + SOCKET fd; ///< Socket for connection. + IpEndpoint addr; ///< Associated address. + bool is_bound = false; ///< Flag for already bound to a local address. + bool is_connected = false; ///< Flag for already connected. + int sock_type = 0; /** Create and initialize the socket for this connection. @@ -160,7 +160,7 @@ struct Server : public Connection { IpEndpoint accept_addr; /// If set, a kernel HTTP accept filter - bool http_accept_filter; + bool http_accept_filter = false; int accept(Connection *c); @@ -173,5 +173,5 @@ struct Server : public Connection { int listen(bool non_blocking, const NetProcessor::AcceptOptions &opt); int setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions &opt); - Server() : Connection(), http_accept_filter(false) { ink_zero(accept_addr); } + Server() : Connection() { ink_zero(accept_addr); } }; diff --git a/iocore/net/P_NetVCTest.h b/iocore/net/P_NetVCTest.h index 08be426a088..eedbb8d2319 100644 --- a/iocore/net/P_NetVCTest.h +++ b/iocore/net/P_NetVCTest.h @@ -73,11 +73,11 @@ class NetTestDriver : public Continuation NetTestDriver(); ~NetTestDriver() override; - int errors; + int errors = 0; protected: - RegressionTest *r; - int *pstatus; + RegressionTest *r = nullptr; + int *pstatus = nullptr; }; class NetVCTest : public Continuation @@ -85,7 +85,7 @@ class NetVCTest : public Continuation public: NetVCTest(); ~NetVCTest() override; - NetVcTestType_t test_cont_type; + NetVcTestType_t test_cont_type = NET_VC_TEST_ACTIVE; int main_handler(int event, void *data); void read_handler(int event); @@ -103,41 +103,41 @@ class NetVCTest : public Continuation void finished(); void record_error(const char *msg); - NetVConnection *test_vc; - RegressionTest *regress; - NetTestDriver *driver; + NetVConnection *test_vc = nullptr; + RegressionTest *regress = nullptr; + NetTestDriver *driver = nullptr; - VIO *read_vio; - VIO *write_vio; + VIO *read_vio = nullptr; + VIO *write_vio = nullptr; - MIOBuffer *read_buffer; - MIOBuffer *write_buffer; + MIOBuffer *read_buffer = nullptr; + MIOBuffer *write_buffer = nullptr; - IOBufferReader *reader_for_rbuf; - IOBufferReader *reader_for_wbuf; + IOBufferReader *reader_for_rbuf = nullptr; + IOBufferReader *reader_for_wbuf = nullptr; - int write_bytes_to_add_per; - int timeout; + int write_bytes_to_add_per = 0; + int timeout = 0; - int actual_bytes_read; - int actual_bytes_sent; + int actual_bytes_read = 0; + int actual_bytes_sent = 0; - bool write_done; - bool read_done; + bool write_done = false; + bool read_done = false; - uint8_t read_seed; - uint8_t write_seed; + uint8_t read_seed = 0; + uint8_t write_seed = 0; - int bytes_to_send; - int bytes_to_read; + int bytes_to_send = 0; + int bytes_to_read = 0; - int nbytes_read; - int nbytes_write; + int nbytes_read = 0; + int nbytes_write = 0; - int expected_read_term; - int expected_write_term; + int expected_read_term = 0; + int expected_write_term = 0; - const char *test_name; - const char *module_name; - const char *debug_tag; + const char *test_name = nullptr; + const char *module_name = nullptr; + const char *debug_tag = nullptr; }; diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 4c111f0e14b..4c27c73e90c 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -117,7 +117,7 @@ class VerifyClient : public ActionItem class TLSValidProtocols : public ActionItem { - bool unset; + bool unset = true; unsigned long protocol_mask; public: @@ -126,7 +126,7 @@ class TLSValidProtocols : public ActionItem #else static const unsigned long max_mask = SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2; #endif - TLSValidProtocols() : unset(true), protocol_mask(max_mask) {} + TLSValidProtocols() : protocol_mask(max_mask) {} TLSValidProtocols(unsigned long protocols) : unset(false), protocol_mask(protocols) {} int SNIAction(Continuation *cont) const override diff --git a/iocore/net/P_SSLCertLookup.h b/iocore/net/P_SSLCertLookup.h index 475d05de0cf..3a793408c27 100644 --- a/iocore/net/P_SSLCertLookup.h +++ b/iocore/net/P_SSLCertLookup.h @@ -61,21 +61,21 @@ struct SSLCertContext { OPT_TUNNEL ///< Just tunnel, don't terminate. }; - SSLCertContext() : ctx(nullptr), opt(OPT_NONE), keyblock(nullptr) {} - explicit SSLCertContext(SSL_CTX *c) : ctx(c), opt(OPT_NONE), keyblock(nullptr) {} - SSLCertContext(SSL_CTX *c, Option o) : ctx(c), opt(o), keyblock(nullptr) {} + SSLCertContext() {} + explicit SSLCertContext(SSL_CTX *c) : ctx(c) {} + SSLCertContext(SSL_CTX *c, Option o) : ctx(c), opt(o) {} SSLCertContext(SSL_CTX *c, Option o, ssl_ticket_key_block *kb) : ctx(c), opt(o), keyblock(kb) {} void release(); - SSL_CTX *ctx; ///< openSSL context. - Option opt; ///< Special handling option. - ssl_ticket_key_block *keyblock; ///< session keys associated with this address + SSL_CTX *ctx = nullptr; ///< openSSL context. + Option opt = OPT_NONE; ///< Special handling option. + ssl_ticket_key_block *keyblock = nullptr; ///< session keys associated with this address }; struct SSLCertLookup : public ConfigInfo { SSLContextStorage *ssl_storage; - SSL_CTX *ssl_default; - bool is_valid; + SSL_CTX *ssl_default = nullptr; + bool is_valid = true; int insert(const char *name, SSLCertContext const &cc); int insert(const IpEndpoint &address, SSLCertContext const &cc); diff --git a/iocore/net/P_SSLNextProtocolSet.h b/iocore/net/P_SSLNextProtocolSet.h index 067690666e6..61d1ded9a8b 100644 --- a/iocore/net/P_SSLNextProtocolSet.h +++ b/iocore/net/P_SSLNextProtocolSet.h @@ -60,8 +60,8 @@ class SSLNextProtocolSet SSLNextProtocolSet &operator=(const SSLNextProtocolSet &) = delete; // disabled private: - mutable unsigned char *npn; - mutable size_t npnsz; + mutable unsigned char *npn = nullptr; + mutable size_t npnsz = 0; NextProtocolEndpoint::list_type endpoints; }; diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index 7a8a9758fb1..b154dc1e9c9 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -43,10 +43,7 @@ typedef int ssl_error_t; @brief Gather user provided settings from ssl_multicert.config in to this single struct */ struct SSLMultiCertConfigParams { - SSLMultiCertConfigParams() : opt(SSLCertContext::OPT_NONE) - { - REC_ReadConfigInt32(session_ticket_enabled, "proxy.config.ssl.server.session_ticket.enable"); - } + SSLMultiCertConfigParams() { REC_ReadConfigInt32(session_ticket_enabled, "proxy.config.ssl.server.session_ticket.enable"); } int session_ticket_enabled; ///< session ticket enabled ats_scoped_str addr; ///< IPv[64] address to match @@ -56,7 +53,7 @@ struct SSLMultiCertConfigParams { ats_scoped_str key; ///< Private key ats_scoped_str dialog; ///< Private key dialog ats_scoped_str servername; ///< Destination server - SSLCertContext::Option opt; ///< SSLCertContext special handling option + SSLCertContext::Option opt = SSLCertContext::OPT_NONE; ///< SSLCertContext special handling option }; /** diff --git a/iocore/net/P_Socks.h b/iocore/net/P_Socks.h index aba0a0b867b..2f875c1e4e3 100644 --- a/iocore/net/P_Socks.h +++ b/iocore/net/P_Socks.h @@ -39,20 +39,20 @@ enum { }; struct socks_conf_struct { - int socks_needed; - int server_connect_timeout; - int socks_timeout; - unsigned char default_version; - char *user_name_n_passwd; - int user_name_n_passwd_len; + int socks_needed = 0; + int server_connect_timeout = 0; + int socks_timeout = 100; + unsigned char default_version = 5; + char *user_name_n_passwd = nullptr; + int user_name_n_passwd_len = 0; - int per_server_connection_attempts; - int connection_attempts; + int per_server_connection_attempts = 1; + int connection_attempts = 0; // the following ports are used by SocksProxy - int accept_enabled; - int accept_port; - unsigned short http_port; + int accept_enabled = 0; + int accept_port = 0; + unsigned short http_port = 1080; #ifdef SOCKS_WITH_TS IpMap ip_map; @@ -63,17 +63,7 @@ struct socks_conf_struct { #endif socks_conf_struct() - : socks_needed(0), - server_connect_timeout(0), - socks_timeout(100), - default_version(5), - user_name_n_passwd(nullptr), - user_name_n_passwd_len(0), - per_server_connection_attempts(1), - connection_attempts(0), - accept_enabled(0), - accept_port(0), - http_port(1080) + { #if !defined(SOCKS_WITH_TS) memset(&server_addr, 0, sizeof(server_addr)); diff --git a/iocore/net/P_UDPConnection.h b/iocore/net/P_UDPConnection.h index ed08d89df99..443df0ed07c 100644 --- a/iocore/net/P_UDPConnection.h +++ b/iocore/net/P_UDPConnection.h @@ -38,14 +38,14 @@ class UDPConnectionInternal : public UDPConnection UDPConnectionInternal(); ~UDPConnectionInternal() override; - Continuation *continuation; - int recvActive; // interested in receiving - int refcount; // public for assertion + Continuation *continuation = nullptr; + int recvActive = 0; // interested in receiving + int refcount = 0; // public for assertion SOCKET fd; IpEndpoint binding; - int binding_valid; - int tobedestroyed; + int binding_valid = 0; + int tobedestroyed = 0; int sendGenerationNum; int64_t lastSentPktTSSeqNum; @@ -60,8 +60,7 @@ class UDPConnectionInternal : public UDPConnection }; TS_INLINE -UDPConnectionInternal::UDPConnectionInternal() - : continuation(nullptr), recvActive(0), refcount(0), fd(-1), binding_valid(0), tobedestroyed(0) +UDPConnectionInternal::UDPConnectionInternal() : fd(-1) { sendGenerationNum = 0; lastSentPktTSSeqNum = -1; diff --git a/iocore/net/P_UDPIOEvent.h b/iocore/net/P_UDPIOEvent.h index b89c9c7651d..593929e0018 100644 --- a/iocore/net/P_UDPIOEvent.h +++ b/iocore/net/P_UDPIOEvent.h @@ -28,7 +28,7 @@ class UDPIOEvent : public Event { public: - UDPIOEvent() : fd(-1), err(0), m(nullptr), handle(nullptr), b(nullptr), bytesTransferred(0){}; + UDPIOEvent() : b(nullptr){}; ~UDPIOEvent() override{}; void @@ -90,12 +90,12 @@ class UDPIOEvent : public Event private: void *operator new(size_t size); // undefined - int fd; - int err; // error code - struct msghdr *m; - void *handle; // some extra data for the client handler - Ptr b; // holds buffer that I/O will go to - int bytesTransferred; // actual bytes transferred + int fd = -1; + int err = 0; // error code + struct msghdr *m = nullptr; + void *handle = nullptr; // some extra data for the client handler + Ptr b; // holds buffer that I/O will go to + int bytesTransferred = 0; // actual bytes transferred }; extern ClassAllocator UDPIOEventAllocator; diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index c23dc07a6e7..5340178e81c 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -61,19 +61,19 @@ extern UDPNetProcessorInternal udpNetInternal; class PacketQueue { public: - PacketQueue() : nPackets(0), now_slot(0) + PacketQueue() { lastPullLongTermQ = 0; init(); } virtual ~PacketQueue() {} - int nPackets; + int nPackets = 0; ink_hrtime lastPullLongTermQ; Queue longTermQ; Queue bucket[N_SLOTS]; ink_hrtime delivery_time[N_SLOTS]; - int now_slot; + int now_slot = 0; void init() diff --git a/iocore/net/P_UDPPacket.h b/iocore/net/P_UDPPacket.h index c76deee7f60..80426caa2fc 100644 --- a/iocore/net/P_UDPPacket.h +++ b/iocore/net/P_UDPPacket.h @@ -44,24 +44,24 @@ class UDPPacketInternal : public UDPPacket SLINK(UDPPacketInternal, alink); // atomic link // packet scheduling stuff: keep it a doubly linked list - uint64_t pktLength; + uint64_t pktLength = 0; - int reqGenerationNum; - ink_hrtime delivery_time; // when to deliver packet + int reqGenerationNum = 0; + ink_hrtime delivery_time = 0; // when to deliver packet Ptr chain; - Continuation *cont; // callback on error - UDPConnectionInternal *conn; // connection where packet should be sent to. + Continuation *cont = nullptr; // callback on error + UDPConnectionInternal *conn = nullptr; // connection where packet should be sent to. - int in_the_priority_queue; - int in_heap; + int in_the_priority_queue = 0; + int in_heap = 0; }; inkcoreapi extern ClassAllocator udpPacketAllocator; TS_INLINE UDPPacketInternal::UDPPacketInternal() - : pktLength(0), reqGenerationNum(0), delivery_time(0), cont(nullptr), conn(nullptr), in_the_priority_queue(0), in_heap(0) + { memset(&from, '\0', sizeof(from)); memset(&to, '\0', sizeof(to)); diff --git a/iocore/net/P_UnixNetState.h b/iocore/net/P_UnixNetState.h index 10cbf4c75e7..3a2265c92c9 100644 --- a/iocore/net/P_UnixNetState.h +++ b/iocore/net/P_UnixNetState.h @@ -43,12 +43,12 @@ class Event; class UnixNetVConnection; struct NetState { - int enabled; + int enabled = 0; VIO vio; Link ready_link; SLink enable_link; - int in_enabled_list; - int triggered; + int in_enabled_list = 0; + int triggered = 0; - NetState() : enabled(0), vio(VIO::NONE), in_enabled_list(0), triggered(0) {} + NetState() : vio(VIO::NONE) {} }; diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index ba5709171ac..f56a208c735 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -232,7 +232,7 @@ class UnixNetVConnection : public NetVConnection UnixNetVConnection *migrateToCurrentThread(Continuation *c, EThread *t); Action action_; - int closed; + int closed = 0; NetState read; NetState write; @@ -244,14 +244,14 @@ class UnixNetVConnection : public NetVConnection LINK(UnixNetVConnection, keep_alive_queue_link); LINK(UnixNetVConnection, active_queue_link); - ink_hrtime inactivity_timeout_in; - ink_hrtime active_timeout_in; - ink_hrtime next_inactivity_timeout_at; - ink_hrtime next_activity_timeout_at; + ink_hrtime inactivity_timeout_in = 0; + ink_hrtime active_timeout_in = 0; + ink_hrtime next_inactivity_timeout_at = 0; + ink_hrtime next_activity_timeout_at = 0; EventIO ep; - NetHandler *nh; - unsigned int id; + NetHandler *nh = nullptr; + unsigned int id = 0; union { unsigned int flags; @@ -264,11 +264,11 @@ class UnixNetVConnection : public NetVConnection }; Connection con; - int recursion; - ink_hrtime submit_time; - OOB_callback *oob_ptr; - bool from_accept_thread; - NetAccept *accept_object; + int recursion = 0; + ink_hrtime submit_time = 0; + OOB_callback *oob_ptr = nullptr; + bool from_accept_thread = false; + NetAccept *accept_object = nullptr; // es - origin_trace associated connections bool origin_trace; diff --git a/iocore/net/SSLCertLookup.cc b/iocore/net/SSLCertLookup.cc index bf6c1a02bdb..641f7abaa5b 100644 --- a/iocore/net/SSLCertLookup.cc +++ b/iocore/net/SSLCertLookup.cc @@ -113,14 +113,14 @@ struct SSLContextStorage { linkage required by @c Trie. */ struct ContextRef { - ContextRef() : idx(-1) {} + ContextRef() {} explicit ContextRef(int n) : idx(n) {} void Print() const { Debug("ssl", "Item=%p SSL_CTX=#%d", this, idx); } - int idx; ///< Index in the context store. + int idx = -1; ///< Index in the context store. LINK(ContextRef, link); ///< Require by @c Trie }; @@ -239,7 +239,7 @@ SSLCertContext::release() ctx = nullptr; } -SSLCertLookup::SSLCertLookup() : ssl_storage(new SSLContextStorage()), ssl_default(nullptr), is_valid(true) {} +SSLCertLookup::SSLCertLookup() : ssl_storage(new SSLContextStorage()) {} SSLCertLookup::~SSLCertLookup() { diff --git a/iocore/net/SSLNextProtocolSet.cc b/iocore/net/SSLNextProtocolSet.cc index 6c9d4df6a44..2adefce1e0c 100644 --- a/iocore/net/SSLNextProtocolSet.cc +++ b/iocore/net/SSLNextProtocolSet.cc @@ -157,7 +157,7 @@ SSLNextProtocolSet::findEndpoint(const unsigned char *proto, unsigned len) const return nullptr; } -SSLNextProtocolSet::SSLNextProtocolSet() : npn(nullptr), npnsz(0) {} +SSLNextProtocolSet::SSLNextProtocolSet() {} SSLNextProtocolSet::~SSLNextProtocolSet() { diff --git a/iocore/net/SSLSessionCache.cc b/iocore/net/SSLSessionCache.cc index 1238b4baeb0..57e01ab23b2 100644 --- a/iocore/net/SSLSessionCache.cc +++ b/iocore/net/SSLSessionCache.cc @@ -36,7 +36,7 @@ #endif /* Session Cache */ -SSLSessionCache::SSLSessionCache() : session_bucket(nullptr), nbuckets(SSLConfigParams::session_cache_number_buckets) +SSLSessionCache::SSLSessionCache() : nbuckets(SSLConfigParams::session_cache_number_buckets) { Debug("ssl.session_cache", "Created new ssl session cache %p with %zu buckets each with size max size %zu", this, nbuckets, SSLConfigParams::session_cache_max_bucket_size); diff --git a/iocore/net/SSLSessionCache.h b/iocore/net/SSLSessionCache.h index 9c763a83434..5a5d50fc058 100644 --- a/iocore/net/SSLSessionCache.h +++ b/iocore/net/SSLSessionCache.h @@ -154,6 +154,6 @@ class SSLSessionCache ~SSLSessionCache(); private: - SSLSessionBucket *session_bucket; + SSLSessionBucket *session_bucket = nullptr; size_t nbuckets; }; diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 181deb3e00d..456fa95dda6 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -866,20 +866,8 @@ UnixNetVConnection::reenable_re(VIO *vio) } } -UnixNetVConnection::UnixNetVConnection() - : closed(0), - inactivity_timeout_in(0), - active_timeout_in(0), - next_inactivity_timeout_at(0), - next_activity_timeout_at(0), - nh(nullptr), - id(0), - flags(0), - recursion(0), - submit_time(0), - oob_ptr(nullptr), - from_accept_thread(false), - accept_object(nullptr) +UnixNetVConnection::UnixNetVConnection() : flags(0) + { SET_HANDLER((NetVConnHandler)&UnixNetVConnection::startEvent); } diff --git a/iocore/net/UnixUDPNet.cc b/iocore/net/UnixUDPNet.cc index e6951c6a54d..eb7cf2b9bb2 100644 --- a/iocore/net/UnixUDPNet.cc +++ b/iocore/net/UnixUDPNet.cc @@ -297,13 +297,10 @@ UDPReadContinuation::UDPReadContinuation(Event *completionToken) : Continuation(nullptr), event(completionToken), readbuf(nullptr), - readlen(0), - fromaddrlen(nullptr), + fd(-1), - ifd(-1), - period(0), - elapsed_time(0), - timeout_interval(0) + ifd(-1) + { if (completionToken->continuation) { this->mutex = completionToken->continuation->mutex; diff --git a/iocore/utils/I_OneWayTunnel.h b/iocore/utils/I_OneWayTunnel.h index 448737c92dc..8a835641107 100644 --- a/iocore/utils/I_OneWayTunnel.h +++ b/iocore/utils/I_OneWayTunnel.h @@ -189,21 +189,21 @@ struct OneWayTunnel : public Continuation { bool last_connection(); - VIO *vioSource; - VIO *vioTarget; - Continuation *cont; - Transform_fn manipulate_fn; - int n_connections; - int lerrno; - - bool single_buffer; - bool close_source; - bool close_target; - bool tunnel_till_done; + VIO *vioSource = nullptr; + VIO *vioTarget = nullptr; + Continuation *cont = nullptr; + Transform_fn manipulate_fn = nullptr; + int n_connections = 0; + int lerrno = 0; + + bool single_buffer = false; + bool close_source = false; + bool close_target = false; + bool tunnel_till_done = false; /** Non-nullptr when this is one side of a two way tunnel. */ - OneWayTunnel *tunnel_peer; - bool free_vcs; + OneWayTunnel *tunnel_peer = nullptr; + bool free_vcs = true; // noncopyable OneWayTunnel(const OneWayTunnel &) = delete; diff --git a/iocore/utils/OneWayTunnel.cc b/iocore/utils/OneWayTunnel.cc index 07607267531..d4484f28d39 100644 --- a/iocore/utils/OneWayTunnel.cc +++ b/iocore/utils/OneWayTunnel.cc @@ -64,22 +64,7 @@ transfer_data(MIOBufferAccessor &in_buf, MIOBufferAccessor &out_buf) out_buf.writer()->fill(n); } -OneWayTunnel::OneWayTunnel() - : Continuation(nullptr), - vioSource(nullptr), - vioTarget(nullptr), - cont(nullptr), - manipulate_fn(nullptr), - n_connections(0), - lerrno(0), - single_buffer(false), - close_source(false), - close_target(false), - tunnel_till_done(false), - tunnel_peer(nullptr), - free_vcs(true) -{ -} +OneWayTunnel::OneWayTunnel() : Continuation(nullptr) {} OneWayTunnel * OneWayTunnel::OneWayTunnel_alloc() diff --git a/lib/records/I_RecHttp.h b/lib/records/I_RecHttp.h index c3db5a71000..fda517614b8 100644 --- a/lib/records/I_RecHttp.h +++ b/lib/records/I_RecHttp.h @@ -239,20 +239,20 @@ struct HttpProxyPort { TRANSPORT_PLUGIN /// < Protocol plugin connection }; - int m_fd; ///< Pre-opened file descriptor if present. - TransportType m_type; ///< Type of connection. - in_port_t m_port; ///< Port on which to listen. - uint8_t m_family; ///< IP address family. + int m_fd; ///< Pre-opened file descriptor if present. + TransportType m_type = TRANSPORT_DEFAULT; ///< Type of connection. + in_port_t m_port = 0; ///< Port on which to listen. + uint8_t m_family = AF_INET; ///< IP address family. /// True if proxy protocol is required on incoming requests. - bool m_proxy_protocol; + bool m_proxy_protocol = false; /// True if inbound connects (from client) are transparent. - bool m_inbound_transparent_p; + bool m_inbound_transparent_p = false; /// True if outbound connections (to origin servers) are transparent. - bool m_outbound_transparent_p; + bool m_outbound_transparent_p = false; // True if transparent pass-through is enabled on this port. - bool m_transparent_passthrough; + bool m_transparent_passthrough = false; /// True if MPTCP is enabled on this port. - bool m_mptcp; + bool m_mptcp = false; /// Local address for inbound connections (listen address). IpAddr m_inbound_ip; /// Local address for outbound connections (to origin server). diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index 0ad3378f62e..5f5ddb79aff 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -197,16 +197,8 @@ HttpProxyPort::Group GLOBAL_DATA; } // namespace HttpProxyPort::Group &HttpProxyPort::m_global = GLOBAL_DATA; -HttpProxyPort::HttpProxyPort() - : m_fd(ts::NO_FD), - m_type(TRANSPORT_DEFAULT), - m_port(0), - m_family(AF_INET), - m_proxy_protocol(false), - m_inbound_transparent_p(false), - m_outbound_transparent_p(false), - m_transparent_passthrough(false), - m_mptcp(false) +HttpProxyPort::HttpProxyPort() : m_fd(ts::NO_FD) + { memcpy(m_host_res_preference, host_res_default_preference_order, sizeof(m_host_res_preference)); } diff --git a/lib/tsconfig/Errata.h b/lib/tsconfig/Errata.h index 6d391ccbe77..eba5bacf4ac 100644 --- a/lib/tsconfig/Errata.h +++ b/lib/tsconfig/Errata.h @@ -517,7 +517,7 @@ struct Errata::Data : public IntrusivePtrCounter { void push(Message && msg); /// Log this when it is deleted. - mutable bool m_log_on_delete; + mutable bool m_log_on_delete = true; //! The message stack. Container m_items; @@ -867,7 +867,7 @@ inline Errata& Errata::doNotLog() { return *this; } -inline Errata::Data::Data() : m_log_on_delete(true) {} +inline Errata::Data::Data() {} inline size_t Errata::Data::size() const { return m_items.size(); } inline Errata::iterator::iterator() { } diff --git a/lib/tsconfig/TsBuilder.h b/lib/tsconfig/TsBuilder.h index f9da52d8efa..de765bee99e 100644 --- a/lib/tsconfig/TsBuilder.h +++ b/lib/tsconfig/TsBuilder.h @@ -36,9 +36,9 @@ class Builder { public: typedef Builder self; struct Handler { - self* _ptr; ///< Pointer to Builder instance. + self* _ptr = nullptr; ///< Pointer to Builder instance. /// Pointer to method to invoke for this event. - void (self::*_method)(Token const& token); + void (self::*_method)(Token const& token) = nullptr; /// Default constructor. Handler(); @@ -92,7 +92,7 @@ class Builder { self& init(); }; -inline Builder::Handler::Handler() : _ptr(nullptr), _method(nullptr) { } +inline Builder::Handler::Handler() { } inline Builder::Builder() { this->init(); } inline Builder::Builder(Configuration const& config) : _config(config) { this->init(); } diff --git a/lib/tsconfig/TsValue.h b/lib/tsconfig/TsValue.h index 8863eb8db6b..7d420dfe745 100644 --- a/lib/tsconfig/TsValue.h +++ b/lib/tsconfig/TsValue.h @@ -217,13 +217,13 @@ namespace detail { /// Get item type. ValueType getType() const; protected: - ValueType _type; ///< Type of value. + ValueType _type = VoidValue; ///< Type of value. ValueIndex _parent = 0; ///< Table index of parent value. ConstBuffer _text; ///< Text of value (if scalar). ConstBuffer _name; ///< Local name of value, if available. - size_t _local_index; ///< Index among siblings. - int _srcLine; ///< Source line. - int _srcColumn; ///< Source column. + size_t _local_index = 0; ///< Index among siblings. + int _srcLine = 0; ///< Source line. + int _srcColumn = 0; ///< Source column. /// Container for children of this item. typedef std::vector ChildGroup; @@ -645,8 +645,8 @@ namespace detail { inline ValueItem const& ValueTable::operator [] (ValueIndex idx) const { return const_cast(this)->operator [] (idx); } inline ValueTable& ValueTable::reset() { _ptr.reset(); return *this; } - inline ValueItem::ValueItem() : _type(VoidValue), _local_index(0), _srcLine(0), _srcColumn(0) {} - inline ValueItem::ValueItem(ValueType type) : _type(type), _local_index(0), _srcLine(0), _srcColumn(0) {} + inline ValueItem::ValueItem() {} + inline ValueItem::ValueItem(ValueType type) : _type(type) {} inline ValueType ValueItem::getType() const { return _type; } } diff --git a/plugins/authproxy/authproxy.cc b/plugins/authproxy/authproxy.cc index 4a5e5be6a0b..8ce13be763e 100644 --- a/plugins/authproxy/authproxy.cc +++ b/plugins/authproxy/authproxy.cc @@ -55,11 +55,11 @@ static TSCont AuthOsDnsContinuation; struct AuthOptions { std::string hostname; - int hostport; - AuthRequestTransform transform; - bool force; + int hostport = -1; + AuthRequestTransform transform = nullptr; + bool force = false; - AuthOptions() : hostport(-1), transform(nullptr), force(false) {} + AuthOptions() {} ~AuthOptions() {} }; @@ -137,27 +137,19 @@ static const StateTransition StateTableInit[] = {{TS_EVENT_HTTP_POST_REMAP, Stat {TS_EVENT_NONE, nullptr, nullptr}}; struct AuthRequestContext { - TSHttpTxn txn; // Original client transaction we are authorizing. - TSCont cont; // Continuation for this state machine. - TSVConn vconn; // Virtual connection to the auth proxy. - TSHttpParser hparser; // HTTP response header parser. - HttpHeader rheader; // HTTP response header. + TSHttpTxn txn = nullptr; // Original client transaction we are authorizing. + TSCont cont = nullptr; // Continuation for this state machine. + TSVConn vconn = nullptr; // Virtual connection to the auth proxy. + TSHttpParser hparser; // HTTP response header parser. + HttpHeader rheader; // HTTP response header. HttpIoBuffer iobuf; - const char *method; // Client request method (e.g. GET) - bool read_body; - - const StateTransition *state; - - AuthRequestContext() - : txn(nullptr), - cont(nullptr), - vconn(nullptr), - hparser(TSHttpParserCreate()), - rheader(), - iobuf(TS_IOBUFFER_SIZE_INDEX_4K), - method(nullptr), - read_body(true), - state(nullptr) + const char *method = nullptr; // Client request method (e.g. GET) + bool read_body = true; + + const StateTransition *state = nullptr; + + AuthRequestContext() : hparser(TSHttpParserCreate()), rheader(), iobuf(TS_IOBUFFER_SIZE_INDEX_4K) + { this->cont = TSContCreate(dispatch, TSMutexCreate()); TSContDataSet(this->cont, this); diff --git a/plugins/cache_promote/cache_promote.cc b/plugins/cache_promote/cache_promote.cc index d05127a548f..23e88c9ea20 100644 --- a/plugins/cache_promote/cache_promote.cc +++ b/plugins/cache_promote/cache_promote.cc @@ -56,7 +56,7 @@ static const struct option longopt[] = { class PromotionPolicy { public: - PromotionPolicy() : _sample(0.0) + PromotionPolicy() { // This doesn't have to be perfect, since this is just chance sampling. // coverity[dont_call] @@ -106,7 +106,7 @@ class PromotionPolicy virtual void usage() const = 0; private: - float _sample; + float _sample = 0.0; }; ////////////////////////////////////////////////////////////////////////////////////////////// @@ -195,7 +195,7 @@ static LRUEntry NULL_LRU_ENTRY; // Used to create an "empty" new LRUEntry class LRUPolicy : public PromotionPolicy { public: - LRUPolicy() : PromotionPolicy(), _buckets(1000), _hits(10), _lock(TSMutexCreate()), _list_size(0), _freelist_size(0) {} + LRUPolicy() : PromotionPolicy(), _lock(TSMutexCreate()) {} ~LRUPolicy() override { TSDebug(PLUGIN_NAME, "deleting LRUPolicy object"); @@ -333,14 +333,14 @@ class LRUPolicy : public PromotionPolicy } private: - unsigned _buckets; - unsigned _hits; + unsigned _buckets = 1000; + unsigned _hits = 10; // For the LRU. Note that we keep track of the List sizes, because some versions fo STL have broken // implementations of size(), making them obsessively slow on calling ::size(). TSMutex _lock; LRUMap _map; LRUList _list, _freelist; - size_t _list_size, _freelist_size; + size_t _list_size = 0, _freelist_size = 0; }; ////////////////////////////////////////////////////////////////////////////////////////////// @@ -349,7 +349,7 @@ class LRUPolicy : public PromotionPolicy class PromotionConfig { public: - PromotionConfig() : _policy(nullptr) {} + PromotionConfig() {} ~PromotionConfig() { delete _policy; } PromotionPolicy * getPolicy() const @@ -402,7 +402,7 @@ class PromotionConfig } private: - PromotionPolicy *_policy; + PromotionPolicy *_policy = nullptr; }; ////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/plugins/cachekey/configs.h b/plugins/cachekey/configs.h index 603ff434d2a..89a67af31fe 100644 --- a/plugins/cachekey/configs.h +++ b/plugins/cachekey/configs.h @@ -41,7 +41,7 @@ enum CacheKeyUriType { class ConfigElements { public: - ConfigElements() : _sort(false), _remove(false), _skip(false) {} + ConfigElements() {} virtual ~ConfigElements(); void setExclude(const char *arg); void setInclude(const char *arg); @@ -82,9 +82,9 @@ class ConfigElements MultiPattern _includePatterns; MultiPattern _excludePatterns; - bool _sort; - bool _remove; - bool _skip; + bool _sort = false; + bool _remove = false; + bool _skip = false; std::map _captures; }; diff --git a/plugins/cachekey/pattern.cc b/plugins/cachekey/pattern.cc index 27eb94ee24b..3672b692800 100644 --- a/plugins/cachekey/pattern.cc +++ b/plugins/cachekey/pattern.cc @@ -38,7 +38,7 @@ replaceString(String &str, const String &from, const String &to) } } -Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(""), _replace(false), _tokenCount(0) {} +Pattern::Pattern() : _pattern(""), _replacement("") {} /** * @brief Initializes PCRE pattern by providing the subject and replacement strings. diff --git a/plugins/cachekey/pattern.h b/plugins/cachekey/pattern.h index 47cd35cbb89..749d243f10b 100644 --- a/plugins/cachekey/pattern.h +++ b/plugins/cachekey/pattern.h @@ -57,16 +57,16 @@ class Pattern bool compile(); void pcreFree(); - pcre *_re; /**< @brief PCRE compiled info structure, computed during initialization */ - pcre_extra *_extra; /**< @brief PCRE study data block, computed during initialization */ + pcre *_re = nullptr; /**< @brief PCRE compiled info structure, computed during initialization */ + pcre_extra *_extra = nullptr; /**< @brief PCRE study data block, computed during initialization */ String _pattern; /**< @brief PCRE pattern string, containing PCRE patterns and capturing groups. */ String _replacement; /**< @brief PCRE replacement string, containing $0..$9 to be replaced with content of the capturing groups */ - bool _replace; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty replacement - string and no replacement needed case */ + bool _replace = false; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty + replacement string and no replacement needed case */ - int _tokenCount; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ + int _tokenCount = 0; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ int _tokens[TOKENCOUNT]; /**< @brief replacement index 0..9, since they can be used in the replacement string in any order */ int _tokenOffset[TOKENCOUNT]; /**< @brief replacement offset inside the replacement string */ }; diff --git a/plugins/conf_remap/conf_remap.cc b/plugins/conf_remap/conf_remap.cc index 4411f8962d3..736b36208ec 100644 --- a/plugins/conf_remap/conf_remap.cc +++ b/plugins/conf_remap/conf_remap.cc @@ -41,12 +41,12 @@ struct RemapConfigs { int _data_len; // Used when data is a string }; - RemapConfigs() : _current(0) { memset(_items, 0, sizeof(_items)); }; + RemapConfigs() { memset(_items, 0, sizeof(_items)); }; bool parse_file(const char *filename); bool parse_inline(const char *arg); Item _items[MAX_OVERRIDABLE_CONFIGS]; - int _current; + int _current = 0; }; // Helper functionfor the parser diff --git a/plugins/escalate/escalate.cc b/plugins/escalate/escalate.cc index f6e88629eb3..a6b1a0eae81 100644 --- a/plugins/escalate/escalate.cc +++ b/plugins/escalate/escalate.cc @@ -51,7 +51,7 @@ struct EscalationState { typedef std::map StatusMapType; - EscalationState() : use_pristine(false) + EscalationState() { cont = TSContCreate(EscalateResponse, nullptr); TSContDataSet(cont, this); @@ -60,7 +60,7 @@ struct EscalationState { ~EscalationState() { TSContDestroy(cont); } TSCont cont; StatusMapType status_map; - bool use_pristine; + bool use_pristine = false; }; // Little helper function, to update the Host portion of a URL, and stringify the result. diff --git a/plugins/esi/combo_handler.cc b/plugins/esi/combo_handler.cc index 4a5c336fb1d..d0c7159d000 100644 --- a/plugins/esi/combo_handler.cc +++ b/plugins/esi/combo_handler.cc @@ -83,12 +83,12 @@ static string COMBO_HANDLER_PATH{DEFAULT_COMBO_HANDLER_PATH}; using StringList = list; struct ClientRequest { - TSHttpStatus status; - const sockaddr *client_addr; + TSHttpStatus status = TS_HTTP_STATUS_OK; + const sockaddr *client_addr = nullptr; StringList file_urls; - bool gzip_accepted; + bool gzip_accepted = false; string defaultBucket; // default Bucket will be set to HOST header - ClientRequest() : status(TS_HTTP_STATUS_OK), client_addr(nullptr), gzip_accepted(false), defaultBucket("l"){}; + ClientRequest() : defaultBucket("l"){}; }; struct InterceptData { @@ -96,11 +96,11 @@ struct InterceptData { TSCont contp; struct IoHandle { - TSVIO vio; - TSIOBuffer buffer; - TSIOBufferReader reader; + TSVIO vio = nullptr; + TSIOBuffer buffer = nullptr; + TSIOBufferReader reader = nullptr; - IoHandle() : vio(nullptr), buffer(nullptr), reader(nullptr){}; + IoHandle(){}; ~IoHandle() { diff --git a/plugins/esi/fetcher/HttpDataFetcherImpl.h b/plugins/esi/fetcher/HttpDataFetcherImpl.h index d8a7fb743c8..1702813af5c 100644 --- a/plugins/esi/fetcher/HttpDataFetcherImpl.h +++ b/plugins/esi/fetcher/HttpDataFetcherImpl.h @@ -110,17 +110,15 @@ class HttpDataFetcherImpl : public HttpDataFetcher struct RequestData { std::string response; std::string raw_response; - const char *body; - int body_len; - TSHttpStatus resp_status; + const char *body = nullptr; + int body_len = 0; + TSHttpStatus resp_status = TS_HTTP_STATUS_NONE; CallbackObjectList callback_objects; - bool complete; - TSMBuffer bufp; - TSMLoc hdr_loc; + bool complete = false; + TSMBuffer bufp = nullptr; + TSMLoc hdr_loc = nullptr; - RequestData() : body(nullptr), body_len(0), resp_status(TS_HTTP_STATUS_NONE), complete(false), bufp(nullptr), hdr_loc(nullptr) - { - } + RequestData() {} }; typedef __gnu_cxx::hash_map UrlToContentMap; diff --git a/plugins/esi/serverIntercept.cc b/plugins/esi/serverIntercept.cc index a7c1c26c9de..0fff06f500c 100644 --- a/plugins/esi/serverIntercept.cc +++ b/plugins/esi/serverIntercept.cc @@ -43,10 +43,10 @@ struct SContData { TSCont contp; struct IoHandle { - TSVIO vio; - TSIOBuffer buffer; - TSIOBufferReader reader; - IoHandle() : vio(nullptr), buffer(nullptr), reader(nullptr){}; + TSVIO vio = nullptr; + TSIOBuffer buffer = nullptr; + TSIOBufferReader reader = nullptr; + IoHandle(){}; ~IoHandle() { if (reader) { diff --git a/plugins/esi/test/TestHttpDataFetcher.h b/plugins/esi/test/TestHttpDataFetcher.h index 9eecd09b688..cd2ee590527 100644 --- a/plugins/esi/test/TestHttpDataFetcher.h +++ b/plugins/esi/test/TestHttpDataFetcher.h @@ -30,7 +30,7 @@ class TestHttpDataFetcher : public HttpDataFetcher { public: - TestHttpDataFetcher() : _n_pending_requests(0), _return_data(true) {} + TestHttpDataFetcher() {} bool addFetchRequest(const std::string &url, FetchedDataProcessor *callback_obj = nullptr) { @@ -84,7 +84,7 @@ class TestHttpDataFetcher : public HttpDataFetcher }; private: - int _n_pending_requests; + int _n_pending_requests = 0; std::string _data; - bool _return_data; + bool _return_data = true; }; diff --git a/plugins/experimental/access_control/pattern.cc b/plugins/experimental/access_control/pattern.cc index 2c04b6a0267..396e222f4f9 100644 --- a/plugins/experimental/access_control/pattern.cc +++ b/plugins/experimental/access_control/pattern.cc @@ -38,7 +38,7 @@ replaceString(String &str, const String &from, const String &to) } } -Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(""), _replace(false), _tokenCount(0) {} +Pattern::Pattern() : _pattern(""), _replacement("") {} /** * @brief Initializes PCRE pattern by providing the subject and replacement strings. diff --git a/plugins/experimental/access_control/pattern.h b/plugins/experimental/access_control/pattern.h index 6f05487ee8f..46df0baa8f2 100644 --- a/plugins/experimental/access_control/pattern.h +++ b/plugins/experimental/access_control/pattern.h @@ -56,16 +56,16 @@ class Pattern bool compile(); void pcreFree(); - pcre *_re; /**< @brief PCRE compiled info structure, computed during initialization */ - pcre_extra *_extra; /**< @brief PCRE study data block, computed during initialization */ + pcre *_re = nullptr; /**< @brief PCRE compiled info structure, computed during initialization */ + pcre_extra *_extra = nullptr; /**< @brief PCRE study data block, computed during initialization */ String _pattern; /**< @brief PCRE pattern string, containing PCRE patterns and capturing groups. */ String _replacement; /**< @brief PCRE replacement string, containing $0..$9 to be replaced with content of the capturing groups */ - bool _replace; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty replacement - string and no replacement needed case */ + bool _replace = false; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty + replacement string and no replacement needed case */ - int _tokenCount; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ + int _tokenCount = 0; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ int _tokens[TOKENCOUNT]; /**< @brief replacement index 0..9, since they can be used in the replacement string in any order */ int _tokenOffset[TOKENCOUNT]; /**< @brief replacement offset inside the replacement string */ }; diff --git a/plugins/experimental/balancer/roundrobin.cc b/plugins/experimental/balancer/roundrobin.cc index 16f0ffd3419..6090ab97795 100644 --- a/plugins/experimental/balancer/roundrobin.cc +++ b/plugins/experimental/balancer/roundrobin.cc @@ -31,7 +31,7 @@ namespace { struct RoundRobinBalancer : public BalancerInstance { - RoundRobinBalancer() : targets(), next(0) {} + RoundRobinBalancer() : targets() {} void push_target(const BalancerTarget &target) override { @@ -45,7 +45,7 @@ struct RoundRobinBalancer : public BalancerInstance { } std::vector targets; - unsigned next; + unsigned next = 0; }; } // namespace diff --git a/plugins/experimental/cookie_remap/cookie_remap.cc b/plugins/experimental/cookie_remap/cookie_remap.cc index d488944ad06..f6ece83b60b 100644 --- a/plugins/experimental/cookie_remap/cookie_remap.cc +++ b/plugins/experimental/cookie_remap/cookie_remap.cc @@ -194,15 +194,11 @@ class subop subop() : cookie(""), operation(""), - op_type(UNKNOWN), - target(UNKNOWN_TARGET), + str_match(""), - regex(nullptr), - regex_extra(nullptr), - regex_ccount(0), - bucket(""), - how_many(0), - out_of(0) + + bucket("") + { TSDebug(MY_NAME, "subop constructor called"); } @@ -392,19 +388,19 @@ class subop private: std::string cookie; std::string operation; - enum operation_type op_type; - enum target_type target; + enum operation_type op_type = UNKNOWN; + enum target_type target = UNKNOWN_TARGET; std::string str_match; - pcre *regex; - pcre_extra *regex_extra; + pcre *regex = nullptr; + pcre_extra *regex_extra = nullptr; std::string regex_string; - int regex_ccount; + int regex_ccount = 0; std::string bucket; - unsigned int how_many; - unsigned int out_of; + unsigned int how_many = 0; + unsigned int out_of = 0; }; typedef std::vector SubOpQueue; diff --git a/plugins/experimental/geoip_acl/acl.h b/plugins/experimental/geoip_acl/acl.h index f66b9c240c6..b8e6b8cf10d 100644 --- a/plugins/experimental/geoip_acl/acl.h +++ b/plugins/experimental/geoip_acl/acl.h @@ -48,7 +48,7 @@ static const int NUM_ISO_CODES = 253; class Acl { public: - Acl() : _allow(true), _added_tokens(0) {} + Acl() {} virtual ~Acl() {} // These have to be implemented for each ACL type virtual void read_regex(const char *fn, int &tokens) = 0; @@ -81,8 +81,8 @@ class Acl protected: std::string _html; - bool _allow; - int _added_tokens; + bool _allow = true; + int _added_tokens = 0; // Class members static GeoDBHandle _geoip; @@ -136,7 +136,7 @@ class RegexAcl class CountryAcl : public Acl { public: - CountryAcl() : _regexes(nullptr) { memset(_iso_country_codes, 0, sizeof(_iso_country_codes)); } + CountryAcl() { memset(_iso_country_codes, 0, sizeof(_iso_country_codes)); } void read_regex(const char *fn, int &tokens) override; int process_args(int argc, char *argv[]) override; bool eval(TSRemapRequestInfo *rri, TSHttpTxn txnp) const override; @@ -144,5 +144,5 @@ class CountryAcl : public Acl private: bool _iso_country_codes[NUM_ISO_CODES]; - RegexAcl *_regexes; + RegexAcl *_regexes = nullptr; }; diff --git a/plugins/experimental/hipes/hipes.cc b/plugins/experimental/hipes/hipes.cc index dfcff4e8137..c90148a33f9 100644 --- a/plugins/experimental/hipes/hipes.cc +++ b/plugins/experimental/hipes/hipes.cc @@ -129,30 +129,24 @@ struct HIPESService { : url_param("url"), path(""), svc_server(""), - svc_port(80), - ssl(false), + hipes_server(HIPES_SERVER_NAME), - hipes_port(80), - default_redirect_flag(1), - x_hipes_header("X-HIPES-Redirect"), - active_timeout(-1), - no_activity_timeout(-1), - connect_timeout(-1), - dns_timeout(-1){}; + + x_hipes_header("X-HIPES-Redirect"){}; std::string url_param; std::string path; std::string svc_server; - int svc_port; - bool ssl; + int svc_port = 80; + bool ssl = false; std::string hipes_server; - int hipes_port; - unsigned int default_redirect_flag; + int hipes_port = 80; + unsigned int default_redirect_flag = 1; std::string x_hipes_header; - int active_timeout; - int no_activity_timeout; - int connect_timeout; - int dns_timeout; + int active_timeout = -1; + int no_activity_timeout = -1; + int connect_timeout = -1; + int dns_timeout = -1; }; /////////////////////////////////////////////////////////////////////////////// diff --git a/plugins/experimental/inliner/chunk-decoder.h b/plugins/experimental/inliner/chunk-decoder.h index 26318b90f7b..0dbf873834d 100644 --- a/plugins/experimental/inliner/chunk-decoder.h +++ b/plugins/experimental/inliner/chunk-decoder.h @@ -66,11 +66,11 @@ class ChunkDecoder }; }; - State::STATES state_; - int64_t size_; + State::STATES state_ = State::kSize; + int64_t size_ = 0; public: - ChunkDecoder() : state_(State::kSize), size_(0) {} + ChunkDecoder() {} void parseSizeCharacter(const char); int parseSize(const char *, const int64_t); int decode(const TSIOBufferReader &); diff --git a/plugins/experimental/inliner/fetcher.h b/plugins/experimental/inliner/fetcher.h index a1cb37005fb..93405f2868f 100644 --- a/plugins/experimental/inliner/fetcher.h +++ b/plugins/experimental/inliner/fetcher.h @@ -63,7 +63,7 @@ namespace ats { struct HttpParser { - bool parsed_; + bool parsed_ = false; TSHttpParser parser_; TSMBuffer buffer_; TSMLoc location_; @@ -77,7 +77,7 @@ struct HttpParser { destroyParser(); } - HttpParser() : parsed_(false), parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) + HttpParser() : parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) { TSHttpHdrTypeSet(buffer_, location_, TS_HTTP_TYPE_RESPONSE); } diff --git a/plugins/experimental/inliner/html-parser.h b/plugins/experimental/inliner/html-parser.h index 38f4b84fbea..c7be9711e46 100644 --- a/plugins/experimental/inliner/html-parser.h +++ b/plugins/experimental/inliner/html-parser.h @@ -101,10 +101,10 @@ namespace inliner }; struct AttributeParser { - Attribute::ATTRIBUTES state_; + Attribute::ATTRIBUTES state_ = Attribute::kPreName; Attributes attributes; - AttributeParser() : state_(Attribute::kPreName) {} + AttributeParser() {} void reset() { @@ -129,11 +129,11 @@ namespace inliner }; struct HtmlParser { - State::STATES state_; - Tag::TAGS tag_; + State::STATES state_ = State::kUndefined; + Tag::TAGS tag_ = Tag::kUndefined; AttributeParser attributeParser_; - HtmlParser() : state_(State::kUndefined), tag_(Tag::kUndefined) {} + HtmlParser() {} virtual ~HtmlParser() {} bool parseTag(const char); size_t parse(const char *, size_t, size_t o = 0); diff --git a/plugins/experimental/inliner/ts.h b/plugins/experimental/inliner/ts.h index 69cf2aa7ca3..9ab8bdd25fc 100644 --- a/plugins/experimental/inliner/ts.h +++ b/plugins/experimental/inliner/ts.h @@ -47,7 +47,7 @@ namespace io struct IO { TSIOBuffer buffer; TSIOBufferReader reader; - TSVIO vio; + TSVIO vio = nullptr; ~IO() { @@ -58,8 +58,8 @@ namespace io TSIOBufferDestroy(buffer); } - IO() : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) {} - IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) { assert(buffer != nullptr); } + IO() : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)) {} + IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)) { assert(buffer != nullptr); } static IO *read(TSVConn, TSCont, const int64_t); static IO * @@ -114,7 +114,7 @@ namespace io typedef std::weak_ptr WriteOperationWeakPointer; struct Lock { - const TSMutex mutex_; + const TSMutex mutex_ = nullptr; ~Lock() { @@ -131,7 +131,7 @@ namespace io } // noncopyable - Lock() : mutex_(nullptr) {} + Lock() {} Lock(const Lock &) = delete; Lock(Lock &&l) : mutex_(l.mutex_) { const_cast(l.mutex_) = nullptr; } diff --git a/plugins/experimental/magick/magick.cc b/plugins/experimental/magick/magick.cc index abcadf82e35..550c818bcd0 100644 --- a/plugins/experimental/magick/magick.cc +++ b/plugins/experimental/magick/magick.cc @@ -267,7 +267,7 @@ struct Image { struct Wand { MagickWand *wand; - void *blob; + void *blob = nullptr; ~Wand() { @@ -278,7 +278,7 @@ struct Wand { } } - Wand() : wand(NewMagickWand()), blob(nullptr) { assert(nullptr != wand); } + Wand() : wand(NewMagickWand()) { assert(nullptr != wand); } void clear() const @@ -512,7 +512,7 @@ struct ImageTransform : TransformationPlugin { struct GlobalHookPlugin : GlobalPlugin { magick::Core core_; - magick::EVPKey *key_; + magick::EVPKey *key_ = nullptr; ThreadPool threadPool_; ~GlobalHookPlugin() override @@ -523,7 +523,7 @@ struct GlobalHookPlugin : GlobalPlugin { } } - GlobalHookPlugin(const char *const f = nullptr) : key_(nullptr), threadPool_(2) + GlobalHookPlugin(const char *const f = nullptr) : threadPool_(2) { if (nullptr != f) { assert(0 < strlen(f)); diff --git a/plugins/experimental/memcache/tsmemcache.h b/plugins/experimental/memcache/tsmemcache.h index c43fc1f032e..fc6c81188a5 100644 --- a/plugins/experimental/memcache/tsmemcache.h +++ b/plugins/experimental/memcache/tsmemcache.h @@ -83,15 +83,15 @@ struct MCAccept : public Continuation { #ifndef HAVE_TLS ProxyAllocator *theMCThreadAllocator; #endif - int accept_port; + int accept_port = 0; int main_event(int event, void *netvc); MCAccept() : #ifndef HAVE_TLS - theMCThreadAllocator(NULL), + theMCThreadAllocator(NULL) #endif - accept_port(0) + { SET_HANDLER(&MCAccept::main_event); } diff --git a/plugins/experimental/mp4/mp4_common.h b/plugins/experimental/mp4/mp4_common.h index 126e4442813..780d238d472 100644 --- a/plugins/experimental/mp4/mp4_common.h +++ b/plugins/experimental/mp4/mp4_common.h @@ -32,7 +32,7 @@ class IOHandle { public: - IOHandle() : vio(nullptr), buffer(nullptr), reader(nullptr){}; + IOHandle(){}; ~IOHandle() { @@ -48,9 +48,9 @@ class IOHandle } public: - TSVIO vio; - TSIOBuffer buffer; - TSIOBufferReader reader; + TSVIO vio = nullptr; + TSIOBuffer buffer = nullptr; + TSIOBufferReader reader = nullptr; }; class Mp4TransformContext diff --git a/plugins/experimental/mp4/mp4_meta.h b/plugins/experimental/mp4/mp4_meta.h index 1e8d1b16626..8ee8f33a19e 100644 --- a/plugins/experimental/mp4/mp4_meta.h +++ b/plugins/experimental/mp4/mp4_meta.h @@ -310,7 +310,7 @@ typedef struct { class BufferHandle { public: - BufferHandle() : buffer(nullptr), reader(nullptr){}; + BufferHandle(){}; ~BufferHandle() { @@ -326,64 +326,41 @@ class BufferHandle } public: - TSIOBuffer buffer; - TSIOBufferReader reader; + TSIOBuffer buffer = nullptr; + TSIOBufferReader reader = nullptr; }; class Mp4Trak { public: - Mp4Trak() - : timescale(0), - duration(0), - time_to_sample_entries(0), - sample_to_chunk_entries(0), - sync_samples_entries(0), - composition_offset_entries(0), - sample_sizes_entries(0), - chunks(0), - start_sample(0), - start_chunk(0), - chunk_samples(0), - chunk_samples_size(0), - start_offset(0), - tkhd_size(0), - mdhd_size(0), - hdlr_size(0), - vmhd_size(0), - smhd_size(0), - dinf_size(0), - size(0) - { - memset(&stsc_chunk_entry, 0, sizeof(mp4_stsc_entry)); - } + Mp4Trak() { memset(&stsc_chunk_entry, 0, sizeof(mp4_stsc_entry)); } ~Mp4Trak() {} public: - uint32_t timescale; - int64_t duration; - - uint32_t time_to_sample_entries; // stsc - uint32_t sample_to_chunk_entries; // stsc - uint32_t sync_samples_entries; // stss - uint32_t composition_offset_entries; // ctts - uint32_t sample_sizes_entries; // stsz - uint32_t chunks; // stco, co64 - - uint32_t start_sample; - uint32_t start_chunk; - uint32_t chunk_samples; - uint64_t chunk_samples_size; - off_t start_offset; - - size_t tkhd_size; - size_t mdhd_size; - size_t hdlr_size; - size_t vmhd_size; - size_t smhd_size; - size_t dinf_size; - size_t size; + uint32_t timescale = 0; + int64_t duration = 0; + + uint32_t time_to_sample_entries = 0; // stsc + uint32_t sample_to_chunk_entries = 0; // stsc + uint32_t sync_samples_entries = 0; // stss + uint32_t composition_offset_entries = 0; // ctts + uint32_t sample_sizes_entries = 0; // stsz + uint32_t chunks = 0; // stco, co64 + + uint32_t start_sample = 0; + uint32_t start_chunk = 0; + uint32_t chunk_samples = 0; + uint64_t chunk_samples_size = 0; + off_t start_offset = 0; + + size_t tkhd_size = 0; + size_t mdhd_size = 0; + size_t hdlr_size = 0; + size_t vmhd_size = 0; + size_t smhd_size = 0; + size_t dinf_size = 0; + size_t size = 0; BufferHandle atoms[MP4_LAST_ATOM + 1]; @@ -394,22 +371,7 @@ class Mp4Meta { public: Mp4Meta() - : start(0), - cl(0), - content_length(0), - meta_atom_size(0), - meta_avail(0), - wait_next(0), - need_size(0), - rs(0), - rate(0), - ftyp_size(0), - moov_size(0), - start_pos(0), - timescale(0), - trak_num(0), - passed(0), - meta_complete(false) + { memset(trak_vec, 0, sizeof(trak_vec)); meta_buffer = TSIOBufferCreate(); @@ -494,17 +456,17 @@ class Mp4Meta void mp4_update_mdhd_duration(Mp4Trak *trak); public: - int64_t start; // requested start time, measured in milliseconds. - int64_t cl; // the total size of the mp4 file - int64_t content_length; // the size of the new mp4 file - int64_t meta_atom_size; + int64_t start = 0; // requested start time, measured in milliseconds. + int64_t cl = 0; // the total size of the mp4 file + int64_t content_length = 0; // the size of the new mp4 file + int64_t meta_atom_size = 0; TSIOBuffer meta_buffer; // meta data to be parsed TSIOBufferReader meta_reader; - int64_t meta_avail; - int64_t wait_next; - int64_t need_size; + int64_t meta_avail = 0; + int64_t wait_next = 0; + int64_t need_size = 0; BufferHandle meta_atom; BufferHandle ftyp_atom; @@ -516,16 +478,16 @@ class Mp4Meta Mp4Trak *trak_vec[MP4_MAX_TRAK_NUM]; - double rs; - double rate; + double rs = 0; + double rate = 0; - int64_t ftyp_size; - int64_t moov_size; - int64_t start_pos; // start position of the new mp4 file - uint32_t timescale; - uint32_t trak_num; - int64_t passed; + int64_t ftyp_size = 0; + int64_t moov_size = 0; + int64_t start_pos = 0; // start position of the new mp4 file + uint32_t timescale = 0; + uint32_t trak_num = 0; + int64_t passed = 0; u_char mdat_atom_header[16]; - bool meta_complete; + bool meta_complete = false; }; diff --git a/plugins/experimental/multiplexer/fetcher.h b/plugins/experimental/multiplexer/fetcher.h index c5abe54d71f..61bc5950206 100644 --- a/plugins/experimental/multiplexer/fetcher.h +++ b/plugins/experimental/multiplexer/fetcher.h @@ -42,7 +42,7 @@ namespace ats { struct HttpParser { - bool parsed_; + bool parsed_ = false; TSHttpParser parser_; TSMBuffer buffer_; TSMLoc location_; @@ -56,7 +56,7 @@ struct HttpParser { destroyParser(); } - HttpParser() : parsed_(false), parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) + HttpParser() : parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) { TSHttpHdrTypeSet(buffer_, location_, TS_HTTP_TYPE_RESPONSE); } diff --git a/plugins/experimental/multiplexer/ts.h b/plugins/experimental/multiplexer/ts.h index 327c6b502f5..5308652e3ba 100644 --- a/plugins/experimental/multiplexer/ts.h +++ b/plugins/experimental/multiplexer/ts.h @@ -38,7 +38,7 @@ namespace io struct IO { TSIOBuffer buffer; TSIOBufferReader reader; - TSVIO vio; + TSVIO vio = nullptr; ~IO() { @@ -52,8 +52,8 @@ namespace io TSIOBufferDestroy(buffer); } - IO() : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) {} - IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) { assert(buffer != nullptr); } + IO() : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)) {} + IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)) { assert(buffer != nullptr); } static IO *read(TSVConn, TSCont, const int64_t); static IO * diff --git a/plugins/experimental/prefetch/configs.h b/plugins/experimental/prefetch/configs.h index 2552c3660af..d1d61a8d6da 100644 --- a/plugins/experimental/prefetch/configs.h +++ b/plugins/experimental/prefetch/configs.h @@ -39,11 +39,8 @@ class PrefetchConfig _nextHeader("X-AppleCDN-Prefetch-Next"), _replaceHost(), _namespace("default"), - _metricsPrefix("prefetch.stats"), - _fetchCount(1), - _fetchMax(0), - _front(false), - _exactMatch(false) + _metricsPrefix("prefetch.stats") + { } @@ -194,9 +191,9 @@ class PrefetchConfig std::string _namespace; std::string _metricsPrefix; std::string _logName; - unsigned _fetchCount; - unsigned _fetchMax; - bool _front; - bool _exactMatch; + unsigned _fetchCount = 1; + unsigned _fetchMax = 0; + bool _front = false; + bool _exactMatch = false; MultiPattern _nextPaths; }; diff --git a/plugins/experimental/prefetch/fetch.cc b/plugins/experimental/prefetch/fetch.cc index 49445c3d713..b34d062a453 100644 --- a/plugins/experimental/prefetch/fetch.cc +++ b/plugins/experimental/prefetch/fetch.cc @@ -114,7 +114,7 @@ createStat(const String &prefix, const String &space, const char *module, const return true; } -BgFetchState::BgFetchState() : _policy(nullptr), _unique(nullptr), _concurrentFetches(0), _concurrentFetchesMax(0), _log(nullptr) +BgFetchState::BgFetchState() { _policyLock = TSMutexCreate(); if (nullptr == _policyLock) { diff --git a/plugins/experimental/prefetch/fetch.h b/plugins/experimental/prefetch/fetch.h index 4d73a7b097a..206771bd0b0 100644 --- a/plugins/experimental/prefetch/fetch.h +++ b/plugins/experimental/prefetch/fetch.h @@ -89,14 +89,14 @@ class BgFetchState void operator=(BgFetchState const &); /* never implement */ /* Fetch policy related */ - FetchPolicy *_policy; /* fetch policy */ - TSMutex _policyLock; /* protects the policy object only */ + FetchPolicy *_policy = nullptr; /* fetch policy */ + TSMutex _policyLock; /* protects the policy object only */ /* Mechanisms to avoid concurrent fetches and applying limits */ - FetchPolicy *_unique; /* make sure we never download same object multiple times at the same time */ - TSMutex _lock; /* protects the deduplication object only */ - size_t _concurrentFetches; - size_t _concurrentFetchesMax; + FetchPolicy *_unique = nullptr; /* make sure we never download same object multiple times at the same time */ + TSMutex _lock; /* protects the deduplication object only */ + size_t _concurrentFetches = 0; + size_t _concurrentFetchesMax = 0; PrefetchMetricInfo _metrics[FETCHES_MAX_METRICS] = { {FETCH_ACTIVE, TS_RECORDDATATYPE_INT, -1}, {FETCH_COMPLETED, TS_RECORDDATATYPE_COUNTER, -1}, {FETCH_ERRORS, TS_RECORDDATATYPE_COUNTER, -1}, {FETCH_TIMEOOUTS, TS_RECORDDATATYPE_COUNTER, -1}, @@ -108,7 +108,7 @@ class BgFetchState {FETCH_POLICY_MAXSIZE, TS_RECORDDATATYPE_INT, -1}}; /* plugin specific fetch logging */ - TSTextLogObject _log; + TSTextLogObject _log = nullptr; }; /** diff --git a/plugins/experimental/prefetch/fetch_policy_lru.h b/plugins/experimental/prefetch/fetch_policy_lru.h index 391023497c9..8f14d889e61 100644 --- a/plugins/experimental/prefetch/fetch_policy_lru.h +++ b/plugins/experimental/prefetch/fetch_policy_lru.h @@ -86,7 +86,7 @@ class FetchPolicyLru : public FetchPolicy { public: /* Default size values are also considered minimum. TODO: find out if this works OK. */ - FetchPolicyLru() : _maxSize(10), _size(0){}; + FetchPolicyLru(){}; ~FetchPolicyLru() override{}; /* Fetch policy interface methods */ @@ -100,6 +100,6 @@ class FetchPolicyLru : public FetchPolicy protected: LruMap _map; LruList _list; - LruList::size_type _maxSize; - LruList::size_type _size; + LruList::size_type _maxSize = 10; + LruList::size_type _size = 0; }; diff --git a/plugins/experimental/prefetch/pattern.cc b/plugins/experimental/prefetch/pattern.cc index 25541a3195d..3e408f70cbc 100644 --- a/plugins/experimental/prefetch/pattern.cc +++ b/plugins/experimental/prefetch/pattern.cc @@ -38,7 +38,7 @@ replaceString(String &str, const String &from, const String &to) } } -Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(""), _tokenCount(0) {} +Pattern::Pattern() : _pattern(""), _replacement("") {} /** * @brief Initializes PCRE pattern by providing the subject and replacement strings. diff --git a/plugins/experimental/prefetch/pattern.h b/plugins/experimental/prefetch/pattern.h index b9270c8c755..1045812b110 100644 --- a/plugins/experimental/prefetch/pattern.h +++ b/plugins/experimental/prefetch/pattern.h @@ -56,13 +56,13 @@ class Pattern bool failed(const String &subject) const; void pcreFree(); - pcre *_re; /**< @brief PCRE compiled info structure, computed during initialization */ - pcre_extra *_extra; /**< @brief PCRE study data block, computed during initialization */ + pcre *_re = nullptr; /**< @brief PCRE compiled info structure, computed during initialization */ + pcre_extra *_extra = nullptr; /**< @brief PCRE study data block, computed during initialization */ String _pattern; /**< @brief PCRE pattern string, containing PCRE patterns and capturing groups. */ String _replacement; /**< @brief PCRE replacement string, containing $0..$9 to be replaced with content of the capturing groups */ - int _tokenCount; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ + int _tokenCount = 0; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ int _tokens[TOKENCOUNT]; /**< @brief replacement index 0..9, since they can be used in the replacement string in any order */ int _tokenOffset[TOKENCOUNT]; /**< @brief replacement offset inside the replacement string */ }; diff --git a/plugins/experimental/prefetch/plugin.cc b/plugins/experimental/prefetch/plugin.cc index f111220050b..fede772f6d8 100644 --- a/plugins/experimental/prefetch/plugin.cc +++ b/plugins/experimental/prefetch/plugin.cc @@ -121,7 +121,7 @@ TSRemapInit(TSRemapInterface *apiInfo, char *errBuf, int erroBufSize) */ struct PrefetchInstance { - PrefetchInstance() : _state(nullptr){}; + PrefetchInstance(){}; private: PrefetchInstance(PrefetchInstance const &); @@ -129,7 +129,7 @@ struct PrefetchInstance { public: PrefetchConfig _config; - BgFetchState *_state; + BgFetchState *_state = nullptr; }; /** diff --git a/plugins/experimental/slice/ContentRange.h b/plugins/experimental/slice/ContentRange.h index 0ca05588dc9..87de6316c87 100644 --- a/plugins/experimental/slice/ContentRange.h +++ b/plugins/experimental/slice/ContentRange.h @@ -25,11 +25,11 @@ Range is converted from closed range into a half open range for. */ struct ContentRange { - int64_t m_beg; - int64_t m_end; // half open - int64_t m_length; // full content length + int64_t m_beg = -1; + int64_t m_end = -1; // half open + int64_t m_length = -1; // full content length - ContentRange() : m_beg(-1), m_end(-1), m_length(-1) {} + ContentRange() {} explicit ContentRange(int64_t const begin, int64_t const end, int64_t const len) : m_beg(begin), m_end(end), m_length(len) {} bool isValid() const diff --git a/plugins/experimental/slice/Range.h b/plugins/experimental/slice/Range.h index 373304db57d..0fb3145ddae 100644 --- a/plugins/experimental/slice/Range.h +++ b/plugins/experimental/slice/Range.h @@ -31,10 +31,10 @@ struct Range { public: static int64_t constexpr maxval = (std::numeric_limits::max() >> 2); - int64_t m_beg; - int64_t m_end; // half open + int64_t m_beg = -1; + int64_t m_end = -1; // half open - Range() : m_beg(-1), m_end(-1) {} + Range() {} explicit Range(int64_t const begin, int64_t const end) : m_beg(begin), m_end(end) {} bool isValid() const; diff --git a/plugins/experimental/ssl_session_reuse/src/publish.cc b/plugins/experimental/ssl_session_reuse/src/publish.cc index a74664c7369..794260611e3 100644 --- a/plugins/experimental/ssl_session_reuse/src/publish.cc +++ b/plugins/experimental/ssl_session_reuse/src/publish.cc @@ -49,14 +49,14 @@ RedisPublisher::start_worker_thread(void *arg) RedisPublisher::RedisPublisher(const std::string &conf) : m_redisEndpointsStr(cDefaultRedisEndpoint), - m_endpointIndex(0), + m_numWorkers(cPubNumWorkerThreads), m_redisConnectTimeout(cDefaultRedisConnectTimeout), m_redisConnectTries(cDefaultRedisConnectTries), m_redisPublishTries(cDefaultRedisPublishTries), m_redisRetryDelay(cDefaultRedisRetryDelay), - m_maxQueuedMessages(cDefaultMaxQueuedMessages), - err(false) + m_maxQueuedMessages(cDefaultMaxQueuedMessages) + { if (Config::getSingleton().loadConfig(conf)) { Config::getSingleton().getValue("pubconfig", "PubNumWorkers", m_numWorkers); diff --git a/plugins/experimental/ssl_session_reuse/src/publisher.h b/plugins/experimental/ssl_session_reuse/src/publisher.h index 2d1a1a190d0..82fec012efd 100644 --- a/plugins/experimental/ssl_session_reuse/src/publisher.h +++ b/plugins/experimental/ssl_session_reuse/src/publisher.h @@ -55,7 +55,7 @@ class RedisPublisher std::vector m_redisEndpoints; std::string m_redisEndpointsStr; - int m_endpointIndex; + int m_endpointIndex = 0; std::mutex m_endpointIndexMutex; std::vector pools; @@ -68,7 +68,7 @@ class RedisPublisher unsigned int m_maxQueuedMessages; unsigned int m_poolRedisConnectTimeout; // milliseconds - bool err; + bool err = false; void runWorker(); diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc index 0d87f5a9d0f..56496aa6367 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc @@ -83,7 +83,7 @@ init_ssl_params(const std::string &conf) return 0; } -ssl_session_param::ssl_session_param() : pub(nullptr) {} +ssl_session_param::ssl_session_param() {} ssl_session_param::~ssl_session_param() { diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_utils.h b/plugins/experimental/ssl_session_reuse/src/ssl_utils.h index bbfbf0c88b6..262728cb758 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_utils.h +++ b/plugins/experimental/ssl_session_reuse/src/ssl_utils.h @@ -40,7 +40,7 @@ struct ssl_session_param { int stek_master; // bool - Am I the STEK setter/rotator for POD? ssl_ticket_key_t ticket_keys[2]; // current and past STEK std::string redis_auth_key_file; - RedisPublisher *pub; + RedisPublisher *pub = nullptr; RedisSubscriber *sub; ssl_session_param(); diff --git a/plugins/experimental/sslheaders/sslheaders.cc b/plugins/experimental/sslheaders/sslheaders.cc index 657a9cd34ed..3fce092de84 100644 --- a/plugins/experimental/sslheaders/sslheaders.cc +++ b/plugins/experimental/sslheaders/sslheaders.cc @@ -329,8 +329,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri */) return TSREMAP_NO_REMAP; } -SslHdrInstance::SslHdrInstance() - : expansions(), attach(SSL_HEADERS_ATTACH_SERVER), cont(TSContCreate(SslHdrExpandRequestHook, nullptr)) +SslHdrInstance::SslHdrInstance() : expansions(), cont(TSContCreate(SslHdrExpandRequestHook, nullptr)) { TSContDataSet(cont, this); } diff --git a/plugins/experimental/sslheaders/sslheaders.h b/plugins/experimental/sslheaders/sslheaders.h index ff72100ec03..7ccb00f2912 100644 --- a/plugins/experimental/sslheaders/sslheaders.h +++ b/plugins/experimental/sslheaders/sslheaders.h @@ -62,10 +62,10 @@ enum ExpansionField { }; struct SslHdrExpansion { - SslHdrExpansion() : name(), scope(SSL_HEADERS_SCOPE_NONE), field(SSL_HEADERS_FIELD_NONE) {} + SslHdrExpansion() : name() {} std::string name; // HTTP header name - ExpansionScope scope; - ExpansionField field; + ExpansionScope scope = SSL_HEADERS_SCOPE_NONE; + ExpansionField field = SSL_HEADERS_FIELD_NONE; // noncopyable but moveable SslHdrExpansion(const SslHdrExpansion &) = delete; @@ -81,7 +81,7 @@ struct SslHdrInstance { ~SslHdrInstance(); expansion_list expansions; - AttachOptions attach; + AttachOptions attach = SSL_HEADERS_ATTACH_SERVER; TSCont cont; void register_hooks(); diff --git a/plugins/experimental/stream_editor/stream_editor.cc b/plugins/experimental/stream_editor/stream_editor.cc index ee079673f8d..a19552e05b4 100644 --- a/plugins/experimental/stream_editor/stream_editor.cc +++ b/plugins/experimental/stream_editor/stream_editor.cc @@ -551,17 +551,17 @@ using ruleset_t = std::vector; using rule_p = ruleset_t::const_iterator; typedef struct contdata_t { - TSCont cont; - TSIOBuffer out_buf; - TSIOBufferReader out_rd; - TSVIO out_vio; + TSCont cont = nullptr; + TSIOBuffer out_buf = nullptr; + TSIOBufferReader out_rd = nullptr; + TSVIO out_vio = nullptr; ruleset_t rules; std::string contbuf; - size_t contbuf_sz; - int64_t bytes_in; - int64_t bytes_out; + size_t contbuf_sz = 0; + int64_t bytes_in = 0; + int64_t bytes_out = 0; /* Use new/delete so destructor does cleanup for us */ - contdata_t() : cont(nullptr), out_buf(nullptr), out_rd(nullptr), out_vio(nullptr), contbuf_sz(0), bytes_in(0), bytes_out(0) {} + contdata_t() {} ~contdata_t() { if (out_rd) { diff --git a/plugins/generator/generator.cc b/plugins/generator/generator.cc index 8eee22f5cda..4dc7f4de045 100644 --- a/plugins/generator/generator.cc +++ b/plugins/generator/generator.cc @@ -109,11 +109,11 @@ lengthof(const char (&)[N]) // for each TSVConn; one to push data into the TSVConn and one to pull // data out. struct IOChannel { - TSVIO vio; + TSVIO vio = nullptr; TSIOBuffer iobuf; TSIOBufferReader reader; - IOChannel() : vio(nullptr), iobuf(TSIOBufferSizedCreate(TS_IOBUFFER_SIZE_INDEX_32K)), reader(TSIOBufferReaderAlloc(iobuf)) {} + IOChannel() : iobuf(TSIOBufferSizedCreate(TS_IOBUFFER_SIZE_INDEX_32K)), reader(TSIOBufferReaderAlloc(iobuf)) {} ~IOChannel() { if (this->reader) { @@ -163,10 +163,10 @@ struct GeneratorHttpHeader { }; struct GeneratorRequest { - off_t nbytes; // Number of bytes to generate. - unsigned flags; - unsigned delay; // Milliseconds to delay before sending a response. - unsigned maxage; // Max age for cache responses. + off_t nbytes = 0; // Number of bytes to generate. + unsigned flags = 0; + unsigned delay = 0; // Milliseconds to delay before sending a response. + unsigned maxage; // Max age for cache responses. IOChannel readio; IOChannel writeio; GeneratorHttpHeader rqheader; @@ -176,7 +176,7 @@ struct GeneratorRequest { ISHEAD = 0x0002, }; - GeneratorRequest() : nbytes(0), flags(0), delay(0), maxage(60 * 60 * 24) {} + GeneratorRequest() : maxage(60 * 60 * 24) {} ~GeneratorRequest() {} }; diff --git a/plugins/regex_remap/regex_remap.cc b/plugins/regex_remap/regex_remap.cc index 484a0d9ca68..28b5223c8a4 100644 --- a/plugins/regex_remap/regex_remap.cc +++ b/plugins/regex_remap/regex_remap.cc @@ -75,21 +75,7 @@ enum ExtraSubstitutions { // length calculations (we need all of them). // struct UrlComponents { - UrlComponents() - : scheme(nullptr), - host(nullptr), - path(nullptr), - query(nullptr), - matrix(nullptr), - port(0), - scheme_len(0), - host_len(0), - path_len(0), - query_len(0), - matrix_len(0), - url_len(0) - { - } + UrlComponents() {} void populate(TSRemapRequestInfo *rri) @@ -104,20 +90,20 @@ struct UrlComponents { url_len = scheme_len + host_len + path_len + query_len + matrix_len + 32; } - const char *scheme; - const char *host; - const char *path; - const char *query; - const char *matrix; - int port; + const char *scheme = nullptr; + const char *host = nullptr; + const char *path = nullptr; + const char *query = nullptr; + const char *matrix = nullptr; + int port = 0; - int scheme_len; - int host_len; - int path_len; - int query_len; - int matrix_len; + int scheme_len = 0; + int host_len = 0; + int path_len = 0; + int query_len = 0; + int matrix_len = 0; - int url_len; // Full length, of all components + int url_len = 0; // Full length, of all components }; /////////////////////////////////////////////////////////////////////////////// @@ -643,29 +629,17 @@ RemapRegex::substitute(char dest[], const char *src, const int ovector[], const // Hold one remap instance struct RemapInstance { - RemapInstance() - : first(nullptr), - last(nullptr), - profile(false), - method(false), - query_string(true), - matrix_params(false), - host(false), - hits(0), - misses(0), - filename("unknown") - { - } - - RemapRegex *first; - RemapRegex *last; - bool profile; - bool method; - bool query_string; - bool matrix_params; - bool host; - int hits; - int misses; + RemapInstance() : filename("unknown") {} + + RemapRegex *first = nullptr; + RemapRegex *last = nullptr; + bool profile = false; + bool method = false; + bool query_string = true; + bool matrix_params = false; + bool host = false; + int hits = 0; + int misses = 0; std::string filename; }; diff --git a/plugins/s3_auth/aws_auth_v4_wrap.h b/plugins/s3_auth/aws_auth_v4_wrap.h index a4351478c61..61480a64dd3 100644 --- a/plugins/s3_auth/aws_auth_v4_wrap.h +++ b/plugins/s3_auth/aws_auth_v4_wrap.h @@ -28,7 +28,7 @@ class HeaderIterator { public: - HeaderIterator() : _bufp(nullptr), _hdrs(TS_NULL_MLOC), _field(TS_NULL_MLOC) {} + HeaderIterator() : _hdrs(TS_NULL_MLOC), _field(TS_NULL_MLOC) {} HeaderIterator(TSMBuffer bufp, TSMLoc hdrs, TSMLoc field) : _bufp(bufp), _hdrs(hdrs), _field(field) {} HeaderIterator(const HeaderIterator &it) { @@ -81,7 +81,7 @@ class HeaderIterator { return TSMimeHdrFieldValueStringGet(_bufp, _hdrs, _field, -1, len); } - TSMBuffer _bufp; + TSMBuffer _bufp = nullptr; TSMLoc _hdrs; TSMLoc _field; }; diff --git a/proxy/CacheControl.h b/proxy/CacheControl.h index 1900a9594ec..3c088b6cabf 100644 --- a/proxy/CacheControl.h +++ b/proxy/CacheControl.h @@ -72,11 +72,11 @@ class CacheControlResult int revalidate_after; int pin_in_cache_for; int ttl_in_cache; - bool never_cache; - bool ignore_client_no_cache; - bool ignore_server_no_cache; - bool ignore_client_cc_max_age; - int cache_responses_to_cookies; ///< Override for caching cookied responses. + bool never_cache = false; + bool ignore_client_no_cache = false; + bool ignore_server_no_cache = false; + bool ignore_client_cc_max_age = true; + int cache_responses_to_cookies = -1; ///< Override for caching cookied responses. // Data for internal use only // @@ -86,29 +86,17 @@ class CacheControlResult // be overriden by something that appeared // earlier in the the config file // - int reval_line; - int never_line; - int pin_line; - int ttl_line; - int ignore_client_line; - int ignore_server_line; + int reval_line = -1; + int never_line = -1; + int pin_line = -1; + int ttl_line = -1; + int ignore_client_line = -1; + int ignore_server_line = -1; }; inline CacheControlResult::CacheControlResult() - : revalidate_after(CC_UNSET_TIME), - pin_in_cache_for(CC_UNSET_TIME), - ttl_in_cache(CC_UNSET_TIME), - never_cache(false), - ignore_client_no_cache(false), - ignore_server_no_cache(false), - ignore_client_cc_max_age(true), - cache_responses_to_cookies(-1), // do not change value - reval_line(-1), - never_line(-1), - pin_line(-1), - ttl_line(-1), - ignore_client_line(-1), - ignore_server_line(-1) + : revalidate_after(CC_UNSET_TIME), pin_in_cache_for(CC_UNSET_TIME), ttl_in_cache(CC_UNSET_TIME) + { } @@ -116,17 +104,15 @@ class CacheControlRecord : public ControlBase { public: CacheControlRecord(); - CacheControlType directive; - int time_arg; - int cache_responses_to_cookies; + CacheControlType directive = CC_INVALID; + int time_arg = 0; + int cache_responses_to_cookies = -1; Result Init(matcher_line *line_info); inkcoreapi void UpdateMatch(CacheControlResult *result, RequestData *rdata); void Print(); }; -inline CacheControlRecord::CacheControlRecord() : ControlBase(), directive(CC_INVALID), time_arg(0), cache_responses_to_cookies(-1) -{ -} +inline CacheControlRecord::CacheControlRecord() : ControlBase() {} // // API to outside world diff --git a/proxy/ControlBase.h b/proxy/ControlBase.h index 844276f8285..76ffb25861d 100644 --- a/proxy/ControlBase.h +++ b/proxy/ControlBase.h @@ -77,7 +77,7 @@ class ControlBase bool CheckModifiers(HttpRequestData *request_data); bool CheckForMatch(HttpRequestData *request_data, int last_number); void Print(); - int line_num; + int line_num = 0; Modifier *findModOfType(Modifier::Type t) const; protected: @@ -97,7 +97,7 @@ class ControlBase void clear(); }; -inline ControlBase::ControlBase() : line_num(0) {} +inline ControlBase::ControlBase() {} inline bool ControlBase::CheckForMatch(HttpRequestData *request_data, int last_number) diff --git a/proxy/ControlMatcher.h b/proxy/ControlMatcher.h index 3c7c725a3a5..50cb2916e6e 100644 --- a/proxy/ControlMatcher.h +++ b/proxy/ControlMatcher.h @@ -138,31 +138,23 @@ class HttpRequestData : public RequestData inkcoreapi sockaddr const *get_client_ip() override; HttpRequestData() - : hdr(nullptr), - hostname_str(nullptr), - api_info(nullptr), - xact_start(0), - incoming_port(0), - tag(nullptr), - internal_txn(false), - cache_info_lookup_url(nullptr), - cache_info_parent_selection_url(nullptr) + { ink_zero(src_ip); ink_zero(dest_ip); } - HTTPHdr *hdr; - char *hostname_str; - HttpApiInfo *api_info; - time_t xact_start; + HTTPHdr *hdr = nullptr; + char *hostname_str = nullptr; + HttpApiInfo *api_info = nullptr; + time_t xact_start = 0; IpEndpoint src_ip; IpEndpoint dest_ip; - uint16_t incoming_port; - char *tag; - bool internal_txn; - URL **cache_info_lookup_url; - URL **cache_info_parent_selection_url; + uint16_t incoming_port = 0; + char *tag = nullptr; + bool internal_txn = false; + URL **cache_info_lookup_url = nullptr; + URL **cache_info_parent_selection_url = nullptr; }; // Mixin class for shared info across all templates. This just wraps the diff --git a/proxy/InkAPIInternal.h b/proxy/InkAPIInternal.h index ccfd08bb559..ee516e238df 100644 --- a/proxy/InkAPIInternal.h +++ b/proxy/InkAPIInternal.h @@ -189,12 +189,12 @@ class FeatureAPIHooks bool has_hooks_for(ID id) const; private: - bool hooks_p; ///< Flag for (not) empty container. + bool hooks_p = false; ///< Flag for (not) empty container. /// The array of hooks lists. APIHooks m_hooks[N]; }; -template FeatureAPIHooks::FeatureAPIHooks() : hooks_p(false) {} +template FeatureAPIHooks::FeatureAPIHooks() {} template FeatureAPIHooks::~FeatureAPIHooks() { diff --git a/proxy/Plugin.cc b/proxy/Plugin.cc index e6b43e58a46..abecdc8ed69 100644 --- a/proxy/Plugin.cc +++ b/proxy/Plugin.cc @@ -51,10 +51,7 @@ using init_func_t = void (*)(int, char **); DLL plugin_reg_list; PluginRegInfo *plugin_reg_current = nullptr; -PluginRegInfo::PluginRegInfo() - : plugin_registered(false), plugin_path(nullptr), plugin_name(nullptr), vendor_name(nullptr), support_email(nullptr), dlh(nullptr) -{ -} +PluginRegInfo::PluginRegInfo() {} PluginRegInfo::~PluginRegInfo() { diff --git a/proxy/Plugin.h b/proxy/Plugin.h index 349a1482fa7..09c3de69fca 100644 --- a/proxy/Plugin.h +++ b/proxy/Plugin.h @@ -29,14 +29,14 @@ struct PluginRegInfo { PluginRegInfo(); ~PluginRegInfo(); - bool plugin_registered; - char *plugin_path; + bool plugin_registered = false; + char *plugin_path = nullptr; - char *plugin_name; - char *vendor_name; - char *support_email; + char *plugin_name = nullptr; + char *vendor_name = nullptr; + char *support_email = nullptr; - void *dlh; + void *dlh = nullptr; LINK(PluginRegInfo, link); }; diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc index 3e488ccc050..b4a62e5845b 100644 --- a/proxy/PluginVC.cc +++ b/proxy/PluginVC.cc @@ -1281,11 +1281,11 @@ class PVCTestDriver : public NetTestDriver int main_handler(int event, void *data); private: - unsigned i; - unsigned completions_received; + unsigned i = 0; + unsigned completions_received = 0; }; -PVCTestDriver::PVCTestDriver() : NetTestDriver(), i(0), completions_received(0) {} +PVCTestDriver::PVCTestDriver() : NetTestDriver() {} PVCTestDriver::~PVCTestDriver() { diff --git a/proxy/PluginVC.h b/proxy/PluginVC.h index 1e879efcced..d6205494328 100644 --- a/proxy/PluginVC.h +++ b/proxy/PluginVC.h @@ -44,10 +44,10 @@ class PluginVCCore; struct PluginVCState { PluginVCState(); VIO vio; - bool shutdown; + bool shutdown = false; }; -inline PluginVCState::PluginVCState() : vio(), shutdown(false) {} +inline PluginVCState::PluginVCState() : vio() {} enum PluginVC_t { PLUGIN_VC_UNKNOWN, @@ -247,37 +247,27 @@ class PluginVCCore : public Continuation void init(); void destroy(); - Continuation *connect_to; - bool connected; + Continuation *connect_to = nullptr; + bool connected = false; - MIOBuffer *p_to_a_buffer; - IOBufferReader *p_to_a_reader; + MIOBuffer *p_to_a_buffer = nullptr; + IOBufferReader *p_to_a_reader = nullptr; - MIOBuffer *a_to_p_buffer; - IOBufferReader *a_to_p_reader; + MIOBuffer *a_to_p_buffer = nullptr; + IOBufferReader *a_to_p_reader = nullptr; IpEndpoint passive_addr_struct; IpEndpoint active_addr_struct; - void *passive_data; - void *active_data; + void *passive_data = nullptr; + void *active_data = nullptr; static int32_t nextid; - unsigned id; + unsigned id = 0; }; -inline PluginVCCore::PluginVCCore() - : active_vc(this), - passive_vc(this), - connect_to(nullptr), - connected(false), - p_to_a_buffer(nullptr), - p_to_a_reader(nullptr), - a_to_p_buffer(nullptr), - a_to_p_reader(nullptr), - passive_data(nullptr), - active_data(nullptr), - id(0) +inline PluginVCCore::PluginVCCore() : active_vc(this), passive_vc(this) + { memset(&active_addr_struct, 0, sizeof active_addr_struct); memset(&passive_addr_struct, 0, sizeof passive_addr_struct); diff --git a/proxy/ProxyClientTransaction.cc b/proxy/ProxyClientTransaction.cc index c00de079a35..71323a4e202 100644 --- a/proxy/ProxyClientTransaction.cc +++ b/proxy/ProxyClientTransaction.cc @@ -27,15 +27,7 @@ #define HttpTxnDebug(fmt, ...) SsnDebug(this, "http_txn", fmt, __VA_ARGS__) -ProxyClientTransaction::ProxyClientTransaction() - : VConnection(nullptr), - parent(nullptr), - current_reader(nullptr), - sm_reader(nullptr), - host_res_style(HOST_RES_NONE), - restart_immediate(false) -{ -} +ProxyClientTransaction::ProxyClientTransaction() : VConnection(nullptr) {} void ProxyClientTransaction::new_transaction() diff --git a/proxy/ProxyClientTransaction.h b/proxy/ProxyClientTransaction.h index ce5d009be9c..c4ebfe0382f 100644 --- a/proxy/ProxyClientTransaction.h +++ b/proxy/ProxyClientTransaction.h @@ -266,18 +266,18 @@ class ProxyClientTransaction : public VConnection virtual void decrement_client_transactions_stat() = 0; protected: - ProxyClientSession *parent; - HttpSM *current_reader; - IOBufferReader *sm_reader; + ProxyClientSession *parent = nullptr; + HttpSM *current_reader = nullptr; + IOBufferReader *sm_reader = nullptr; /// DNS resolution preferences. - HostResStyle host_res_style; + HostResStyle host_res_style = HOST_RES_NONE; /// Local outbound address control. in_port_t outbound_port{0}; IpAddr outbound_ip4; IpAddr outbound_ip6; - bool restart_immediate; + bool restart_immediate = false; private: }; diff --git a/proxy/StatPages.h b/proxy/StatPages.h index 3d204f5f325..cbce9cb703b 100644 --- a/proxy/StatPages.h +++ b/proxy/StatPages.h @@ -64,13 +64,13 @@ typedef Action *(*StatPagesFunc)(Continuation *cont, HTTPHdr *header); struct StatPageData { - char *data; - char *type; - int length; + char *data = nullptr; + char *type = nullptr; + int length = 0; - StatPageData() : data(nullptr), type(nullptr), length(0) {} - StatPageData(char *adata) : data(adata), type(nullptr) { length = strlen(adata); } - StatPageData(char *adata, int alength) : data(adata), type(nullptr), length(alength) {} + StatPageData() {} + StatPageData(char *adata) : data(adata) { length = strlen(adata); } + StatPageData(char *adata, int alength) : data(adata), length(alength) {} }; struct StatPagesManager { diff --git a/proxy/Transform.cc b/proxy/Transform.cc index d15f18ad139..3049305812b 100644 --- a/proxy/Transform.cc +++ b/proxy/Transform.cc @@ -524,8 +524,7 @@ TransformVConnection::backlog(uint64_t limit) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -TransformControl::TransformControl() - : Continuation(new_ProxyMutex()), m_hooks(), m_tvc(nullptr), m_read_buf(nullptr), m_write_buf(nullptr) +TransformControl::TransformControl() : Continuation(new_ProxyMutex()), m_hooks() { SET_HANDLER(&TransformControl::handle_event); diff --git a/proxy/Transform.h b/proxy/Transform.h index 5d2e3ed057c..2030eb6df03 100644 --- a/proxy/Transform.h +++ b/proxy/Transform.h @@ -30,10 +30,10 @@ #define TRANSFORM_READ_READY (TRANSFORM_EVENTS_START + 0) typedef struct _RangeRecord { - _RangeRecord() : _start(-1), _end(-1), _done_byte(-1) {} - int64_t _start; - int64_t _end; - int64_t _done_byte; + _RangeRecord() {} + int64_t _start = -1; + int64_t _end = -1; + int64_t _done_byte = -1; } RangeRecord; class TransformProcessor diff --git a/proxy/TransformInternal.h b/proxy/TransformInternal.h index 1fdbc706a86..02b862fb4ac 100644 --- a/proxy/TransformInternal.h +++ b/proxy/TransformInternal.h @@ -87,9 +87,9 @@ class TransformControl : public Continuation public: APIHooks m_hooks; - VConnection *m_tvc; - IOBufferReader *m_read_buf; - MIOBuffer *m_write_buf; + VConnection *m_tvc = nullptr; + IOBufferReader *m_read_buf = nullptr; + MIOBuffer *m_write_buf = nullptr; }; class NullTransform : public INKVConnInternal diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 50726d0702d..0bbde741c23 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -1873,19 +1873,8 @@ ClassAllocator httpCacheAltAllocator("httpCacheAltAllocator"); -------------------------------------------------------------------------*/ int constexpr HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS; -HTTPCacheAlt::HTTPCacheAlt() - : m_magic(CACHE_ALT_MAGIC_ALIVE), - m_writeable(1), - m_unmarshal_len(-1), - m_id(-1), - m_rid(-1), - m_request_hdr(), - m_response_hdr(), - m_request_sent_time(0), - m_response_received_time(0), - m_frag_offset_count(0), - m_frag_offsets(nullptr), - m_ext_buffer(nullptr) +HTTPCacheAlt::HTTPCacheAlt() : m_request_hdr(), m_response_hdr() + { memset(&m_object_key[0], 0, CRYPTO_HASH_SIZE); m_object_size[0] = 0; diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h index a3adb759a40..94e1a103365 100644 --- a/proxy/hdrs/HTTP.h +++ b/proxy/hdrs/HTTP.h @@ -1292,18 +1292,18 @@ struct HTTPCacheAlt { void copy_frag_offsets_from(HTTPCacheAlt *src); void destroy(); - uint32_t m_magic; + uint32_t m_magic = CACHE_ALT_MAGIC_ALIVE; // Writeable is set to true is we reside // in a buffer owned by this structure. // INVARIANT: if own the buffer this HttpCacheAlt // we also own the buffers for the request & // response headers - int32_t m_writeable; - int32_t m_unmarshal_len; + int32_t m_writeable = 1; + int32_t m_unmarshal_len = -1; - int32_t m_id; - int32_t m_rid; + int32_t m_id = -1; + int32_t m_rid = -1; int32_t m_object_key[sizeof(CryptoHash) / sizeof(int32_t)]; int32_t m_object_size[2]; @@ -1311,12 +1311,12 @@ struct HTTPCacheAlt { HTTPHdr m_request_hdr; HTTPHdr m_response_hdr; - time_t m_request_sent_time; - time_t m_response_received_time; + time_t m_request_sent_time = 0; + time_t m_response_received_time = 0; /// # of fragment offsets in this alternate. /// @note This is one less than the number of fragments. - int m_frag_offset_count; + int m_frag_offset_count = 0; /// Type of offset for a fragment. typedef uint64_t FragOffset; /// Table of fragment offsets. @@ -1324,7 +1324,7 @@ struct HTTPCacheAlt { /// first byte past the end of fragment 0 which is also the first /// byte of fragment 1. For this reason there is no fragment offset /// for the last fragment. - FragOffset *m_frag_offsets; + FragOffset *m_frag_offsets = nullptr; /// # of fragment offsets built in to object. static int constexpr N_INTEGRAL_FRAG_OFFSETS = 4; /// Integral fragment offset table. @@ -1337,7 +1337,7 @@ struct HTTPCacheAlt { // We don't want to use a ref count ptr (Ptr<>) // since our ownership model requires explicit // destroys and ref count pointers defeat this - RefCountObj *m_ext_buffer; + RefCountObj *m_ext_buffer = nullptr; }; class HTTPInfo @@ -1345,9 +1345,9 @@ class HTTPInfo public: typedef HTTPCacheAlt::FragOffset FragOffset; ///< Import type. - HTTPCacheAlt *m_alt; + HTTPCacheAlt *m_alt = nullptr; - HTTPInfo() : m_alt(nullptr) {} + HTTPInfo() {} ~HTTPInfo() { clear(); } void clear() diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index 1435f4b391a..825d19c4cde 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -341,10 +341,10 @@ HdrHeap::unmarshal_size() const // struct MarshalXlate { - char const *start; - char const *end; - char const *offset; - MarshalXlate() : start(nullptr), end(nullptr), offset(nullptr) {} + char const *start = nullptr; + char const *end = nullptr; + char const *offset = nullptr; + MarshalXlate() {} }; struct HeapCheck { @@ -458,7 +458,7 @@ struct HeapCheck { // struct HdrHeapSDKHandle { public: - HdrHeapSDKHandle() : m_heap(nullptr) {} + HdrHeapSDKHandle() {} ~HdrHeapSDKHandle() { clear(); } // clear() only deallocates chained SDK return values // The underlying MBuffer is left untouched @@ -471,7 +471,7 @@ struct HdrHeapSDKHandle { void set(const HdrHeapSDKHandle *from); const char *make_sdk_string(const char *raw_str, int raw_str_len); - HdrHeap *m_heap; + HdrHeap *m_heap = nullptr; // In order to prevent gratitous refcounting, // automatic C++ copies are disabled! diff --git a/proxy/hdrs/HdrTest.h b/proxy/hdrs/HdrTest.h index 7f8dc839b78..f9b29d9fed2 100644 --- a/proxy/hdrs/HdrTest.h +++ b/proxy/hdrs/HdrTest.h @@ -39,9 +39,9 @@ class HdrTest { public: - RegressionTest *rtest; + RegressionTest *rtest = nullptr; - HdrTest() : rtest(nullptr){}; + HdrTest(){}; ~HdrTest(){}; int go(RegressionTest *t, int atype); diff --git a/proxy/hdrs/MIME.h b/proxy/hdrs/MIME.h index 972af091572..de5ea81b756 100644 --- a/proxy/hdrs/MIME.h +++ b/proxy/hdrs/MIME.h @@ -965,9 +965,9 @@ MIMEField::has_dups() const ***********************************************************************/ struct MIMEFieldIter { - MIMEFieldIter() : m_slot(0), m_block(nullptr) {} - uint32_t m_slot; - MIMEFieldBlockImpl *m_block; + MIMEFieldIter() {} + uint32_t m_slot = 0; + MIMEFieldBlockImpl *m_block = nullptr; }; /*------------------------------------------------------------------------- diff --git a/proxy/hdrs/URL.h b/proxy/hdrs/URL.h index a98d24c1d84..0cecc705a25 100644 --- a/proxy/hdrs/URL.h +++ b/proxy/hdrs/URL.h @@ -223,7 +223,7 @@ url_canonicalize_port(int type, int port) class URL : public HdrHeapSDKHandle { public: - URLImpl *m_url_impl; + URLImpl *m_url_impl = nullptr; URL(); ~URL(); @@ -290,7 +290,7 @@ class URL : public HdrHeapSDKHandle /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -inline URL::URL() : m_url_impl(nullptr) {} +inline URL::URL() {} /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/http/Http1ClientSession.cc b/proxy/http/Http1ClientSession.cc index 9f1a00224e2..10ac26c3ae2 100644 --- a/proxy/http/Http1ClientSession.cc +++ b/proxy/http/Http1ClientSession.cc @@ -62,24 +62,7 @@ ink_mutex debug_cs_list_mutex; ClassAllocator http1ClientSessionAllocator("http1ClientSessionAllocator"); -Http1ClientSession::Http1ClientSession() - : client_vc(nullptr), - magic(HTTP_CS_MAGIC_DEAD), - transact_count(0), - tcp_init_cwnd_set(false), - half_close(false), - conn_decrease(false), - read_buffer(nullptr), - sm_reader(nullptr), - read_state(HCS_INIT), - ka_vio(nullptr), - slave_ka_vio(nullptr), - bound_ss(nullptr), - released_transactions(0), - f_outbound_transparent(false), - f_transparent_passthrough(false) -{ -} +Http1ClientSession::Http1ClientSession() {} void Http1ClientSession::destroy() diff --git a/proxy/http/Http1ClientSession.h b/proxy/http/Http1ClientSession.h index ce8346665ce..74d7a99ff4c 100644 --- a/proxy/http/Http1ClientSession.h +++ b/proxy/http/Http1ClientSession.h @@ -184,33 +184,33 @@ class Http1ClientSession : public ProxyClientSession HCS_CLOSED, }; - NetVConnection *client_vc; - int magic; - int transact_count; - bool tcp_init_cwnd_set; - bool half_close; - bool conn_decrease; + NetVConnection *client_vc = nullptr; + int magic = HTTP_SS_MAGIC_DEAD; + int transact_count = 0; + bool tcp_init_cwnd_set = false; + bool half_close = false; + bool conn_decrease = false; - MIOBuffer *read_buffer; - IOBufferReader *sm_reader; + MIOBuffer *read_buffer = nullptr; + IOBufferReader *sm_reader = nullptr; - C_Read_State read_state; + C_Read_State read_state = HCS_INIT; - VIO *ka_vio; - VIO *slave_ka_vio; + VIO *ka_vio = nullptr; + VIO *slave_ka_vio = nullptr; - HttpServerSession *bound_ss; + HttpServerSession *bound_ss = nullptr; - int released_transactions; + int released_transactions = 0; public: // Link debug_link; LINK(Http1ClientSession, debug_link); /// Set outbound connection to transparent. - bool f_outbound_transparent; + bool f_outbound_transparent = false; /// Transparently pass-through non-HTTP traffic. - bool f_transparent_passthrough; + bool f_transparent_passthrough = false; Http1ClientTransaction trans; }; diff --git a/proxy/http/HttpCacheSM.cc b/proxy/http/HttpCacheSM.cc index 52190bc4c17..8a859f09a0a 100644 --- a/proxy/http/HttpCacheSM.cc +++ b/proxy/http/HttpCacheSM.cc @@ -45,7 +45,7 @@ Debug("http_cache", "[%" PRId64 "] [%s, %s]", master_sm->sm_id, #state_name, HttpDebugNames::get_event_name(event)); \ } -HttpCacheAction::HttpCacheAction() : sm(nullptr) {} +HttpCacheAction::HttpCacheAction() {} void HttpCacheAction::cancel(Continuation *c) @@ -61,25 +61,9 @@ HttpCacheAction::cancel(Continuation *c) HttpCacheSM::HttpCacheSM() : Continuation(nullptr), - cache_read_vc(nullptr), - cache_write_vc(nullptr), - read_locked(false), - write_locked(false), - readwhilewrite_inprogress(false), - master_sm(nullptr), - pending_action(nullptr), - captive_action(), - open_read_cb(false), - open_write_cb(false), - open_read_tries(0), - read_request_hdr(nullptr), - http_params(nullptr), - read_pin_in_cache(0), - retry_write(true), - open_write_tries(0), - lookup_url(nullptr), - lookup_max_recursive(0), - current_lookup_level(0) + + captive_action() + { } diff --git a/proxy/http/HttpCacheSM.h b/proxy/http/HttpCacheSM.h index 435279fb436..89b8542ec58 100644 --- a/proxy/http/HttpCacheSM.h +++ b/proxy/http/HttpCacheSM.h @@ -49,7 +49,7 @@ struct HttpCacheAction : public Action { { sm = sm_arg; }; - HttpCacheSM *sm; + HttpCacheSM *sm = nullptr; }; class HttpCacheSM : public Continuation @@ -70,16 +70,16 @@ class HttpCacheSM : public Continuation Action *open_write(const HttpCacheKey *key, URL *url, HTTPHdr *request, CacheHTTPInfo *old_info, time_t pin_in_cache, bool retry, bool allow_multiple); - CacheVConnection *cache_read_vc; - CacheVConnection *cache_write_vc; + CacheVConnection *cache_read_vc = nullptr; + CacheVConnection *cache_write_vc = nullptr; - bool read_locked; - bool write_locked; + bool read_locked = false; + bool write_locked = false; // Flag to check whether read-while-write is in progress or not - bool readwhilewrite_inprogress; + bool readwhilewrite_inprogress = false; - HttpSM *master_sm; - Action *pending_action; + HttpSM *master_sm = nullptr; + Action *pending_action = nullptr; // Function to set readwhilewrite_inprogress flag inline void @@ -190,24 +190,24 @@ class HttpCacheSM : public Continuation int state_cache_open_write(int event, void *data); HttpCacheAction captive_action; - bool open_read_cb; - bool open_write_cb; + bool open_read_cb = false; + bool open_write_cb = false; // Open read parameters - int open_read_tries; - HTTPHdr *read_request_hdr; - OverridableHttpConfigParams *http_params; - time_t read_pin_in_cache; + int open_read_tries = 0; + HTTPHdr *read_request_hdr = nullptr; + OverridableHttpConfigParams *http_params = nullptr; + time_t read_pin_in_cache = 0; // Open write parameters - bool retry_write; - int open_write_tries; + bool retry_write = true; + int open_write_tries = 0; // Common parameters - URL *lookup_url; + URL *lookup_url = nullptr; HttpCacheKey cache_key; // to keep track of multiple cache lookups - int lookup_max_recursive; - int current_lookup_level; + int lookup_max_recursive = 0; + int current_lookup_level = 0; }; diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index e7aae44d6a4..0bb0a354d64 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -358,11 +358,11 @@ extern RecRawStatBlock *http_rsb; // (corresponds to a "*" in the config file) ///////////////////////////////////////////////////////////// struct HttpConfigPortRange { - int low; - int high; - HttpConfigPortRange *next; + int low = 0; + int high = 0; + HttpConfigPortRange *next = nullptr; - HttpConfigPortRange() : low(0), high(0), next(nullptr) {} + HttpConfigPortRange() {} ~HttpConfigPortRange() { if (next) @@ -436,176 +436,53 @@ static std::map action_map = { // and State (txn) structure. It allows for certain configs // to be overridable per transaction more easily. struct OverridableHttpConfigParams { - OverridableHttpConfigParams() - : maintain_pristine_host_hdr(1), - chunking_enabled(1), - negative_caching_enabled(0), - negative_revalidating_enabled(0), - cache_when_to_revalidate(0), - keep_alive_enabled_in(1), - keep_alive_enabled_out(1), - keep_alive_post_out(1), - server_session_sharing_match(TS_SERVER_SESSION_SHARING_MATCH_BOTH), - auth_server_session_private(1), - fwd_proxy_auth_to_parent(0), - uncacheable_requests_bypass_parent(1), - attach_server_session_to_client(0), - forward_connect_method(0), - insert_age_in_response(1), - anonymize_remove_from(0), - anonymize_remove_referer(0), - anonymize_remove_user_agent(0), - anonymize_remove_cookie(0), - anonymize_remove_client_ip(0), - anonymize_insert_client_ip(1), - proxy_response_server_enabled(1), - proxy_response_hsts_include_subdomains(0), - insert_squid_x_forwarded_for(1), - insert_forwarded(HttpForwarded::OptionBitSet()), - send_http11_requests(1), - cache_http(1), - cache_ignore_client_no_cache(1), - cache_ignore_client_cc_max_age(0), - cache_ims_on_client_no_cache(1), - cache_ignore_server_no_cache(0), - cache_responses_to_cookies(1), - cache_ignore_auth(0), - cache_urls_that_look_dynamic(1), - cache_required_headers(2), - cache_range_lookup(1), - cache_range_write(0), - allow_multi_range(0), - cache_enable_default_vary_headers(0), - ignore_accept_mismatch(0), - ignore_accept_language_mismatch(0), - ignore_accept_encoding_mismatch(0), - ignore_accept_charset_mismatch(0), - insert_request_via_string(1), - insert_response_via_string(0), - doc_in_cache_skip_dns(1), - flow_control_enabled(0), - normalize_ae(0), - srv_enabled(0), - parent_failures_update_hostdb(0), - cache_open_write_fail_action(0), - post_check_content_length_enabled(1), - request_buffer_enabled(0), - allow_half_open(1), - ssl_client_verify_server(0), - ssl_client_verify_server_policy(nullptr), - ssl_client_verify_server_properties(nullptr), - ssl_client_sni_policy(nullptr), - redirect_use_orig_cache_key(0), - number_of_redirections(0), - proxy_response_hsts_max_age(-1), - negative_caching_lifetime(1800), - negative_revalidating_lifetime(1800), - sock_recv_buffer_size_out(0), - sock_send_buffer_size_out(0), - sock_option_flag_out(0), - sock_packet_mark_out(0), - sock_packet_tos_out(0), - server_tcp_init_cwnd(0), - request_hdr_max_size(131072), - response_hdr_max_size(131072), - cache_heuristic_min_lifetime(3600), - cache_heuristic_max_lifetime(86400), - cache_guaranteed_min_lifetime(0), - cache_guaranteed_max_lifetime(31536000), - cache_max_stale_age(604800), - keep_alive_no_activity_timeout_in(120), - keep_alive_no_activity_timeout_out(120), - transaction_no_activity_timeout_in(30), - transaction_no_activity_timeout_out(30), - transaction_active_timeout_out(0), - transaction_active_timeout_in(900), - websocket_active_timeout(3600), - websocket_inactive_timeout(600), - connect_attempts_max_retries(0), - connect_attempts_max_retries_dead_server(3), - connect_attempts_rr_retries(3), - connect_attempts_timeout(30), - post_connect_attempts_timeout(1800), - parent_connect_attempts(4), - parent_retry_time(300), - parent_fail_threshold(10), - per_parent_connect_attempts(2), - parent_connect_timeout(30), - down_server_timeout(300), - client_abort_threshold(10), - max_cache_open_read_retries(-1), - cache_open_read_retry_time(10), - cache_generation_number(-1), - max_cache_open_write_retries(1), - background_fill_active_timeout(60), - http_chunking_size(4096), - flow_high_water_mark(0), - flow_low_water_mark(0), - default_buffer_size_index(8), - default_buffer_water_mark(32768), - slow_log_threshold(0), - body_factory_template_base(nullptr), - body_factory_template_base_len(0), - proxy_response_server_string(nullptr), - proxy_response_server_string_len(0), - global_user_agent_header(nullptr), - global_user_agent_header_size(0), - cache_heuristic_lm_factor(0.10), - background_fill_threshold(0.5), - ssl_client_cert_filename(nullptr), - ssl_client_private_key_filename(nullptr), - ssl_client_ca_cert_filename(nullptr), - cache_vary_default_text(nullptr), - cache_vary_default_images(nullptr), - cache_vary_default_other(nullptr) - { - } + OverridableHttpConfigParams() : insert_forwarded(HttpForwarded::OptionBitSet()) {} // A simple rules here: // * Place all MgmtByte configs before all other configs - MgmtByte maintain_pristine_host_hdr; - MgmtByte chunking_enabled; + MgmtByte maintain_pristine_host_hdr = 1; + MgmtByte chunking_enabled = 1; //////////////////////////////// // Negative Response Caching // //////////////////////////////// - MgmtByte negative_caching_enabled; - MgmtByte negative_revalidating_enabled; + MgmtByte negative_caching_enabled = 0; + MgmtByte negative_revalidating_enabled = 0; - MgmtByte cache_when_to_revalidate; + MgmtByte cache_when_to_revalidate = 0; - MgmtByte keep_alive_enabled_in; - MgmtByte keep_alive_enabled_out; - MgmtByte keep_alive_post_out; // share server sessions for post + MgmtByte keep_alive_enabled_in = 1; + MgmtByte keep_alive_enabled_out = 1; + MgmtByte keep_alive_post_out = 1; // share server sessions for post - MgmtByte server_session_sharing_match; + MgmtByte server_session_sharing_match = TS_SERVER_SESSION_SHARING_MATCH_BOTH; // MgmtByte share_server_sessions; - MgmtByte auth_server_session_private; - MgmtByte fwd_proxy_auth_to_parent; - MgmtByte uncacheable_requests_bypass_parent; - MgmtByte attach_server_session_to_client; + MgmtByte auth_server_session_private = 1; + MgmtByte fwd_proxy_auth_to_parent = 0; + MgmtByte uncacheable_requests_bypass_parent = 1; + MgmtByte attach_server_session_to_client = 0; - MgmtByte forward_connect_method; + MgmtByte forward_connect_method = 0; - MgmtByte insert_age_in_response; + MgmtByte insert_age_in_response = 1; /////////////////////////////////////////////////////////////////// // Privacy: fields which are removed from the user agent request // /////////////////////////////////////////////////////////////////// - MgmtByte anonymize_remove_from; - MgmtByte anonymize_remove_referer; - MgmtByte anonymize_remove_user_agent; - MgmtByte anonymize_remove_cookie; - MgmtByte anonymize_remove_client_ip; - MgmtByte anonymize_insert_client_ip; + MgmtByte anonymize_remove_from = 0; + MgmtByte anonymize_remove_referer = 0; + MgmtByte anonymize_remove_user_agent = 0; + MgmtByte anonymize_remove_cookie = 0; + MgmtByte anonymize_remove_client_ip = 0; + MgmtByte anonymize_insert_client_ip = 1; - MgmtByte proxy_response_server_enabled; - MgmtByte proxy_response_hsts_include_subdomains; + MgmtByte proxy_response_server_enabled = 1; + MgmtByte proxy_response_hsts_include_subdomains = 0; ///////////////////// // X-Forwarded-For // ///////////////////// - MgmtByte insert_squid_x_forwarded_for; + MgmtByte insert_squid_x_forwarded_for = 1; /////////////// // Forwarded // @@ -615,197 +492,197 @@ struct OverridableHttpConfigParams { ////////////////////// // Version Hell // ////////////////////// - MgmtByte send_http11_requests; + MgmtByte send_http11_requests = 1; /////////////////// // cache control // /////////////////// - MgmtByte cache_http; - MgmtByte cache_ignore_client_no_cache; - MgmtByte cache_ignore_client_cc_max_age; - MgmtByte cache_ims_on_client_no_cache; - MgmtByte cache_ignore_server_no_cache; - MgmtByte cache_responses_to_cookies; - MgmtByte cache_ignore_auth; - MgmtByte cache_urls_that_look_dynamic; - MgmtByte cache_required_headers; - MgmtByte cache_range_lookup; - MgmtByte cache_range_write; - MgmtByte allow_multi_range; - - MgmtByte cache_enable_default_vary_headers; - - MgmtByte ignore_accept_mismatch; - MgmtByte ignore_accept_language_mismatch; - MgmtByte ignore_accept_encoding_mismatch; - MgmtByte ignore_accept_charset_mismatch; - - MgmtByte insert_request_via_string; - MgmtByte insert_response_via_string; + MgmtByte cache_http = 1; + MgmtByte cache_ignore_client_no_cache = 1; + MgmtByte cache_ignore_client_cc_max_age = 0; + MgmtByte cache_ims_on_client_no_cache = 1; + MgmtByte cache_ignore_server_no_cache = 0; + MgmtByte cache_responses_to_cookies = 1; + MgmtByte cache_ignore_auth = 0; + MgmtByte cache_urls_that_look_dynamic = 1; + MgmtByte cache_required_headers = 2; + MgmtByte cache_range_lookup = 1; + MgmtByte cache_range_write = 0; + MgmtByte allow_multi_range = 0; + + MgmtByte cache_enable_default_vary_headers = 0; + + MgmtByte ignore_accept_mismatch = 0; + MgmtByte ignore_accept_language_mismatch = 0; + MgmtByte ignore_accept_encoding_mismatch = 0; + MgmtByte ignore_accept_charset_mismatch = 0; + + MgmtByte insert_request_via_string = 1; + MgmtByte insert_response_via_string = 0; ////////////////////// // DOC IN CACHE NO DNS// ////////////////////// - MgmtByte doc_in_cache_skip_dns; - MgmtByte flow_control_enabled; + MgmtByte doc_in_cache_skip_dns = 1; + MgmtByte flow_control_enabled = 0; //////////////////////////////// // Optimize gzip alternates // //////////////////////////////// - MgmtByte normalize_ae; + MgmtByte normalize_ae = 0; ////////////////////////// // hostdb/dns variables // ////////////////////////// - MgmtByte srv_enabled; - MgmtByte parent_failures_update_hostdb; + MgmtByte srv_enabled = 0; + MgmtByte parent_failures_update_hostdb = 0; - MgmtByte cache_open_write_fail_action; + MgmtByte cache_open_write_fail_action = 0; //////////////////////// // Check Post request // //////////////////////// - MgmtByte post_check_content_length_enabled; + MgmtByte post_check_content_length_enabled = 1; //////////////////////////////////////////////// // Buffer post body before connecting servers // //////////////////////////////////////////////// - MgmtByte request_buffer_enabled; + MgmtByte request_buffer_enabled = 0; ///////////////////////////////////////////////// // Keep connection open after client sends FIN // ///////////////////////////////////////////////// - MgmtByte allow_half_open; + MgmtByte allow_half_open = 1; ///////////////////////////// // server verification mode// ///////////////////////////// - MgmtByte ssl_client_verify_server; - char *ssl_client_verify_server_policy; - char *ssl_client_verify_server_properties; - char *ssl_client_sni_policy; + MgmtByte ssl_client_verify_server = 0; + char *ssl_client_verify_server_policy = nullptr; + char *ssl_client_verify_server_properties = nullptr; + char *ssl_client_sni_policy = nullptr; ////////////////// // Redirection // ////////////////// - MgmtByte redirect_use_orig_cache_key; - MgmtInt number_of_redirections; + MgmtByte redirect_use_orig_cache_key = 0; + MgmtInt number_of_redirections = 0; - MgmtInt proxy_response_hsts_max_age; + MgmtInt proxy_response_hsts_max_age = -1; //////////////////////////////// // Negative cache lifetimes // //////////////////////////////// - MgmtInt negative_caching_lifetime; - MgmtInt negative_revalidating_lifetime; + MgmtInt negative_caching_lifetime = 1800; + MgmtInt negative_revalidating_lifetime = 1800; /////////////////////////////////////// // origin server connection settings // /////////////////////////////////////// - MgmtInt sock_recv_buffer_size_out; - MgmtInt sock_send_buffer_size_out; - MgmtInt sock_option_flag_out; - MgmtInt sock_packet_mark_out; - MgmtInt sock_packet_tos_out; + MgmtInt sock_recv_buffer_size_out = 0; + MgmtInt sock_send_buffer_size_out = 0; + MgmtInt sock_option_flag_out = 0; + MgmtInt sock_packet_mark_out = 0; + MgmtInt sock_packet_tos_out = 0; /////////////////////////////// // Initial congestion window // /////////////////////////////// - MgmtInt server_tcp_init_cwnd; + MgmtInt server_tcp_init_cwnd = 0; /////////////// // Hdr Limit // /////////////// - MgmtInt request_hdr_max_size; - MgmtInt response_hdr_max_size; + MgmtInt request_hdr_max_size = 131072; + MgmtInt response_hdr_max_size = 131072; ///////////////////// // cache variables // ///////////////////// - MgmtInt cache_heuristic_min_lifetime; - MgmtInt cache_heuristic_max_lifetime; - MgmtInt cache_guaranteed_min_lifetime; - MgmtInt cache_guaranteed_max_lifetime; - MgmtInt cache_max_stale_age; + MgmtInt cache_heuristic_min_lifetime = 3600; + MgmtInt cache_heuristic_max_lifetime = 86400; + MgmtInt cache_guaranteed_min_lifetime = 0; + MgmtInt cache_guaranteed_max_lifetime = 31536000; + MgmtInt cache_max_stale_age = 604800; /////////////////////////////////////////////////// // connection variables. timeouts are in seconds // /////////////////////////////////////////////////// - MgmtInt keep_alive_no_activity_timeout_in; - MgmtInt keep_alive_no_activity_timeout_out; - MgmtInt transaction_no_activity_timeout_in; - MgmtInt transaction_no_activity_timeout_out; - MgmtInt transaction_active_timeout_out; - MgmtInt transaction_active_timeout_in; - MgmtInt websocket_active_timeout; - MgmtInt websocket_inactive_timeout; + MgmtInt keep_alive_no_activity_timeout_in = 120; + MgmtInt keep_alive_no_activity_timeout_out = 120; + MgmtInt transaction_no_activity_timeout_in = 30; + MgmtInt transaction_no_activity_timeout_out = 30; + MgmtInt transaction_active_timeout_out = 0; + MgmtInt transaction_active_timeout_in = 900; + MgmtInt websocket_active_timeout = 3600; + MgmtInt websocket_inactive_timeout = 600; //////////////////////////////////// // origin server connect attempts // //////////////////////////////////// - MgmtInt connect_attempts_max_retries; - MgmtInt connect_attempts_max_retries_dead_server; - MgmtInt connect_attempts_rr_retries; - MgmtInt connect_attempts_timeout; - MgmtInt post_connect_attempts_timeout; + MgmtInt connect_attempts_max_retries = 0; + MgmtInt connect_attempts_max_retries_dead_server = 3; + MgmtInt connect_attempts_rr_retries = 3; + MgmtInt connect_attempts_timeout = 30; + MgmtInt post_connect_attempts_timeout = 1800; //////////////////////////////////// // parent proxy connect attempts // /////////////////////////////////// - MgmtInt parent_connect_attempts; - MgmtInt parent_retry_time; - MgmtInt parent_fail_threshold; - MgmtInt per_parent_connect_attempts; - MgmtInt parent_connect_timeout; + MgmtInt parent_connect_attempts = 4; + MgmtInt parent_retry_time = 300; + MgmtInt parent_fail_threshold = 10; + MgmtInt per_parent_connect_attempts = 2; + MgmtInt parent_connect_timeout = 30; - MgmtInt down_server_timeout; - MgmtInt client_abort_threshold; + MgmtInt down_server_timeout = 300; + MgmtInt client_abort_threshold = 10; // open read failure retries. - MgmtInt max_cache_open_read_retries; - MgmtInt cache_open_read_retry_time; // time is in mseconds - MgmtInt cache_generation_number; + MgmtInt max_cache_open_read_retries = -1; + MgmtInt cache_open_read_retry_time = 10; // time is in mseconds + MgmtInt cache_generation_number = -1; // open write failure retries. - MgmtInt max_cache_open_write_retries; + MgmtInt max_cache_open_write_retries = 1; - MgmtInt background_fill_active_timeout; + MgmtInt background_fill_active_timeout = 60; - MgmtInt http_chunking_size; // Maximum chunk size for chunked output. - MgmtInt flow_high_water_mark; ///< Flow control high water mark. - MgmtInt flow_low_water_mark; ///< Flow control low water mark. + MgmtInt http_chunking_size = 4096; // Maximum chunk size for chunked output. + MgmtInt flow_high_water_mark = 0; ///< Flow control high water mark. + MgmtInt flow_low_water_mark = 0; ///< Flow control low water mark. - MgmtInt default_buffer_size_index; - MgmtInt default_buffer_water_mark; - MgmtInt slow_log_threshold; + MgmtInt default_buffer_size_index = 8; + MgmtInt default_buffer_water_mark = 32768; + MgmtInt slow_log_threshold = 0; OutboundConnTrack::TxnConfig outbound_conntrack; /////////////////////////////////////////////////////////////////// // Server header // /////////////////////////////////////////////////////////////////// - char *body_factory_template_base; - size_t body_factory_template_base_len; - char *proxy_response_server_string; // This does not get free'd by us! - size_t proxy_response_server_string_len; // Updated when server_string is set. + char *body_factory_template_base = nullptr; + size_t body_factory_template_base_len = 0; + char *proxy_response_server_string = nullptr; // This does not get free'd by us! + size_t proxy_response_server_string_len = 0; // Updated when server_string is set. /////////////////////////////////////////////////////////////////// // Global User Agent header // /////////////////////////////////////////////////////////////////// - char *global_user_agent_header; // This does not get free'd by us! - size_t global_user_agent_header_size; // Updated when user_agent is set. + char *global_user_agent_header = nullptr; // This does not get free'd by us! + size_t global_user_agent_header_size = 0; // Updated when user_agent is set. - MgmtFloat cache_heuristic_lm_factor; - MgmtFloat background_fill_threshold; + MgmtFloat cache_heuristic_lm_factor = 0.10; + MgmtFloat background_fill_threshold = 0.5; // Various strings, good place for them here ... - char *ssl_client_cert_filename; - char *ssl_client_private_key_filename; - char *ssl_client_ca_cert_filename; + char *ssl_client_cert_filename = nullptr; + char *ssl_client_private_key_filename = nullptr; + char *ssl_client_ca_cert_filename = nullptr; - char *cache_vary_default_text; - char *cache_vary_default_images; - char *cache_vary_default_other; + char *cache_vary_default_text = nullptr; + char *cache_vary_default_images = nullptr; + char *cache_vary_default_other = nullptr; }; ///////////////////////////////////////////////////////////// diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index 09530976492..7863833cf36 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -119,12 +119,12 @@ ssl_unregister_protocol(const char *protocol, Continuation *contp) */ struct HttpProxyAcceptor { /// Accept continuation. - Continuation *_accept; + Continuation *_accept = nullptr; /// Options for @c NetProcessor. NetProcessor::AcceptOptions _net_opt; /// Default constructor. - HttpProxyAcceptor() : _accept(nullptr) {} + HttpProxyAcceptor() {} }; /** Global acceptors. diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index 8ceb79edd7a..76a0d1392fd 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -126,10 +126,10 @@ HttpVCTable::is_table_clear() const } struct HttpTransformInfo { - HttpVCTableEntry *entry; - VConnection *vc; + HttpVCTableEntry *entry = nullptr; + VConnection *vc = nullptr; - HttpTransformInfo() : entry(nullptr), vc(nullptr) {} + HttpTransformInfo() {} }; enum { diff --git a/proxy/http/HttpSessionAccept.h b/proxy/http/HttpSessionAccept.h index 0f9978bf0a6..362dd8bfb86 100644 --- a/proxy/http/HttpSessionAccept.h +++ b/proxy/http/HttpSessionAccept.h @@ -58,7 +58,7 @@ class HttpSessionAcceptOptions HttpSessionAcceptOptions(); // Connection type (HttpProxyPort::TransportType) - int transport_type; + int transport_type = 0; /// Set the transport type. self &setTransportType(int); /// Local address to bind for outbound connections. @@ -70,15 +70,15 @@ class HttpSessionAcceptOptions /// Set the outbound IP address to @a ip. self &setOutboundIp(IpEndpoint *ip); /// Local port for outbound connection. - uint16_t outbound_port; + uint16_t outbound_port = 0; /// Set outbound port. self &setOutboundPort(uint16_t); /// Outbound transparent. - bool f_outbound_transparent; + bool f_outbound_transparent = false; /// Set outbound transparency. self &setOutboundTransparent(bool); /// Transparent pass-through. - bool f_transparent_passthrough; + bool f_transparent_passthrough = false; /// Set transparent passthrough. self &setTransparentPassthrough(bool); /// Host address resolution preference order. @@ -92,7 +92,7 @@ class HttpSessionAcceptOptions }; inline HttpSessionAcceptOptions::HttpSessionAcceptOptions() - : transport_type(0), outbound_port(0), f_outbound_transparent(false), f_transparent_passthrough(false) + { memcpy(host_res_preference, host_res_default_preference_order, sizeof(host_res_preference)); } diff --git a/proxy/http/HttpSessionManager.h b/proxy/http/HttpSessionManager.h index f83196aad0e..4112e3b8ae7 100644 --- a/proxy/http/HttpSessionManager.h +++ b/proxy/http/HttpSessionManager.h @@ -101,7 +101,7 @@ class ServerSessionPool : public Continuation class HttpSessionManager { public: - HttpSessionManager() : m_g_pool(nullptr) {} + HttpSessionManager() {} ~HttpSessionManager() {} HSMresult_t acquire_session(Continuation *cont, sockaddr const *addr, const char *hostname, ProxyClientTransaction *ua_txn, HttpSM *sm); @@ -113,7 +113,7 @@ class HttpSessionManager private: /// Global pool, used if not per thread pools. /// @internal We delay creating this because the session manager is created during global statics init. - ServerSessionPool *m_g_pool; + ServerSessionPool *m_g_pool = nullptr; }; extern HttpSessionManager httpSessionManager; diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 51901b83090..8e37e72d321 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -44,26 +44,7 @@ static const char *const CHUNK_HEADER_FMT = "%" PRIx64 "\r\n"; // a block in the input stream. static int const CHUNK_IOBUFFER_SIZE_INDEX = MIN_IOBUFFER_SIZE; -ChunkedHandler::ChunkedHandler() - : action(ACTION_UNSET), - chunked_reader(nullptr), - dechunked_buffer(nullptr), - dechunked_size(0), - dechunked_reader(nullptr), - chunked_buffer(nullptr), - chunked_size(0), - truncation(false), - skip_bytes(0), - state(CHUNK_READ_CHUNK), - cur_chunk_size(0), - bytes_left(0), - last_server_event(VC_EVENT_NONE), - running_sum(0), - num_digits(0), - max_chunk_size(DEFAULT_MAX_CHUNK_SIZE), - max_chunk_header_len(0) -{ -} +ChunkedHandler::ChunkedHandler() : max_chunk_size(DEFAULT_MAX_CHUNK_SIZE) {} void ChunkedHandler::init(IOBufferReader *buffer_in, HttpTunnelProducer *p) @@ -393,32 +374,7 @@ ChunkedHandler::generate_chunked_content() return false; } -HttpTunnelProducer::HttpTunnelProducer() - : consumer_list(), - self_consumer(nullptr), - vc(nullptr), - vc_handler(nullptr), - read_vio(nullptr), - read_buffer(nullptr), - buffer_start(nullptr), - vc_type(HT_HTTP_SERVER), - chunking_action(TCA_PASSTHRU_DECHUNKED_CONTENT), - do_chunking(false), - do_dechunking(false), - do_chunked_passthru(false), - init_bytes_done(0), - nbytes(0), - ntodo(0), - bytes_read(0), - handler_state(0), - last_event(0), - num_consumers(0), - alive(false), - read_success(false), - flow_control_source(nullptr), - name(nullptr) -{ -} +HttpTunnelProducer::HttpTunnelProducer() : consumer_list() {} uint64_t HttpTunnelProducer::backlog(uint64_t limit) @@ -484,23 +440,7 @@ HttpTunnelProducer::set_throttle_src(HttpTunnelProducer *srcp) } } -HttpTunnelConsumer::HttpTunnelConsumer() - : link(), - producer(nullptr), - self_producer(nullptr), - vc_type(HT_HTTP_CLIENT), - vc(nullptr), - buffer_reader(nullptr), - vc_handler(nullptr), - write_vio(nullptr), - skip_bytes(0), - bytes_written(0), - handler_state(0), - alive(false), - write_success(false), - name(nullptr) -{ -} +HttpTunnelConsumer::HttpTunnelConsumer() : link() {} HttpTunnel::HttpTunnel() : Continuation(nullptr) {} diff --git a/proxy/http/HttpTunnel.h b/proxy/http/HttpTunnel.h index e47160d9a0c..9a91f5026e6 100644 --- a/proxy/http/HttpTunnel.h +++ b/proxy/http/HttpTunnel.h @@ -94,27 +94,27 @@ struct ChunkedHandler { enum Action { ACTION_DOCHUNK = 0, ACTION_DECHUNK, ACTION_PASSTHRU, ACTION_UNSET }; - Action action; + Action action = ACTION_UNSET; - IOBufferReader *chunked_reader; - MIOBuffer *dechunked_buffer; - int64_t dechunked_size; + IOBufferReader *chunked_reader = nullptr; + MIOBuffer *dechunked_buffer = nullptr; + int64_t dechunked_size = 0; - IOBufferReader *dechunked_reader; - MIOBuffer *chunked_buffer; - int64_t chunked_size; + IOBufferReader *dechunked_reader = nullptr; + MIOBuffer *chunked_buffer = nullptr; + int64_t chunked_size = 0; - bool truncation; - int64_t skip_bytes; + bool truncation = false; + int64_t skip_bytes = 0; - ChunkedState state; - int64_t cur_chunk_size; - int64_t bytes_left; - int last_server_event; + ChunkedState state = CHUNK_READ_CHUNK; + int64_t cur_chunk_size = 0; + int64_t bytes_left = 0; + int last_server_event = VC_EVENT_NONE; // Parsing Info - int running_sum; - int num_digits; + int running_sum = 0; + int num_digits = 0; /// @name Output data. //@{ @@ -125,7 +125,7 @@ struct ChunkedHandler { /// It holds the header for a maximal sized chunk which will cover /// almost all output chunks. char max_chunk_header[16]; - int max_chunk_header_len; + int max_chunk_header_len = 0; //@} ChunkedHandler(); @@ -152,22 +152,22 @@ struct HttpTunnelConsumer { HttpTunnelConsumer(); LINK(HttpTunnelConsumer, link); - HttpTunnelProducer *producer; - HttpTunnelProducer *self_producer; + HttpTunnelProducer *producer = nullptr; + HttpTunnelProducer *self_producer = nullptr; - HttpTunnelType_t vc_type; - VConnection *vc; - IOBufferReader *buffer_reader; - HttpConsumerHandler vc_handler; - VIO *write_vio; + HttpTunnelType_t vc_type = HT_HTTP_CLIENT; + VConnection *vc = nullptr; + IOBufferReader *buffer_reader = nullptr; + HttpConsumerHandler vc_handler = nullptr; + VIO *write_vio = nullptr; - int64_t skip_bytes; // bytes to skip at beginning of stream - int64_t bytes_written; // total bytes written to the vc - int handler_state; // state used the handlers + int64_t skip_bytes = 0; // bytes to skip at beginning of stream + int64_t bytes_written = 0; // total bytes written to the vc + int handler_state = 0; // state used the handlers - bool alive; - bool write_success; - const char *name; + bool alive = false; + bool write_success = false; + const char *name = nullptr; /** Check if this consumer is downstream from @a vc. @return @c true if any producer in the tunnel eventually feeds @@ -184,37 +184,37 @@ struct HttpTunnelProducer { HttpTunnelProducer(); DLL consumer_list; - HttpTunnelConsumer *self_consumer; - VConnection *vc; - HttpProducerHandler vc_handler; - VIO *read_vio; - MIOBuffer *read_buffer; - IOBufferReader *buffer_start; - HttpTunnelType_t vc_type; + HttpTunnelConsumer *self_consumer = nullptr; + VConnection *vc = nullptr; + HttpProducerHandler vc_handler = nullptr; + VIO *read_vio = nullptr; + MIOBuffer *read_buffer = nullptr; + IOBufferReader *buffer_start = nullptr; + HttpTunnelType_t vc_type = HT_HTTP_SERVER; ChunkedHandler chunked_handler; - TunnelChunkingAction_t chunking_action; + TunnelChunkingAction_t chunking_action = TCA_PASSTHRU_DECHUNKED_CONTENT; - bool do_chunking; - bool do_dechunking; - bool do_chunked_passthru; + bool do_chunking = false; + bool do_dechunking = false; + bool do_chunked_passthru = false; - int64_t init_bytes_done; // bytes passed in buffer - int64_t nbytes; // total bytes (client's perspective) - int64_t ntodo; // what this vc needs to do - int64_t bytes_read; // total bytes read from the vc - int handler_state; // state used the handlers - int last_event; ///< Tracking for flow control restarts. + int64_t init_bytes_done = 0; // bytes passed in buffer + int64_t nbytes = 0; // total bytes (client's perspective) + int64_t ntodo = 0; // what this vc needs to do + int64_t bytes_read = 0; // total bytes read from the vc + int handler_state = 0; // state used the handlers + int last_event = 0; ///< Tracking for flow control restarts. - int num_consumers; + int num_consumers = 0; - bool alive; - bool read_success; + bool alive = false; + bool read_success = false; /// Flag and pointer for active flow control throttling. /// If this is set, it points at the source producer that is under flow control. /// If @c NULL then data flow is not being throttled. - HttpTunnelProducer *flow_control_source; - const char *name; + HttpTunnelProducer *flow_control_source = nullptr; + const char *name = nullptr; /** Get the largest number of bytes any consumer has not consumed. Use @a limit if you only need to check if the backlog is at least @a limit. @@ -261,9 +261,9 @@ class HttpTunnel : public Continuation // Default value for high and low water marks. static uint64_t const DEFAULT_WATER_MARK = 1 << 16; - uint64_t high_water; ///< Buffered data limit - throttle if more than this. - uint64_t low_water; ///< Unthrottle if less than this buffered. - bool enabled_p; ///< Flow control state (@c false means disabled). + uint64_t high_water; ///< Buffered data limit - throttle if more than this. + uint64_t low_water; ///< Unthrottle if less than this buffered. + bool enabled_p = false; ///< Flow control state (@c false means disabled). /// Default constructor. FlowControl(); @@ -575,4 +575,4 @@ HttpTunnelProducer::unthrottle() } } -inline HttpTunnel::FlowControl::FlowControl() : high_water(DEFAULT_WATER_MARK), low_water(DEFAULT_WATER_MARK), enabled_p(false) {} +inline HttpTunnel::FlowControl::FlowControl() : high_water(DEFAULT_WATER_MARK), low_water(DEFAULT_WATER_MARK) {} diff --git a/proxy/http/HttpUpdateSM.cc b/proxy/http/HttpUpdateSM.cc index efe7acbdcfd..fa89ac58663 100644 --- a/proxy/http/HttpUpdateSM.cc +++ b/proxy/http/HttpUpdateSM.cc @@ -42,7 +42,7 @@ ClassAllocator httpUpdateSMAllocator("httpUpdateSMAllocator"); Debug("http", "[%" PRId64 "] [%s, %s]", sm_id, #state_name, HttpDebugNames::get_event_name(event)); \ } -HttpUpdateSM::HttpUpdateSM() : cb_occured(false), cb_cont(nullptr), cb_action(), cb_event(HTTP_SCH_UPDATE_EVENT_ERROR) {} +HttpUpdateSM::HttpUpdateSM() : cb_action(), cb_event(HTTP_SCH_UPDATE_EVENT_ERROR) {} void HttpUpdateSM::destroy() diff --git a/proxy/http/HttpUpdateSM.h b/proxy/http/HttpUpdateSM.h index 6c298225225..a55258923ce 100644 --- a/proxy/http/HttpUpdateSM.h +++ b/proxy/http/HttpUpdateSM.h @@ -55,8 +55,8 @@ class HttpUpdateSM : public HttpSM Action *start_scheduled_update(Continuation *cont, HTTPHdr *req); // private: - bool cb_occured; - Continuation *cb_cont; + bool cb_occured = false; + Continuation *cb_cont = nullptr; Action cb_action; int cb_event; diff --git a/proxy/http/remap/AclFiltering.cc b/proxy/http/remap/AclFiltering.cc index 16bfe592333..64673694feb 100644 --- a/proxy/http/remap/AclFiltering.cc +++ b/proxy/http/remap/AclFiltering.cc @@ -51,8 +51,7 @@ acl_filter_rule::reset() internal = 0; } -acl_filter_rule::acl_filter_rule() - : next(nullptr), filter_name(nullptr), allow_flag(1), src_ip_valid(0), active_queue_flag(0), internal(0), argc(0) +acl_filter_rule::acl_filter_rule() : allow_flag(1), src_ip_valid(0), active_queue_flag(0), internal(0) { standard_method_lookup.resize(HTTP_WKSIDX_METHODS_CNT); ink_zero(argv); diff --git a/proxy/http/remap/AclFiltering.h b/proxy/http/remap/AclFiltering.h index b1ce7f824c1..ceba59a7c2c 100644 --- a/proxy/http/remap/AclFiltering.h +++ b/proxy/http/remap/AclFiltering.h @@ -67,16 +67,16 @@ class acl_filter_rule void reset(); public: - acl_filter_rule *next; - char *filter_name; // optional filter name - unsigned int allow_flag : 1, // action allow deny - src_ip_valid : 1, // src_ip range valid + acl_filter_rule *next = nullptr; + char *filter_name = nullptr; // optional filter name + unsigned int allow_flag : 1, // action allow deny + src_ip_valid : 1, // src_ip range valid in_ip_valid : 1, active_queue_flag : 1, // filter is in active state (used by .useflt directive) internal : 1; // filter internal HTTP requests // we need arguments as string array for directive processing - int argc; // argument counter (only for filter defs) + int argc = 0; // argument counter (only for filter defs) char *argv[ACL_FILTER_MAX_ARGV]; // argument strings (only for filter defs) // methods diff --git a/proxy/http/remap/RemapConfig.cc b/proxy/http/remap/RemapConfig.cc index 36440e39765..6b5a5e990f1 100644 --- a/proxy/http/remap/RemapConfig.cc +++ b/proxy/http/remap/RemapConfig.cc @@ -79,7 +79,7 @@ clear_xstr_array(char *v[], size_t vsize) } BUILD_TABLE_INFO::BUILD_TABLE_INFO() - : remap_optflg(0), paramc(0), argc(0), ip_allow_check_enabled_p(true), accept_check_p(true), rules_list(nullptr), rewrite(nullptr) + { memset(this->paramv, 0, sizeof(this->paramv)); memset(this->argv, 0, sizeof(this->argv)); diff --git a/proxy/http/remap/RemapConfig.h b/proxy/http/remap/RemapConfig.h index e0466406e1d..8cac3d821b3 100644 --- a/proxy/http/remap/RemapConfig.h +++ b/proxy/http/remap/RemapConfig.h @@ -46,16 +46,16 @@ struct BUILD_TABLE_INFO { BUILD_TABLE_INFO(); ~BUILD_TABLE_INFO(); - unsigned long remap_optflg; - int paramc; - int argc; + unsigned long remap_optflg = 0; + int paramc = 0; + int argc = 0; char *paramv[BUILD_TABLE_MAX_ARGS]; char *argv[BUILD_TABLE_MAX_ARGS]; - bool ip_allow_check_enabled_p; - bool accept_check_p; - acl_filter_rule *rules_list; // all rules defined in config files as .define_filter foobar @src_ip=..... - UrlRewrite *rewrite; // Pointer to the UrlRewrite object we are parsing for. + bool ip_allow_check_enabled_p = true; + bool accept_check_p = true; + acl_filter_rule *rules_list = nullptr; // all rules defined in config files as .define_filter foobar @src_ip=..... + UrlRewrite *rewrite = nullptr; // Pointer to the UrlRewrite object we are parsing for. // Clear the argument vector. void reset(); diff --git a/proxy/http/remap/RemapProcessor.h b/proxy/http/remap/RemapProcessor.h index f3665b8a50f..d5e5aced5b2 100644 --- a/proxy/http/remap/RemapProcessor.h +++ b/proxy/http/remap/RemapProcessor.h @@ -38,7 +38,7 @@ class RemapProcessor : public Processor { public: - RemapProcessor() : ET_REMAP(0), _use_separate_remap_thread(false) {} + RemapProcessor() {} ~RemapProcessor() override {} bool setup_for_remap(HttpTransact::State *s, UrlRewrite *table); bool finish_remap(HttpTransact::State *s, UrlRewrite *table); @@ -58,8 +58,8 @@ class RemapProcessor : public Processor } private: - EventType ET_REMAP; - bool _use_separate_remap_thread; + EventType ET_REMAP = 0; + bool _use_separate_remap_thread = false; }; /** diff --git a/proxy/http/remap/UrlMapping.h b/proxy/http/remap/UrlMapping.h index 96d2109541b..1b55e52b72f 100644 --- a/proxy/http/remap/UrlMapping.h +++ b/proxy/http/remap/UrlMapping.h @@ -55,7 +55,7 @@ class referer_info class redirect_tag_str { public: - redirect_tag_str() : next(nullptr), chunk_str(nullptr), type(0) {} + redirect_tag_str() {} ~redirect_tag_str() { type = 0; @@ -65,9 +65,9 @@ class redirect_tag_str } } - redirect_tag_str *next; - char *chunk_str; - char type; /* s - string, r - referer, t - url_to, f - url_from, o - origin url */ + redirect_tag_str *next = nullptr; + char *chunk_str = nullptr; + char type = 0; /* s - string, r - referer, t - url_to, f - url_from, o - origin url */ static redirect_tag_str *parse_format_redirect_url(char *url); }; @@ -134,8 +134,8 @@ class url_mapping class UrlMappingContainer { public: - UrlMappingContainer() : _mapping(nullptr), _toURLPtr(nullptr), _heap(nullptr) {} - explicit UrlMappingContainer(HdrHeap *heap) : _mapping(nullptr), _toURLPtr(nullptr), _heap(heap) {} + UrlMappingContainer() {} + explicit UrlMappingContainer(HdrHeap *heap) : _heap(heap) {} ~UrlMappingContainer() { deleteToURL(); } URL * getToURL() const @@ -200,8 +200,8 @@ class UrlMappingContainer UrlMappingContainer &operator=(const UrlMappingContainer &rhs) = delete; private: - url_mapping *_mapping; - URL *_toURLPtr; + url_mapping *_mapping = nullptr; + URL *_toURLPtr = nullptr; URL _toURL; - HdrHeap *_heap; + HdrHeap *_heap = nullptr; }; diff --git a/proxy/http2/HPACK.h b/proxy/http2/HPACK.h index 9d0f9f56295..9f93d4e4fba 100644 --- a/proxy/http2/HPACK.h +++ b/proxy/http2/HPACK.h @@ -55,10 +55,10 @@ enum class HpackMatch { // Result of looking for a header field in IndexingTable struct HpackLookupResult { - HpackLookupResult() : index(0), index_type(HpackIndex::NONE), match_type(HpackMatch::NONE) {} - int index; - HpackIndex index_type; - HpackMatch match_type; + HpackLookupResult() {} + int index = 0; + HpackIndex index_type = HpackIndex::NONE; + HpackMatch match_type = HpackMatch::NONE; }; class MIMEFieldWrapper diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index 9289b4090c0..e705d74705a 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -259,28 +259,25 @@ struct Http2SettingsParameter { // [RFC 7540] 6.3 PRIORITY Format struct Http2Priority { - Http2Priority() - : exclusive_flag(false), weight(HTTP2_PRIORITY_DEFAULT_WEIGHT), stream_dependency(HTTP2_PRIORITY_DEFAULT_STREAM_DEPENDENCY) - { - } + Http2Priority() : weight(HTTP2_PRIORITY_DEFAULT_WEIGHT), stream_dependency(HTTP2_PRIORITY_DEFAULT_STREAM_DEPENDENCY) {} - bool exclusive_flag; + bool exclusive_flag = false; uint8_t weight; uint32_t stream_dependency; }; // [RFC 7540] 6.2 HEADERS Format struct Http2HeadersParameter { - Http2HeadersParameter() : pad_length(0) {} - uint8_t pad_length; + Http2HeadersParameter() {} + uint8_t pad_length = 0; Http2Priority priority; }; // [RFC 7540] 6.8 GOAWAY Format struct Http2Goaway { - Http2Goaway() : last_streamid(0), error_code(Http2ErrorCode::HTTP2_ERROR_NO_ERROR) {} - Http2StreamId last_streamid; - Http2ErrorCode error_code; + Http2Goaway() {} + Http2StreamId last_streamid = 0; + Http2ErrorCode error_code = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; // NOTE: we don't (de)serialize the variable length debug data at this layer // because there's @@ -296,9 +293,9 @@ struct Http2RstStream { // [RFC 7540] 6.6 PUSH_PROMISE Format struct Http2PushPromise { - Http2PushPromise() : pad_length(0), promised_streamid(0) {} - uint8_t pad_length; - Http2StreamId promised_streamid; + Http2PushPromise() {} + uint8_t pad_length = 0; + Http2StreamId promised_streamid = 0; }; static inline bool diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 5f22c7bfe48..23b0f53e820 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -53,7 +53,7 @@ size_t const HTTP2_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ_BUFFE // To support Upgrade: h2c struct Http2UpgradeContext { - Http2UpgradeContext() : req_header(nullptr) {} + Http2UpgradeContext() {} ~Http2UpgradeContext() { if (req_header) { @@ -63,7 +63,7 @@ struct Http2UpgradeContext { } // Modified request header - HTTPHdr *req_header; + HTTPHdr *req_header = nullptr; // Decoded HTTP2-Settings Header Field Http2ConnectionSettings client_settings; diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc index 4437266f120..e952605b4e8 100644 --- a/proxy/logging/LogConfig.cc +++ b/proxy/logging/LogConfig.cc @@ -284,18 +284,8 @@ LogConfig::read_configuration_variables() -------------------------------------------------------------------------*/ // TODO: Is UINT_MAX here really correct? -LogConfig::LogConfig() - : initialized(false), - reconfiguration_needed(false), - logging_space_exhausted(false), - m_space_used(0), - m_partition_space_left((int64_t)UINT_MAX), - m_log_collation_accept(nullptr), - m_disk_full(false), - m_disk_low(false), - m_partition_full(false), - m_partition_low(false), - m_log_directory_inaccessible(false) +LogConfig::LogConfig() : m_partition_space_left((int64_t)UINT_MAX) + { // Setup the default values for all LogConfig public variables so that // a LogConfig object is valid upon return from the constructor even diff --git a/proxy/logging/LogConfig.h b/proxy/logging/LogConfig.h index fc413902783..fdf3def9799 100644 --- a/proxy/logging/LogConfig.h +++ b/proxy/logging/LogConfig.h @@ -232,10 +232,10 @@ class LogConfig : public ConfigInfo } public: - bool initialized; - bool reconfiguration_needed; - bool logging_space_exhausted; - int64_t m_space_used; + bool initialized = false; + bool reconfiguration_needed = false; + bool logging_space_exhausted = false; + int64_t m_space_used = 0; int64_t m_partition_space_left; bool roll_log_files_now; // signal that files must be rolled @@ -289,13 +289,13 @@ class LogConfig : public ConfigInfo // bool use_orphan_log_space_value; - LogCollationAccept *m_log_collation_accept; + LogCollationAccept *m_log_collation_accept = nullptr; - bool m_disk_full; - bool m_disk_low; - bool m_partition_full; - bool m_partition_low; - bool m_log_directory_inaccessible; + bool m_disk_full = false; + bool m_disk_low = false; + bool m_partition_full = false; + bool m_partition_low = false; + bool m_log_directory_inaccessible = false; // noncopyable // -- member functions not allowed -- diff --git a/proxy/logging/LogField.cc b/proxy/logging/LogField.cc index 0b1ce143580..563638d55e3 100644 --- a/proxy/logging/LogField.cc +++ b/proxy/logging/LogField.cc @@ -701,7 +701,7 @@ LogField::fieldlist_contains_aggregates(const char *fieldlist) heap with "new" and that each element is on at most ONE list. To enforce this, items are copied by default, using the copy ctor. -------------------------------------------------------------------------*/ -LogFieldList::LogFieldList() : m_marshal_len(0) {} +LogFieldList::LogFieldList() {} LogFieldList::~LogFieldList() { diff --git a/proxy/logging/LogField.h b/proxy/logging/LogField.h index 6ca2a64c18d..94823e4d8f3 100644 --- a/proxy/logging/LogField.h +++ b/proxy/logging/LogField.h @@ -264,7 +264,7 @@ class LogFieldList LogFieldList &operator=(const LogFieldList &rhs) = delete; private: - unsigned m_marshal_len; + unsigned m_marshal_len = 0; Queue m_field_list; std::string _badSymbols; }; diff --git a/proxy/logging/LogFieldAliasMap.h b/proxy/logging/LogFieldAliasMap.h index 9f69cc36a50..673cdf749a6 100644 --- a/proxy/logging/LogFieldAliasMap.h +++ b/proxy/logging/LogFieldAliasMap.h @@ -103,11 +103,11 @@ table->init(3, 1, "one", 2, "two", 7, "seven") *****************************************************************************/ struct LogFieldAliasTableEntry { - bool valid; // entry in table is valid - char *name; // the string equivalent - size_t length; // the length of the string + bool valid = false; // entry in table is valid + char *name = nullptr; // the string equivalent + size_t length = 0; // the length of the string - LogFieldAliasTableEntry() : valid(false), name(nullptr), length(0) {} + LogFieldAliasTableEntry() {} ~LogFieldAliasTableEntry() { if (name) { @@ -119,13 +119,13 @@ struct LogFieldAliasTableEntry { class LogFieldAliasTable : public LogFieldAliasMap { private: - IntType m_min; // minimum numeric value - IntType m_max; // maximum numeric value - IntType m_entries; // number of entries in table - LogFieldAliasTableEntry *m_table; // array of table entries + IntType m_min = 0; // minimum numeric value + IntType m_max = 0; // maximum numeric value + IntType m_entries = 0; // number of entries in table + LogFieldAliasTableEntry *m_table = nullptr; // array of table entries public: - LogFieldAliasTable() : m_min(0), m_max(0), m_entries(0), m_table(nullptr) {} + LogFieldAliasTable() {} ~LogFieldAliasTable() override { delete[] m_table; } void init(size_t numPairs, ...); diff --git a/proxy/logging/LogFilter.cc b/proxy/logging/LogFilter.cc index e863194cef2..900268f06bf 100644 --- a/proxy/logging/LogFilter.cc +++ b/proxy/logging/LogFilter.cc @@ -927,7 +927,7 @@ filters_are_equal(LogFilter *filt1, LogFilter *filt2) add() function is overloaded for each sub-type of LogFilter. -------------------------------------------------------------------------*/ -LogFilterList::LogFilterList() : m_does_conjunction(true) {} +LogFilterList::LogFilterList() {} /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/logging/LogFilter.h b/proxy/logging/LogFilter.h index 7d9d6a115e5..94d5ff7e7ba 100644 --- a/proxy/logging/LogFilter.h +++ b/proxy/logging/LogFilter.h @@ -301,7 +301,7 @@ class LogFilterList private: Queue m_filter_list; - bool m_does_conjunction; + bool m_does_conjunction = true; // If m_does_conjunction = true // toss_this_entry returns true // if ANY filter tosses entry away. diff --git a/proxy/logging/LogObject.h b/proxy/logging/LogObject.h index 9c59b849b70..761530c8c16 100644 --- a/proxy/logging/LogObject.h +++ b/proxy/logging/LogObject.h @@ -66,10 +66,10 @@ class LogBufferManager { private: ASLL(LogBuffer, write_link) write_list; - int _num_flush_buffers; + int _num_flush_buffers = 0; public: - LogBufferManager() : _num_flush_buffers(0) {} + LogBufferManager() {} inline void add_to_flush_queue(LogBuffer *buffer) { diff --git a/proxy/logging/LogSock.cc b/proxy/logging/LogSock.cc index 4a37d0c8c6f..7746b9668dc 100644 --- a/proxy/logging/LogSock.cc +++ b/proxy/logging/LogSock.cc @@ -38,7 +38,7 @@ static const int LS_PROTOCOL = 0; first entry of the table (index 0) to be the port on which new connections are accepted. */ -LogSock::LogSock(int max_connects) : ct((ConnectTable *)nullptr), m_accept_connections(false), m_max_connections(max_connects + 1) +LogSock::LogSock(int max_connects) : ct((ConnectTable *)nullptr), m_max_connections(max_connects + 1) { ink_assert(m_max_connections > 0); diff --git a/proxy/logging/LogSock.h b/proxy/logging/LogSock.h index eda5d1a126e..d1ad1e9389c 100644 --- a/proxy/logging/LogSock.h +++ b/proxy/logging/LogSock.h @@ -123,6 +123,6 @@ class LogSock ConnectTable *ct; // list of all connections; index 0 is // the accept port. - bool m_accept_connections; // do we accept new connections? - int m_max_connections; // max size of all tables + bool m_accept_connections = false; // do we accept new connections? + int m_max_connections; // max size of all tables }; diff --git a/src/traffic_logstats/logstats.cc b/src/traffic_logstats/logstats.cc index 52a5390ffd7..2cb72233b61 100644 --- a/src/traffic_logstats/logstats.cc +++ b/src/traffic_logstats/logstats.cc @@ -599,39 +599,25 @@ struct CommandLineArgs { char log_file[1024]; char origin_file[1024]; char origin_list[MAX_ORIG_STRING]; - int max_origins; + int max_origins = 0; char state_tag[1024]; - int64_t min_hits; - int max_age; + int64_t min_hits = 0; + int max_age = 0; int line_len; - int incremental; // Do an incremental run - int tail; // Tail the log file - int summary; // Summary only - int json; // JSON output - int cgi; // CGI output (typically with json) - int urls; // Produce JSON output of URL stats, arg is LRU size - int show_urls; // Max URLs to show - int as_object; // Show the URL stats as a single JSON object (not array) - int concise; // Eliminate metrics that can be inferred by other values - int report_per_user; // A flag to aggregate and report stats per user instead of per host if 'true' (default 'false') - int no_format_check; // A flag to skip the log format check if any of the fields is not a standard squid log format field. - - CommandLineArgs() - : max_origins(0), - min_hits(0), - max_age(0), - line_len(DEFAULT_LINE_LEN), - incremental(0), - tail(0), - summary(0), - json(0), - cgi(0), - urls(0), - show_urls(0), - as_object(0), - concise(0), - report_per_user(0), - no_format_check(0) + int incremental = 0; // Do an incremental run + int tail = 0; // Tail the log file + int summary = 0; // Summary only + int json = 0; // JSON output + int cgi = 0; // CGI output (typically with json) + int urls = 0; // Produce JSON output of URL stats, arg is LRU size + int show_urls = 0; // Max URLs to show + int as_object = 0; // Show the URL stats as a single JSON object (not array) + int concise = 0; // Eliminate metrics that can be inferred by other values + int report_per_user = 0; // A flag to aggregate and report stats per user instead of per host if 'true' (default 'false') + int no_format_check = 0; // A flag to skip the log format check if any of the fields is not a standard squid log format field. + + CommandLineArgs() : line_len(DEFAULT_LINE_LEN) + { log_file[0] = '\0'; origin_file[0] = '\0'; @@ -731,10 +717,10 @@ enum ExitLevel { }; struct ExitStatus { - ExitLevel level; + ExitLevel level = EXIT_OK; char notice[1024]; - ExitStatus() : level(EXIT_OK) { memset(notice, 0, sizeof(notice)); } + ExitStatus() { memset(notice, 0, sizeof(notice)); } void set(ExitLevel l, const char *n = nullptr) { diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index c35f00c9f90..092745010dc 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -399,7 +399,7 @@ class DiagsLogContinuation : public Continuation class MemoryLimit : public Continuation { public: - MemoryLimit() : Continuation(new_ProxyMutex()), _memory_limit(0) + MemoryLimit() : Continuation(new_ProxyMutex()) { memset(&_usage, 0, sizeof(_usage)); SET_HANDLER(&MemoryLimit::periodic); @@ -449,7 +449,7 @@ class MemoryLimit : public Continuation } private: - int64_t _memory_limit; + int64_t _memory_limit = 0; struct rusage _usage; }; @@ -1173,18 +1173,18 @@ struct ShowStats : public Continuation { #ifdef ENABLE_TIME_TRACE FILE *fp; #endif - int cycle; - int64_t last_cc; - int64_t last_rb; - int64_t last_w; - int64_t last_r; - int64_t last_wb; - int64_t last_nrb; - int64_t last_nw; - int64_t last_nr; - int64_t last_nwb; - int64_t last_p; - int64_t last_o; + int cycle = 0; + int64_t last_cc = 0; + int64_t last_rb = 0; + int64_t last_w = 0; + int64_t last_r = 0; + int64_t last_wb = 0; + int64_t last_nrb = 0; + int64_t last_nw = 0; + int64_t last_nr = 0; + int64_t last_nwb = 0; + int64_t last_p = 0; + int64_t last_o = 0; int mainEvent(int event, Event *e) { @@ -1284,20 +1284,8 @@ struct ShowStats : public Continuation { #endif return EVENT_CONT; } - ShowStats() - : Continuation(nullptr), - cycle(0), - last_cc(0), - last_rb(0), - last_w(0), - last_r(0), - last_wb(0), - last_nrb(0), - last_nw(0), - last_nr(0), - last_nwb(0), - last_p(0), - last_o(0) + ShowStats() : Continuation(nullptr) + { SET_HANDLER(&ShowStats::mainEvent); #ifdef ENABLE_TIME_TRACE @@ -1352,9 +1340,9 @@ init_http_header() #if TS_HAS_TESTS struct RegressionCont : public Continuation { - int initialized; - int waits; - int started; + int initialized = 0; + int waits = 0; + int started = 0; int mainEvent(int event, Event *e) @@ -1383,10 +1371,7 @@ struct RegressionCont : public Continuation { return EVENT_CONT; } - RegressionCont() : Continuation(new_ProxyMutex()), initialized(0), waits(0), started(0) - { - SET_HANDLER(&RegressionCont::mainEvent); - } + RegressionCont() : Continuation(new_ProxyMutex()) { SET_HANDLER(&RegressionCont::mainEvent); } }; static void diff --git a/src/tscore/BaseLogFile.cc b/src/tscore/BaseLogFile.cc index 64968153605..921a1608c84 100644 --- a/src/tscore/BaseLogFile.cc +++ b/src/tscore/BaseLogFile.cc @@ -46,15 +46,13 @@ BaseLogFile::BaseLogFile(const char *name, uint64_t sig) : m_name(ats_strdup(nam * This copy constructor creates a BaseLogFile based on a given copy. */ BaseLogFile::BaseLogFile(const BaseLogFile ©) - : m_fp(nullptr), - m_start_time(copy.m_start_time), - m_end_time(0L), - m_bytes_written(0), + : m_start_time(copy.m_start_time), + m_name(ats_strdup(copy.m_name)), m_hostname(ats_strdup(copy.m_hostname)), - m_is_regfile(false), + m_is_init(copy.m_is_init), - m_meta_info(nullptr), + m_signature(copy.m_signature), m_has_signature(copy.m_has_signature) { diff --git a/src/tscore/HashMD5.cc b/src/tscore/HashMD5.cc index 7f1dedc1b80..f8b4ff5642e 100644 --- a/src/tscore/HashMD5.cc +++ b/src/tscore/HashMD5.cc @@ -23,7 +23,7 @@ #include "tscore/ink_config.h" #include "tscore/HashMD5.h" -ATSHashMD5::ATSHashMD5() : md_len(0), finalized(false) +ATSHashMD5::ATSHashMD5() { ctx = EVP_MD_CTX_new(); int ret = EVP_DigestInit_ex(ctx, EVP_md5(), nullptr); diff --git a/src/tscore/ink_cap.cc b/src/tscore/ink_cap.cc index f33b732e972..7fd3289eb82 100644 --- a/src/tscore/ink_cap.cc +++ b/src/tscore/ink_cap.cc @@ -486,8 +486,7 @@ ElevateAccess::releasePrivilege() #endif ElevateAccess::ElevateAccess(unsigned lvl) - : elevated(false), - saved_uid(geteuid()), + : saved_uid(geteuid()), level(lvl) #if TS_USE_POSIX_CAP , diff --git a/src/tscore/ink_resource.cc b/src/tscore/ink_resource.cc index 08ba118983b..c557ad29e0a 100644 --- a/src/tscore/ink_resource.cc +++ b/src/tscore/ink_resource.cc @@ -39,7 +39,7 @@ ink_mutex ResourceTracker::resourceLock = PTHREAD_MUTEX_INITIALIZER; class Resource { public: - Resource() : _incrementCount(0), _decrementCount(0), _value(0), _symbol(nullptr) { _name[0] = '\0'; } + Resource() { _name[0] = '\0'; } void increment(const int64_t size); int64_t getValue() const @@ -86,10 +86,10 @@ class Resource } private: - int64_t _incrementCount; - int64_t _decrementCount; - int64_t _value; - const void *_symbol; + int64_t _incrementCount = 0; + int64_t _decrementCount = 0; + int64_t _value = 0; + const void *_symbol = nullptr; char _name[128]; }; diff --git a/src/tscpp/api/Headers.cc b/src/tscpp/api/Headers.cc index 174588391ae..204d7a597a8 100644 --- a/src/tscpp/api/Headers.cc +++ b/src/tscpp/api/Headers.cc @@ -101,11 +101,11 @@ HeaderFieldName::operator!=(const std::string &field_name) * @private */ struct HeaderFieldValueIteratorState : noncopyable { - TSMBuffer hdr_buf_; - TSMLoc hdr_loc_; - TSMLoc field_loc_; - int index_; - HeaderFieldValueIteratorState() : hdr_buf_(nullptr), hdr_loc_(nullptr), field_loc_(nullptr), index_(0) {} + TSMBuffer hdr_buf_ = nullptr; + TSMLoc hdr_loc_ = nullptr; + TSMLoc field_loc_ = nullptr; + int index_ = 0; + HeaderFieldValueIteratorState() {} void reset(TSMBuffer bufp, TSMLoc hdr_loc, TSMLoc field_loc, int index) { diff --git a/src/tscpp/api/InterceptPlugin.cc b/src/tscpp/api/InterceptPlugin.cc index 8ccc5a2ac4c..db0bd00a700 100644 --- a/src/tscpp/api/InterceptPlugin.cc +++ b/src/tscpp/api/InterceptPlugin.cc @@ -45,10 +45,10 @@ struct InterceptPlugin::State { TSVConn net_vc_ = nullptr; struct IoHandle { - TSVIO vio_; - TSIOBuffer buffer_; - TSIOBufferReader reader_; - IoHandle() : vio_(nullptr), buffer_(nullptr), reader_(nullptr){}; + TSVIO vio_ = nullptr; + TSIOBuffer buffer_ = nullptr; + TSIOBufferReader reader_ = nullptr; + IoHandle(){}; ~IoHandle() { if (reader_) { diff --git a/src/tscpp/api/Logger.cc b/src/tscpp/api/Logger.cc index da94f313baa..7d44a0e3d4f 100644 --- a/src/tscpp/api/Logger.cc +++ b/src/tscpp/api/Logger.cc @@ -42,22 +42,17 @@ using atscppapi::Logger; */ struct atscppapi::LoggerState : noncopyable { std::string filename_; - bool add_timestamp_; - bool rename_file_; - Logger::LogLevel level_; - bool rolling_enabled_; - int rolling_interval_seconds_; - TSTextLogObject text_log_obj_; - bool initialized_; + bool add_timestamp_ = false; + bool rename_file_ = false; + Logger::LogLevel level_ = Logger::LOG_LEVEL_NO_LOG; + bool rolling_enabled_ = false; + int rolling_interval_seconds_ = -1; + TSTextLogObject text_log_obj_ = nullptr; + bool initialized_ = false; LoggerState() - : add_timestamp_(false), - rename_file_(false), - level_(Logger::LOG_LEVEL_NO_LOG), - rolling_enabled_(false), - rolling_interval_seconds_(-1), - text_log_obj_(nullptr), - initialized_(false){}; + + {}; ~LoggerState(){}; }; diff --git a/src/tscpp/api/Request.cc b/src/tscpp/api/Request.cc index eec8244354a..74c45c744d8 100644 --- a/src/tscpp/api/Request.cc +++ b/src/tscpp/api/Request.cc @@ -32,24 +32,16 @@ using std::string; * @private */ struct atscppapi::RequestState : noncopyable { - TSMBuffer hdr_buf_; - TSMLoc hdr_loc_; - TSMLoc url_loc_; + TSMBuffer hdr_buf_ = nullptr; + TSMLoc hdr_loc_ = nullptr; + TSMLoc url_loc_ = nullptr; Url url_; Headers headers_; /* method and version are stored here for the case of an unbound request */ - HttpMethod method_; - HttpVersion version_; - bool destroy_buf_; - RequestState() - : hdr_buf_(nullptr), - hdr_loc_(nullptr), - url_loc_(nullptr), - method_(HTTP_METHOD_UNKNOWN), - version_(HTTP_VERSION_UNKNOWN), - destroy_buf_(false) - { - } + HttpMethod method_ = HTTP_METHOD_UNKNOWN; + HttpVersion version_ = HTTP_VERSION_UNKNOWN; + bool destroy_buf_ = false; + RequestState() {} }; Request::Request() diff --git a/src/tscpp/api/Response.cc b/src/tscpp/api/Response.cc index bc2f0017059..7624e8861c2 100644 --- a/src/tscpp/api/Response.cc +++ b/src/tscpp/api/Response.cc @@ -32,10 +32,10 @@ namespace atscppapi * @private */ struct ResponseState : noncopyable { - TSMBuffer hdr_buf_; - TSMLoc hdr_loc_; + TSMBuffer hdr_buf_ = nullptr; + TSMLoc hdr_loc_ = nullptr; Headers headers_; - ResponseState() : hdr_buf_(nullptr), hdr_loc_(nullptr) {} + ResponseState() {} }; } // namespace atscppapi diff --git a/src/tscpp/api/Stat.cc b/src/tscpp/api/Stat.cc index b1422fa796d..0a28230cf3c 100644 --- a/src/tscpp/api/Stat.cc +++ b/src/tscpp/api/Stat.cc @@ -28,7 +28,7 @@ using namespace atscppapi; using std::string; -Stat::Stat() : stat_id_(TS_ERROR) +Stat::Stat() { // ATS Guarantees that stat ids will always be > 0. So we can use stat_id_ > 0 to // verify that this stat has been properly initialized. From e4470b0861898de1ce961dcb808d908afcaa174d Mon Sep 17 00:00:00 2001 From: Aaron Canary Date: Tue, 26 Mar 2019 11:10:52 -0500 Subject: [PATCH 387/526] HostDB: cancel timeout on free --- iocore/hostdb/HostDB.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index e4a1427be68..d32f87a5f97 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -124,6 +124,9 @@ hostdb_cont_free(HostDBContinuation *cont) if (cont->pending_action) { cont->pending_action->cancel(); } + if (cont->timeout) { + cont->timeout->cancel(); + } cont->mutex = nullptr; cont->action.mutex = nullptr; hostDBContAllocator.free(cont); From 5b24c07f7d0b2c52ffeb525712902aa2430347a4 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 27 Mar 2019 14:07:08 +0000 Subject: [PATCH 388/526] Fix stale heap triggering assert in TSHttpTxnCachedRespGet --- src/traffic_server/InkAPI.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 0454459159f..fb1e0d0ac00 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -5037,9 +5037,10 @@ TSHttpTxnCachedRespGet(TSHttpTxn txnp, TSMBuffer *bufp, TSMLoc *obj) HdrHeapSDKHandle **handle = &(sm->t_state.cache_resp_hdr_heap_handle); if (*handle == nullptr) { - *handle = (HdrHeapSDKHandle *)sm->t_state.arena.alloc(sizeof(HdrHeapSDKHandle)); - (*handle)->m_heap = cached_hdr->m_heap; + *handle = (HdrHeapSDKHandle *)sm->t_state.arena.alloc(sizeof(HdrHeapSDKHandle)); } + // Always reset the m_heap to make sure the heap is not stale + (*handle)->m_heap = cached_hdr->m_heap; *(reinterpret_cast(bufp)) = *handle; *obj = reinterpret_cast(cached_hdr->m_http); From 0d333ed3b3c08b7270b90a835f721603fa40ddf4 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 27 Mar 2019 10:33:35 +0100 Subject: [PATCH 389/526] Allows logging of negative values Before, all negative values would be logged as exactly "0". --- proxy/logging/LogAccess.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index a2df5f657c5..4fc737b19d5 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -511,23 +511,27 @@ LogAccess::unmarshal_itoa(int64_t val, char *dest, int field_width, char leading { ink_assert(dest != nullptr); - char *p = dest; + char *p = dest; + bool negative = false; - if (val <= 0) { - *p-- = '0'; - while (dest - p < field_width) { - *p-- = leading_char; - } - return (int)(dest - p); + if (val < 0) { + negative = true; + val = -val; } - while (val) { + do { *p-- = '0' + (val % 10); val /= 10; - } + } while (val); + while (dest - p < field_width) { *p-- = leading_char; } + + if (negative) { + *p-- = '-'; + } + return (int)(dest - p); } From e92a11774c60992130fc044f6d047c9ab0bf80ce Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Thu, 28 Mar 2019 15:59:56 +0100 Subject: [PATCH 390/526] Remove quotes from records.config documentation examples Quotes are not needed for these values This is a follow up to 405ce9f5791746a5b91e90db3d7649aa8ba2b2e5 --- doc/admin-guide/files/records.config.en.rst | 2 +- doc/admin-guide/monitoring/alarms.en.rst | 2 +- doc/admin-guide/security/index.en.rst | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index d87e8425da5..44aedd521d7 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -2712,7 +2712,7 @@ HostDB Set the interval (in seconds) in which to re-query DNS regardless of TTL status. -.. ts:cv:: CONFIG proxy.config.hostdb.filename STRING "host.db" +.. ts:cv:: CONFIG proxy.config.hostdb.filename STRING host.db The filename to persist hostdb to on disk. diff --git a/doc/admin-guide/monitoring/alarms.en.rst b/doc/admin-guide/monitoring/alarms.en.rst index 0749f352fca..c4a23ccf3aa 100644 --- a/doc/admin-guide/monitoring/alarms.en.rst +++ b/doc/admin-guide/monitoring/alarms.en.rst @@ -35,7 +35,7 @@ occurs, follow the steps below: #. Set :ts:cv:`proxy.config.alarm_email` in :file:`records.config` to the email address you want to receive alarm notifications. :: - CONFIG proxy.config.alarm_email STRING "alerts@example.com" + CONFIG proxy.config.alarm_email STRING alerts@example.com #. Run the command :option:`traffic_ctl config reload` to apply the configuration changes. diff --git a/doc/admin-guide/security/index.en.rst b/doc/admin-guide/security/index.en.rst index 29d9da1a8e1..37ed09660cb 100644 --- a/doc/admin-guide/security/index.en.rst +++ b/doc/admin-guide/security/index.en.rst @@ -146,7 +146,7 @@ Client/Traffic Server connections, you must do the following: The list of acceptable CA signers is configured with :ts:cv:`proxy.config.ssl.CA.cert.path` in :file:`records.config`. :: - CONFIG proxy.config.ssl.CA.cert.path STRING "/opt/CA/certs/private-ca.pem" + CONFIG proxy.config.ssl.CA.cert.path STRING /opt/CA/certs/private-ca.pem #. Run the command :option:`traffic_ctl server restart` to restart Traffic Server. @@ -208,16 +208,16 @@ and origin server connections, you must do the following: :file:`records.config` in the setting :ts:cv:`proxy.config.ssl.client.cert.path` and :ts:cv:`proxy.config.ssl.client.cert.filename`. :: - CONFIG proxy.config.ssl.client.cert.path STRING "/opt/ts/etc/ssl/certs/" - CONFIG proxy.config.ssl.client.cert.filename STRING "client.pem" + CONFIG proxy.config.ssl.client.cert.path STRING /opt/ts/etc/ssl/certs/ + CONFIG proxy.config.ssl.client.cert.filename STRING client.pem You must also provide the paths to the private key for this certificate, unless the key is contained within the same file as the certificate, using :ts:cv:`proxy.config.ssl.client.private_key.path` and :ts:cv:`proxy.config.ssl.client.private_key.filename`. :: - CONFIG proxy.config.ssl.client.private_key.path STRING "/opt/ts/etc/ssl/keys/" - CONFIG proxy.config.ssl.client.private_key.filename STRING "client.pem" + CONFIG proxy.config.ssl.client.private_key.path STRING /opt/ts/etc/ssl/keys/ + CONFIG proxy.config.ssl.client.private_key.filename STRING client.pem #. Enable or disable, per your security policy, server SSL certificate verification using :ts:cv:`proxy.config.ssl.client.verify.server.policy` in @@ -230,8 +230,8 @@ and origin server connections, you must do the following: :ts:cv:`proxy.config.ssl.client.CA.cert.path` and :ts:cv:`proxy.config.ssl.client.CA.cert.filename`. :: - CONFIG proxy.config.ssl.client.CA.cert.path STRING "/opt/ts/etc/ssl/certs/" - CONFIG proxy.config.ssl.client.CA.cert.filename STRING "CAs.pem" + CONFIG proxy.config.ssl.client.CA.cert.path STRING /opt/ts/etc/ssl/certs/ + CONFIG proxy.config.ssl.client.CA.cert.filename STRING CAs.pem #. Run the command :option:`traffic_ctl server restart` to restart Traffic Server. From c570e0afc8bfbe2e99be28ca94bdb415c485c979 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Fri, 22 Mar 2019 22:53:48 +0000 Subject: [PATCH 391/526] autest for slice plugin --- plugins/experimental/slice/Makefile.tsxs | 24 ++- plugins/experimental/slice/README.md | 2 +- plugins/experimental/slice/client.cc | 5 + plugins/experimental/slice/server.cc | 2 +- tests/gold_tests/pluginTest/slice/curlsort.sh | 33 +++ .../slice/gold/slice_200.stderr.gold | 1 + .../slice/gold/slice_200.stdout.gold | 8 + .../slice/gold/slice_206.stderr.gold | 1 + .../slice/gold/slice_206.stdout.gold | 9 + .../slice/gold/slice_first.stderr.gold | 1 + .../slice/gold/slice_first.stdout.gold | 9 + .../slice/gold/slice_last.stderr.gold | 1 + .../slice/gold/slice_last.stdout.gold | 9 + .../slice/gold/slice_mid.stderr.gold | 1 + .../slice/gold/slice_mid.stdout.gold | 9 + .../gold_tests/pluginTest/slice/slice.test.py | 188 ++++++++++++++++++ 16 files changed, 290 insertions(+), 13 deletions(-) create mode 100755 tests/gold_tests/pluginTest/slice/curlsort.sh create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/slice.test.py diff --git a/plugins/experimental/slice/Makefile.tsxs b/plugins/experimental/slice/Makefile.tsxs index b3b5d4f014d..0992d8cb27e 100644 --- a/plugins/experimental/slice/Makefile.tsxs +++ b/plugins/experimental/slice/Makefile.tsxs @@ -14,9 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -all: slice +TSXS?=tsxs +PLUGIN=slice -slice_la_SOURCES = \ +all: $(PLUGIN).so + +SOURCES = \ Config.cc \ ContentRange.cc \ Data.cc \ @@ -29,7 +32,7 @@ slice_la_SOURCES = \ slice.cc \ transfer.cc \ -slice_la_HEADERS = \ +HEADERS = \ Config.h \ ContentRange.h \ Data.h \ @@ -43,22 +46,21 @@ slice_la_HEADERS = \ slice.h \ transfer.h \ -slice: $(slice_la_SOURCES) $(slice_la_HEADERS) - tsxs -v -o slice.so $(slice_la_SOURCES) +$(PLUGIN).so: $(SOURCES) $(HEADERS) + $(TSXS) -v -o $(PLUGIN).so $(SOURCES) -install: slice $(slice_la_SOURCES) $(slice_la_HEADERS) - tsxs -v -o slice.so -i +install: all + $(TSXS) -v -o $(PLUGIN).so -i -CXX = c++ -std=c++11 -#CXXFLAGS = -pipe -Wall -Wno-deprecated-declarations -Qunused-arguments -Wextra -Wno-ignored-qualifiers -Wno-unused-parameter -O3 -fno-strict-aliasing -Wno-invalid-offsetof -mcx16 -CXXFLAGS = -pipe -Wall -Wno-deprecated-declarations -Wextra -Wno-ignored-qualifiers -Wno-unused-parameter -O3 -fno-strict-aliasing -Wno-invalid-offsetof -mcx16 TSINCLUDE = $(shell tsxs -q INCLUDEDIR) +TSCXX = $(shell tsxs -q CXX) +TSCXXFLAGS = $(shell tsxs -q CXXFLAGS) #PREFIX = $(shell tsxs -q PREFIX) #LIBS = -L$(PREFIX)/lib -latscppapi #LIBS = $(PREFIX)/lib/libtsutil.la slice_test: slice_test.cc ContentRange.cc Range.cc - $(CXX) -o $@ $^ $(CXXFLAGS) -I$(TSINCLUDE) -DUNITTEST + $(TSCXX) -o $@ $^ $(TSCXXFLAGS) -I$(TSINCLUDE) -DUNITTEST clean: rm -fv *.lo *.so diff --git a/plugins/experimental/slice/README.md b/plugins/experimental/slice/README.md index 731fd579241..dc56997ce56 100644 --- a/plugins/experimental/slice/README.md +++ b/plugins/experimental/slice/README.md @@ -31,7 +31,7 @@ cache_range_requests.so **Note**: cache_range_requests **MUST** follow slice.so Put these plugins at the end of the plugin list **Note**: blockbytes is defined in bytes. 1048576 (1MB) is the default. -For testing purposes an unchecked value of "blockbytestest" is also available. +For testing purposes an unchecked value of "bytesover" is also available. Debug output can be enable by setting the debug tag: **slice** diff --git a/plugins/experimental/slice/client.cc b/plugins/experimental/slice/client.cc index e29f9e715ac..d52887e5278 100644 --- a/plugins/experimental/slice/client.cc +++ b/plugins/experimental/slice/client.cc @@ -65,6 +65,11 @@ requestBlock(TSCont contp, Data *const data) TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_upstream.m_write.m_iobuf); TSVIOReenable(data->m_upstream.m_write.m_vio); + /* + std::string const headerstr(header.toString()); + DEBUG_LOG("Headers\n%s", headerstr.c_str()); + */ + // get ready for data back from the server data->m_upstream.setupVioRead(contp); diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc index 508a455ecab..f2f4dee5af3 100644 --- a/plugins/experimental/slice/server.cc +++ b/plugins/experimental/slice/server.cc @@ -35,7 +35,7 @@ shutdown(TSCont const contp, Data *const data) } ContentRange -contentRangeFrom(HttpHeader &header) +contentRangeFrom(HttpHeader const &header) { ContentRange bcr; diff --git a/tests/gold_tests/pluginTest/slice/curlsort.sh b/tests/gold_tests/pluginTest/slice/curlsort.sh new file mode 100755 index 00000000000..eabf5618ff2 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/curlsort.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmd='curl' +for arg in "$@"; do + case "$arg" in + *\'*) +# arg=`printf '%s' "$arg" | sed s/'/'\"'\"'/g"` + arg=`printf '%s' "$arg"` + ;; + *) : ;; + esac + cmd="$cmd '$arg'" +done + +cmd="$cmd -s -D /dev/stdout -o /dev/stderr" + +eval " $cmd" | sort diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold new file mode 100644 index 00000000000..24ad29c1a82 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold @@ -0,0 +1 @@ +lets go surfin now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold new file mode 100644 index 00000000000..1e704c37b3d --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold @@ -0,0 +1,8 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 18 +Date: `` +Etag: "path" +HTTP/1.1 200 OK +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold new file mode 100644 index 00000000000..24ad29c1a82 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold @@ -0,0 +1 @@ +lets go surfin now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold new file mode 100644 index 00000000000..d4bd2de8782 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 18 +Content-Range: bytes 0-17/18 +Date: `` +Etag: "path" +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold new file mode 100644 index 00000000000..10893bb0688 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold @@ -0,0 +1 @@ +lets go`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold new file mode 100644 index 00000000000..235b074c618 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 7 +Content-Range: bytes 0-6/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold new file mode 100644 index 00000000000..6c9cadd2dfd --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold @@ -0,0 +1 @@ + now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold new file mode 100644 index 00000000000..88fa42bcfba --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 4 +Content-Range: bytes 14-17/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold new file mode 100644 index 00000000000..a9dc13c610e --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold @@ -0,0 +1 @@ +go surfin no`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold new file mode 100644 index 00000000000..5067f1cce49 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 12 +Content-Range: bytes 5-16/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/slice.test.py b/tests/gold_tests/pluginTest/slice/slice.test.py new file mode 100644 index 00000000000..a3e299a29d5 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice.test.py @@ -0,0 +1,188 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Basic slice plugin test +''' + +## Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with slice plugin +# Request content through the slice plugin + +Test.SkipUnless( + Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), + Condition.PluginExists('slice.so'), +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager") + +# default root +request_header_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_chk = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +server.addResponse("sessionlog.json", request_header_chk, response_header_chk) + +#block_bytes = 7 +body = "lets go surfin now" + +request_header = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, +} + +server.addResponse("sessionlog.json", request_header, response_header) + +ts.Setup.CopyAs('curlsort.sh', Test.RunDirectory) +curl_and_args = 'sh curlsort.sh -H "Host: www.example.com"' + +# set up whole asset fetch into cache +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) +) + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'slice', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.response_via_str': 3, + 'proxy.config.http.server_ports': '{}'.format(ts.Variables.port), +}) + +# 0 Test - Prefetch entire asset into cache +tr = Test.AddTestRun("Fetch first slice range") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_200.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_200.stderr.gold" +tr.StillRunningAfter = ts + +block_bytes = 7 + +# 1 - Reconfigure remap.config with slice plugin +tr = Test.AddTestRun("Load Slice plugin") +remap_config_path = ts.Disk.remap_config.Name +tr.Disk.File(remap_config_path, typename="ats:config").AddLines([ + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=slice.so @pparam=bytesover:{}'.format(block_bytes) +]) + +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 2 Test - First complete slice +tr = Test.AddTestRun("Fetch first slice range") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 0-6' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_first.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_first.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Test - Last slice auto +tr = Test.AddTestRun("Last slice -- 14-") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 4 Test - Last slice exact +tr = Test.AddTestRun("Last slice 14-17") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-17' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 5 Test - Last slice truncated +tr = Test.AddTestRun("Last truncated slice 14-20") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-20' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 6 Test - Whole asset via slices +tr = Test.AddTestRun("Whole asset via slices") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_200.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_200.stderr.gold" +tr.StillRunningAfter = ts + +# 7 Test - Whole asset via range +tr = Test.AddTestRun("Whole asset via range") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 0-' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_206.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_206.stderr.gold" +tr.StillRunningAfter = ts + +# 8 Test - Non aligned slice request +tr = Test.AddTestRun("Non aligned slice request") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 5-16' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_mid.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_mid.stderr.gold" +tr.StillRunningAfter = ts + From cdcd881e48bc25745d45f1c33a5f3ffc5d906d74 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 15 Mar 2019 09:08:42 -0600 Subject: [PATCH 392/526] Adds a new log tag for the MPTCP state of a connection Possible values are: -1 == MPTCP was not enabled on the listening port 0 == MPTCP was enabled, but not negotiated 1 == MPTCP was enabled, and succesfully negotiated --- doc/admin-guide/logging/formatting.en.rst | 13 +++++++--- include/tscore/ink_platform.h | 9 +++++++ iocore/net/I_NetProcessor.h | 6 +++++ iocore/net/I_NetVConnection.h | 18 ++++++++++++-- iocore/net/P_UnixNetVConnection.h | 16 ++++++++++++ iocore/net/UnixNetAccept.cc | 10 ++++++++ iocore/net/UnixNetProcessor.cc | 1 + mgmt/LocalManager.cc | 12 +++------ proxy/PluginVC.cc | 6 +++++ proxy/PluginVC.h | 1 + proxy/http/HttpProxyServerMain.cc | 1 + proxy/http/HttpSM.cc | 11 ++++++--- proxy/http/HttpSM.h | 30 +++++++++++------------ proxy/logging/Log.cc | 5 ++++ proxy/logging/LogAccess.cc | 15 ++++++++++++ proxy/logging/LogAccess.h | 1 + 16 files changed, 121 insertions(+), 34 deletions(-) diff --git a/doc/admin-guide/logging/formatting.en.rst b/doc/admin-guide/logging/formatting.en.rst index 1a4bce0f9ce..1217663d629 100644 --- a/doc/admin-guide/logging/formatting.en.rst +++ b/doc/admin-guide/logging/formatting.en.rst @@ -45,7 +45,7 @@ The return value from the ``format`` function is the log format object which may then be supplied to the appropriate ``log.*`` functions that define your logging destinations. -A very simple exampe, which contains only the timestamp of when the event began +A very simple example, which contains only the timestamp of when the event began and the canonical URL of the request, would look like: .. code:: yaml @@ -233,7 +233,7 @@ Error Code The log fields of error code which is triggered session close or transaction close. The first byte of this field indicates that the error code is session level (``S``) or transaction level (``T``). -When no error code is received or transmitted, these fileds are ``-``. +When no error code is received or transmitted, these fields are ``-``. For HTTP/2, error code are described in RFC 7540 section 7. ===== =============== ========================================================= @@ -660,6 +660,7 @@ TCP Details ~~~~~~~~~~~ .. _cqtr: +.. _cqmpt: The following logging fields reveal information about the TCP layer of client, proxy, and origin server connections. @@ -670,6 +671,10 @@ Field Source Description cqtr Client Request TCP reused status of the connection between the client and |TS| proxy, indicating whether the request was delivered through an already established connection. +cqmpt Client Request Indicates the MPTCP state of the connection. ``-1`` means + MPTCP was not enabled on the listening port, whereas ``0`` + and ``1`` indicates whether MPTCP was successfully + negotiated or not. ===== ============== ========================================================== .. _admin-logging-fields-time: @@ -698,7 +703,7 @@ The logging fields expose a variety of timing related information about client, proxy, and origin transactions. Variants of some of the fields provide timing resolution of the same underlying detail in milliseconds and seconds (both fractional and rounded-down integers). These variants are particularly useful -in accomodating the emulation of other HTTP proxy softwares' logging formats. +in accommodating the emulation of other HTTP proxy softwares' logging formats. Other fields in this category provide variously formatted timestamps of particular events within the current transaction (e.g. the time at which a @@ -731,7 +736,7 @@ ms Proxy Timestamp in milliseconds of a specific milestone which milestone to use. msdms Proxy Difference in milliseconds between the timestamps of two milestones. See note below about - specifying which miletones to use. + specifying which milestones to use. stms Proxy-Origin Connection Time (in milliseconds) spent accessing the origin server. Measured from the time the connection between proxy and origin is established to the diff --git a/include/tscore/ink_platform.h b/include/tscore/ink_platform.h index a7e0e8ae9b9..6bef79d7855 100644 --- a/include/tscore/ink_platform.h +++ b/include/tscore/ink_platform.h @@ -190,3 +190,12 @@ typedef unsigned int in_addr_t; // on various OSs (linux-4096,osx/bsd-1024, // windows-260,etc) #endif + +// This is a little bit of a hack for now, until MPTCP has landed upstream in Linux land. +#ifndef MPTCP_ENABLED +#if defined(linux) +#define MPTCP_ENABLED 42 +#else +#define MPTCP_ENABLED 0 +#endif +#endif diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index b8dee4e9dbc..570e7f3496f 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -97,6 +97,12 @@ class NetProcessor : public Processor */ bool f_inbound_transparent; + /** MPTCP enabled on listener. + @internal For logging and metrics purposes to know whether the + listener enabled MPTCP or not. + */ + bool f_mptcp; + /// Proxy Protocol enabled bool f_proxy_protocol; diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index e9065680549..e1e95044777 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -21,9 +21,11 @@ limitations under the License. */ - #pragma once +#include +#include + #include "tscore/ink_inet.h" #include "I_Action.h" #include "I_VConnection.h" @@ -32,7 +34,6 @@ #include "I_IOBuffer.h" #include "I_Socks.h" #include "ts/apidefs.h" -#include #include "YamlSNIConfig.h" #include "tscpp/util/TextView.h" #include "tscore/IpMap.h" @@ -649,6 +650,9 @@ class NetVConnection : public AnnotatedVConnection /** Set remote sock addr struct. */ virtual void set_remote_addr(const sockaddr *) = 0; + /** Set the MPTCP state for this connection */ + virtual void set_mptcp_state() = 0; + // for InkAPI bool get_is_internal_request() const @@ -668,6 +672,14 @@ class NetVConnection : public AnnotatedVConnection { return is_transparent; } + + /// Get the MPTCP state of the VC. + std::optional + get_mptcp_state() const + { + return mptcp_state; + } + /// Set the transparency state. void set_is_transparent(bool state = true) @@ -823,6 +835,8 @@ class NetVConnection : public AnnotatedVConnection bool is_transparent = false; /// Set if proxy protocol is enabled bool is_proxy_protocol = false; + /// This is essentially a tri-state, we leave it undefined to mean no MPTCP support + std::optional mptcp_state; /// Set if the next write IO that empties the write buffer should generate an event. int write_buffer_empty_event = 0; /// NetVConnection Context. diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index f56a208c735..46111bcad92 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -292,6 +292,7 @@ class UnixNetVConnection : public NetVConnection ink_hrtime get_active_timeout() override; void set_local_addr() override; + void set_mptcp_state() override; void set_remote_addr() override; void set_remote_addr(const sockaddr *) override; int set_tcp_init_cwnd(int init_cwnd) override; @@ -347,6 +348,21 @@ UnixNetVConnection::set_local_addr() ATS_UNUSED_RETURN(safe_getsockname(con.fd, &local_addr.sa, &local_sa_size)); } +// Update the internal VC state variable for MPTCP +inline void +UnixNetVConnection::set_mptcp_state() +{ + int mptcp_enabled = -1; + int mptcp_enabled_size = sizeof(mptcp_enabled); + + if (0 == safe_getsockopt(con.fd, IPPROTO_TCP, MPTCP_ENABLED, (char *)&mptcp_enabled, &mptcp_enabled_size)) { + Debug("socket_mptcp", "MPTCP socket state: %d", mptcp_enabled); + mptcp_state = mptcp_enabled > 0 ? true : false; + } else { + Debug("socket_mptcp", "MPTCP failed getsockopt(): %s", strerror(errno)); + } +} + inline ink_hrtime UnixNetVConnection::get_active_timeout() { diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index 0164d64b4b3..a1141e4c48b 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -117,6 +117,9 @@ net_accept(NetAccept *na, void *ep, bool blockable) vc->set_is_transparent(na->opt.f_inbound_transparent); vc->set_is_proxy_protocol(na->opt.f_proxy_protocol); vc->set_context(NET_VCONNECTION_IN); + if (na->opt.f_mptcp) { + vc->set_mptcp_state(); // Try to get the MPTCP state, and update accordingly + } #ifdef USE_EDGE_TRIGGER // Set the vc as triggered and place it in the read ready queue later in case there is already data on the socket. if (na->server.http_accept_filter) { @@ -354,6 +357,9 @@ NetAccept::do_blocking_accept(EThread *t) vc->options.ip_family = opt.ip_family; vc->apply_options(); vc->set_context(NET_VCONNECTION_IN); + if (opt.f_mptcp) { + vc->set_mptcp_state(); // Try to get the MPTCP state, and update accordingly + } vc->accept_object = this; #ifdef USE_EDGE_TRIGGER // Set the vc as triggered and place it in the read ready queue later in case there is already data on the socket. @@ -503,6 +509,10 @@ NetAccept::acceptFastEvent(int event, void *ep) vc->options.ip_family = opt.ip_family; vc->apply_options(); vc->set_context(NET_VCONNECTION_IN); + if (opt.f_mptcp) { + vc->set_mptcp_state(); // Try to get the MPTCP state, and update accordingly + } + #ifdef USE_EDGE_TRIGGER // Set the vc as triggered and place it in the read ready queue later in case there is already data on the socket. if (server.http_accept_filter) { diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index 436e6faac37..e9896d52a6e 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -51,6 +51,7 @@ NetProcessor::AcceptOptions::reset() packet_tos = 0; tfo_queue_length = 0; f_inbound_transparent = false; + f_mptcp = false; f_proxy_protocol = false; return *this; } diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index a9205bed65d..8caf6f03303 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -47,14 +47,6 @@ using namespace std::literals; static const std::string_view MGMT_OPT{"-M"}; static const std::string_view RUNROOT_OPT{"--run-root="}; -#ifndef MPTCP_ENABLED -#if defined(linux) -#define MPTCP_ENABLED 42 -#else -#define MPTCP_ENABLED 0 -#endif -#endif - void LocalManager::mgmtCleanup() { @@ -1043,11 +1035,13 @@ LocalManager::bindProxyPort(HttpProxyPort &port) err = setsockopt(port.m_fd, IPPROTO_TCP, MPTCP_ENABLED, &one, sizeof(one)); if (err < 0) { mgmt_log("[bindProxyPort] Unable to enable MPTCP: %s\n", strerror(errno)); + Debug("lm_mptcp", "[bindProxyPort] Unable to enable MPTCP: %s", strerror(errno)); } else { mgmt_log("[bindProxyPort] Successfully enabled MPTCP on %d\n", port.m_port); + Debug("lm_mptcp", "[bindProxyPort] Successfully enabled MPTCP on %d\n", port.m_port); } #else - Debug("lm", "[bindProxyPort] Multipath TCP requested but not configured on this host"); + Debug("lm_mptcp", "[bindProxyPort] Multipath TCP requested but not configured on this host"); #endif } diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc index b4a62e5845b..79abc43663a 100644 --- a/proxy/PluginVC.cc +++ b/proxy/PluginVC.cc @@ -963,6 +963,12 @@ PluginVC::set_remote_addr(const sockaddr * /* new_sa ATS_UNUSED */) return; } +void +PluginVC::set_mptcp_state() +{ + return; +} + int PluginVC::set_tcp_init_cwnd(int /* init_cwnd ATS_UNUSED */) { diff --git a/proxy/PluginVC.h b/proxy/PluginVC.h index d6205494328..6ee6b4db041 100644 --- a/proxy/PluginVC.h +++ b/proxy/PluginVC.h @@ -102,6 +102,7 @@ class PluginVC : public NetVConnection, public PluginIdentity void set_local_addr() override; void set_remote_addr() override; void set_remote_addr(const sockaddr *) override; + void set_mptcp_state() override; int set_tcp_init_cwnd(int init_cwnd) override; int set_tcp_congestion_control(int) override; diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index 7863833cf36..11abf906584 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -158,6 +158,7 @@ make_net_accept_options(const HttpProxyPort *port, unsigned nthreads) if (port) { net.f_inbound_transparent = port->m_inbound_transparent_p; + net.f_mptcp = port->m_mptcp; net.ip_family = port->m_family; net.local_port = port->m_port; net.f_proxy_protocol = port->m_proxy_protocol; diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 81ec27fff47..3b9ae712d05 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -465,12 +465,15 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader _client_connection_id = p->connection_id(); } } - // We've already verified that the netvc is !nullptr above, and netvc == ua_txn->get_netvc() - is_internal = netvc->get_is_internal_request(); - // Collect log & stats information - client_tcp_reused = !(ua_txn->is_first_transaction()); + // Collect log & stats information. We've already verified that the netvc is !nullptr above, + // and netvc == ua_txn->get_netvc(). SSLNetVConnection *ssl_vc = dynamic_cast(netvc); + + is_internal = netvc->get_is_internal_request(); + mptcp_state = netvc->get_mptcp_state(); + client_tcp_reused = !(ua_txn->is_first_transaction()); + if (ssl_vc != nullptr) { client_connection_is_ssl = true; client_ssl_reused = ssl_vc->getSSLSessionCacheHit(); diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index 76a0d1392fd..30ace9ed0b4 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -29,9 +29,11 @@ ****************************************************************************/ - #pragma once +#include +#include + #include "tscore/ink_platform.h" #include "P_EventSystem.h" #include "HttpCacheSM.h" @@ -41,9 +43,7 @@ #include "InkAPIInternal.h" #include "../ProxyClientTransaction.h" #include "HdrUtils.h" -#include #include "tscore/History.h" -//#include "AuthHttpAdapter.h" #define HTTP_API_CONTINUE (INK_API_EVENT_EVENTS_START + 0) #define HTTP_API_ERROR (INK_API_EVENT_EVENTS_START + 1) @@ -310,12 +310,12 @@ class HttpSM : public Continuation // YTS Team, yamsat Plugin bool enable_redirection = false; // To check if redirection is enabled + bool post_failed = false; // Added to identify post failure + bool debug_on = false; // Transaction specific debug flag char *redirect_url = nullptr; // url for force redirect (provide users a functionality to redirect to another url when needed) int redirect_url_len = 0; - int redirection_tries = 0; // To monitor number of redirections - int64_t transfered_bytes = 0; // Added to calculate POST data - bool post_failed = false; // Added to identify post failure - bool debug_on = false; // Transaction specific debug flag + int redirection_tries = 0; // To monitor number of redirections + int64_t transfered_bytes = 0; // Added to calculate POST data // Tunneling request to plugin HttpPluginTunnel_t plugin_tunnel_type = HTTP_NO_PLUGIN_TUNNEL; @@ -335,9 +335,9 @@ class HttpSM : public Continuation void postbuf_copy_partial_data(); void postbuf_init(IOBufferReader *ua_reader); void set_postbuf_done(bool done); + IOBufferReader *get_postbuf_clone_reader(); bool get_postbuf_done(); bool is_postbuf_valid(); - IOBufferReader *get_postbuf_clone_reader(); protected: int reentrancy_count = 0; @@ -536,17 +536,17 @@ class HttpSM : public Continuation int pushed_response_hdr_bytes = 0; int64_t pushed_response_body_bytes = 0; bool client_tcp_reused = false; - // Info about client's SSL connection. - bool client_ssl_reused = false; - bool client_connection_is_ssl = false; - bool is_internal = false; + bool client_ssl_reused = false; + bool client_connection_is_ssl = false; + bool is_internal = false; + bool server_connection_is_ssl = false; + bool is_waiting_for_full_body = false; + bool is_using_post_buffer = false; + std::optional mptcp_state; // Don't initialize, that marks it as "not defined". const char *client_protocol = "-"; const char *client_sec_protocol = "-"; const char *client_cipher_suite = "-"; int server_transact_count = 0; - bool server_connection_is_ssl = false; - bool is_waiting_for_full_body = false; - bool is_using_post_buffer = false; TransactionMilestones milestones; ink_hrtime api_timer = 0; diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc index b3a2354bb0c..04038f19747 100644 --- a/proxy/logging/Log.cc +++ b/proxy/logging/Log.cc @@ -502,6 +502,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("cqint", field); + field = new LogField("client_req_mptcp", "cqmpt", LogField::sINT, &LogAccess::marshal_client_req_mptcp_state, + &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("cqmpt", field); + field = new LogField("client_sec_protocol", "cqssv", LogField::STRING, &LogAccess::marshal_client_security_protocol, (LogField::UnmarshalFunc)&LogAccess::unmarshal_str); global_field_list.add(field, false); diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index 4fc737b19d5..202cfaab4d0 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -1725,6 +1725,21 @@ LogAccess::marshal_client_req_is_internal(char *buf) return INK_MIN_ALIGN; } +int +LogAccess::marshal_client_req_mptcp_state(char *buf) +{ + if (buf) { + int val = -1; + + if (m_http_sm->mptcp_state.has_value()) { + val = m_http_sm->mptcp_state.value() ? 1 : 0; + } else { + } + marshal_int(buf, val); + } + return INK_MIN_ALIGN; +} + /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/logging/LogAccess.h b/proxy/logging/LogAccess.h index 1e052825fb7..c91520412f4 100644 --- a/proxy/logging/LogAccess.h +++ b/proxy/logging/LogAccess.h @@ -150,6 +150,7 @@ class LogAccess inkcoreapi int marshal_client_req_is_ssl(char *); // INT inkcoreapi int marshal_client_req_ssl_reused(char *); // INT inkcoreapi int marshal_client_req_is_internal(char *); // INT + inkcoreapi int marshal_client_req_mptcp_state(char *); // INT inkcoreapi int marshal_client_security_protocol(char *); // STR inkcoreapi int marshal_client_security_cipher_suite(char *); // STR inkcoreapi int marshal_client_finish_status_code(char *); // INT From f7865e98926c824e8ab2517ad3e1aebd0efa7d18 Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Thu, 28 Mar 2019 09:59:14 -0500 Subject: [PATCH 393/526] check return value of open --- plugins/experimental/ssl_session_reuse/src/config.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/experimental/ssl_session_reuse/src/config.cc b/plugins/experimental/ssl_session_reuse/src/config.cc index a2d94571c4a..9c3caf0e3bf 100644 --- a/plugins/experimental/ssl_session_reuse/src/config.cc +++ b/plugins/experimental/ssl_session_reuse/src/config.cc @@ -56,13 +56,13 @@ Config::loadConfig(const std::string &filename) m_filename = filename; - int fd = (this->m_filename.length() > 0 ? open(m_filename.c_str(), O_RDONLY) : 0); + int fd = (this->m_filename.length() > 0 ? open(m_filename.c_str(), O_RDONLY) : ts::NO_FD); struct stat info; - if (0 == fstat(fd, &info)) { + if (fd > 0 && 0 == fstat(fd, &info)) { size_t n = info.st_size; std::string config_data; config_data.resize(n); - if (read(fd, const_cast(config_data.data()), n) < 0) { + if (read(fd, const_cast(config_data.data()), n) != n) { close(fd); return success; } From 17183e7b7615caf55efa32644228c0ec87e6c335 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 27 Mar 2019 01:28:56 +0100 Subject: [PATCH 394/526] Adds a little wrapper script to build inside vscode --- tools/code-make | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100755 tools/code-make diff --git a/tools/code-make b/tools/code-make new file mode 100755 index 00000000000..281ab56e336 --- /dev/null +++ b/tools/code-make @@ -0,0 +1,68 @@ +#!/usr/bin/env perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This tool is a "replacement" for make, which wraps the make command around +# a filter that tries to normalize the output such that the filenames are relative +# to the build directory. This passes all arguments passed along to make itself, +# and you can override the "make" command using the MAKE environment variable. + +use strict; +use Term::ANSIColor; + +my %COLORS = ( + "_header" => "italic green", + "_line" => "cyan", + "_message" => "italic magenta", + "error" => "red", + "warning" => "yellow", + "note" => "blue" +); +my $CMD = $ENV{"MAKE"} || "make" . " 2>&1 " . join(" ", @ARGV); +my @DIRS = (); + +print colored("Running: $CMD\n", $COLORS{"_header"}); + +open my $cmd, '-|', $CMD; +while (my $line = <$cmd>) { + if ($line =~ /(error|warning|note):/) { + my $msg = $1; + my @parts = split(/:/, $line); + my $file = $parts[0]; + + if (substr($file, 0, 1) ne "/") { + $file =~ s/^[\.\/]+//; # Strip leading ./, ../, ../../, etc. + + # Lazy eval on this, assuming that we will not find errors typically... + if (!@DIRS) { + @DIRS = split(/\n/, `find . -type d | fgrep -v -e .deps -e .libs -e .git -e .vscode`); + } + + foreach (@DIRS) { + if (-f "$_/$file") { + $file = "$_/$file"; + last; + } + } + } + print colored("$file:$parts[1]:$parts[2]:", $COLORS{"_line"}); + print colored("$msg", $COLORS{"$msg"}); + print colored("$parts[4]\n", $COLORS{"_message"}); + } else { + print $line; + } +} From 4621f36e587a51cad4a6b015205a6ddd07aa60c2 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 26 Mar 2019 10:32:10 +0100 Subject: [PATCH 395/526] Don't allow header values to overflow into negative values For example, CC: max-age=2147483648 will turn into a negative values and hence, can not be served out of cache. Since this parser deals with proper negative value, it seems reasonable to clamp this to INT_MAX. This fixes several issue from the cache-tests harness. --- proxy/hdrs/MIME.cc | 121 +++++++++++++++++++++++++++------------- proxy/hdrs/MIME.h | 12 ++-- proxy/hdrs/test_mime.cc | 41 ++++++++++++++ 3 files changed, 128 insertions(+), 46 deletions(-) diff --git a/proxy/hdrs/MIME.cc b/proxy/hdrs/MIME.cc index 8d223242ca1..c23e2646f82 100644 --- a/proxy/hdrs/MIME.cc +++ b/proxy/hdrs/MIME.cc @@ -3061,12 +3061,18 @@ mime_parse_int(const char *buf, const char *end) return 0; } - if (is_digit(*buf)) // fast case - { + if (is_digit(*buf)) { // fast case num = *buf++ - '0'; while ((buf != end) && is_digit(*buf)) { - num = (num * 10) + (*buf++ - '0'); + if (num != INT_MAX) { + int new_num = (num * 10) + (*buf++ - '0'); + + num = (new_num < num ? INT_MAX : new_num); // Check for overflow + } else { + ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT + } } + return num; } else { num = 0; @@ -3083,7 +3089,13 @@ mime_parse_int(const char *buf, const char *end) // NOTE: we first compute the value as negative then correct the // sign back to positive. This enables us to correctly parse MININT. while ((buf != end) && is_digit(*buf)) { - num = (num * 10) - (*buf++ - '0'); + if (num != INT_MIN) { + int new_num = (num * 10) - (*buf++ - '0'); + + num = (new_num > num ? INT_MIN : new_num); // Check for overflow, so to speak, see above re: negative + } else { + ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT + } } if (!negative) { @@ -3418,7 +3430,7 @@ mime_parse_date(const char *buf, const char *end) return t; } -int +bool mime_parse_day(const char *&buf, const char *end, int *day) { const char *e; @@ -3434,14 +3446,14 @@ mime_parse_day(const char *&buf, const char *end, int *day) *day = day_names_dfa->match({buf, size_t(e - buf)}); if (*day < 0) { - return 0; + return false; } else { buf = e; - return 1; + return true; } } -int +bool mime_parse_month(const char *&buf, const char *end, int *month) { const char *e; @@ -3457,20 +3469,20 @@ mime_parse_month(const char *&buf, const char *end, int *month) *month = month_names_dfa->match({buf, size_t(e - buf)}); if (*month < 0) { - return 0; + return false; } else { buf = e; - return 1; + return true; } } -int +bool mime_parse_mday(const char *&buf, const char *end, int *mday) { return mime_parse_integer(buf, end, mday); } -int +bool mime_parse_year(const char *&buf, const char *end, int *year) { int val; @@ -3480,7 +3492,7 @@ mime_parse_year(const char *&buf, const char *end, int *year) } if ((buf == end) || (*buf == '\0')) { - return 0; + return false; } val = 0; @@ -3497,59 +3509,88 @@ mime_parse_year(const char *&buf, const char *end, int *year) *year = val; - return 1; + return true; } -int +bool mime_parse_time(const char *&buf, const char *end, int *hour, int *min, int *sec) { if (!mime_parse_integer(buf, end, hour)) { - return 0; + return false; } if (!mime_parse_integer(buf, end, min)) { - return 0; + return false; } if (!mime_parse_integer(buf, end, sec)) { - return 0; + return false; } - return 1; + return true; } -// TODO: Do we really need mime_parse_int() and mime_parse_integer() ? I know -// they have slightly different prototypes, but still... -int +// This behaves slightly different than mime_parse_int(), int that we actually +// return a "bool" for success / failure on "reasonable" parsing. This kinda +// dumb, because we have two interfaces, where one does not move along the +// buf pointer, but this one does (and the ones using this function do). +bool mime_parse_integer(const char *&buf, const char *end, int *integer) { - int val; - bool negative; - - negative = false; - while ((buf != end) && *buf && !is_digit(*buf) && (*buf != '-')) { buf += 1; } if ((buf == end) || (*buf == '\0')) { - return 0; + return false; } - if (*buf == '-') { - negative = true; - buf += 1; - } + int32_t num; + bool negative; - val = 0; - while ((buf != end) && is_digit(*buf)) { - val = (val * 10) + (*buf++ - '0'); - } + // This code is copied verbatim from mime_parse_int ... Sigh. Maybe amc is right, and + // we really need to clean this up. But, as such, we should redo all these interfaces, + // and that's a big undertaking (and we'd want to move these strings all to string_view's). + if (is_digit(*buf)) { // fast case + num = *buf++ - '0'; + while ((buf != end) && is_digit(*buf)) { + if (num != INT_MAX) { + int new_num = (num * 10) + (*buf++ - '0'); - if (negative) { - *integer = -val; + num = (new_num < num ? INT_MAX : new_num); // Check for overflow + } else { + ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT + } + } } else { - *integer = val; + num = 0; + negative = false; + + while ((buf != end) && ParseRules::is_space(*buf)) { + buf += 1; + } + + if ((buf != end) && (*buf == '-')) { + negative = true; + buf += 1; + } + // NOTE: we first compute the value as negative then correct the + // sign back to positive. This enables us to correctly parse MININT. + while ((buf != end) && is_digit(*buf)) { + if (num != INT_MIN) { + int new_num = (num * 10) - (*buf++ - '0'); + + num = (new_num > num ? INT_MIN : new_num); // Check for overflow, so to speak, see above re: negative + } else { + ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT + } + } + + if (!negative) { + num = -num; + } } - return 1; + *integer = num; + + return true; } /*********************************************************************** diff --git a/proxy/hdrs/MIME.h b/proxy/hdrs/MIME.h index de5ea81b756..311b5cce39a 100644 --- a/proxy/hdrs/MIME.h +++ b/proxy/hdrs/MIME.h @@ -784,12 +784,12 @@ uint32_t mime_parse_uint(const char *buf, const char *end = nullptr); int64_t mime_parse_int64(const char *buf, const char *end = nullptr); int mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp); time_t mime_parse_date(const char *buf, const char *end = nullptr); -int mime_parse_day(const char *&buf, const char *end, int *day); -int mime_parse_month(const char *&buf, const char *end, int *month); -int mime_parse_mday(const char *&buf, const char *end, int *mday); -int mime_parse_year(const char *&buf, const char *end, int *year); -int mime_parse_time(const char *&buf, const char *end, int *hour, int *min, int *sec); -int mime_parse_integer(const char *&buf, const char *end, int *integer); +bool mime_parse_day(const char *&buf, const char *end, int *day); +bool mime_parse_month(const char *&buf, const char *end, int *month); +bool mime_parse_mday(const char *&buf, const char *end, int *mday); +bool mime_parse_year(const char *&buf, const char *end, int *year); +bool mime_parse_time(const char *&buf, const char *end, int *hour, int *min, int *sec); +bool mime_parse_integer(const char *&buf, const char *end, int *integer); /*********************************************************************** * * diff --git a/proxy/hdrs/test_mime.cc b/proxy/hdrs/test_mime.cc index b5055f6bb27..eba287d1b19 100644 --- a/proxy/hdrs/test_mime.cc +++ b/proxy/hdrs/test_mime.cc @@ -52,6 +52,47 @@ REGRESSION_TEST(MIME)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatu hdr.destroy(); } +REGRESSION_TEST(MIME_PARSERS)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) +{ + const char *end; + int value; + TestBox box(t, pstatus); + box = REGRESSION_TEST_PASSED; + + std::vector> tests = {{"0", 0}, + {"1234", 1234}, + {"-1234", -1234}, + {"2147483647", 2147483647}, + {"-2147483648", 2147483648}, + {"2147483648", INT_MAX}, + {"-2147483649", INT_MIN}, + {"2147483647", INT_MAX}, + {"-2147483648", INT_MIN}, + {"999999999999", INT_MAX}, + {"-999999999999", INT_MIN}}; + + for (const auto &it : tests) { + auto [buf, val] = it; + + end = buf + strlen(buf); + box.check(mime_parse_int(buf, end) == val, "Failed mime_parse_int"); + box.check(mime_parse_integer(buf, end, &value), "Failed mime_parse_integer call"); + box.check(value == val, "Failed mime_parse_integer value"); + } + + // Also check the date parser, which relies heavily on the mime_parse_integer() function + const char *date1 = "Sun, 05 Dec 1999 08:49:37 GMT"; + const char *date2 = "Sunday, 05-Dec-1999 08:49:37 GMT"; + + int d1 = mime_parse_date(date1, date1 + strlen(date1)); + int d2 = mime_parse_date(date2, date2 + strlen(date2)); + + box.check(d1 == d2, "Failed mime_parse_date"); + + printf("Date1: %d\n", d1); + printf("Date2: %d\n", d2); +} + int main(int argc, const char **argv) { From ab95cd7688ae624ff557980aa949e72b9ccaf984 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 1 Apr 2019 14:17:56 +0000 Subject: [PATCH 396/526] Fix compile problems with PR-5211 --- include/ts/apidefs.h.in | 7 +++++++ include/tscore/ink_defs.h | 9 --------- iocore/eventsystem/I_EventSystem.h | 1 + mgmt/api/NetworkUtilsRemote.h | 1 + plugins/experimental/ssl_session_reuse/src/config.cc | 3 ++- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 688e313dbee..4841d5602ed 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -1286,6 +1286,13 @@ typedef enum { #define TS_CRUUID_STRING_LEN (TS_UUID_STRING_LEN + 19 + 1) /* UUID-len + len(uint64_t) + '-' */ typedef struct tsapi_uuid *TSUuid; +#ifdef __cplusplus +namespace ts +{ + static const int NO_FD = -1; ///< No or invalid file descriptor. +} +#endif + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/include/tscore/ink_defs.h b/include/tscore/ink_defs.h index 251fcbf626a..1b81e799e09 100644 --- a/include/tscore/ink_defs.h +++ b/include/tscore/ink_defs.h @@ -130,12 +130,3 @@ int ink_login_name_max(); // Get the hardware topology hwloc_topology_t ink_get_topology(); #endif - -/** Constants. - */ -#ifdef __cplusplus -namespace ts -{ -static const int NO_FD = -1; ///< No or invalid file descriptor. -} -#endif diff --git a/iocore/eventsystem/I_EventSystem.h b/iocore/eventsystem/I_EventSystem.h index 4675a5e8c16..a062b5c31a8 100644 --- a/iocore/eventsystem/I_EventSystem.h +++ b/iocore/eventsystem/I_EventSystem.h @@ -26,6 +26,7 @@ #define _I_EventSystem_h #include "tscore/ink_platform.h" +#include "ts/apidefs.h" #include "I_IOBuffer.h" #include "I_Action.h" diff --git a/mgmt/api/NetworkUtilsRemote.h b/mgmt/api/NetworkUtilsRemote.h index 31e8734ba57..30f1a485c22 100644 --- a/mgmt/api/NetworkUtilsRemote.h +++ b/mgmt/api/NetworkUtilsRemote.h @@ -35,6 +35,7 @@ #pragma once #include "mgmtapi.h" +#include "ts/apidefs.h" #include "NetworkMessage.h" #include "EventCallback.h" diff --git a/plugins/experimental/ssl_session_reuse/src/config.cc b/plugins/experimental/ssl_session_reuse/src/config.cc index 9c3caf0e3bf..1aa089c5dce 100644 --- a/plugins/experimental/ssl_session_reuse/src/config.cc +++ b/plugins/experimental/ssl_session_reuse/src/config.cc @@ -23,6 +23,7 @@ */ #include +#include #include #include #include @@ -62,7 +63,7 @@ Config::loadConfig(const std::string &filename) size_t n = info.st_size; std::string config_data; config_data.resize(n); - if (read(fd, const_cast(config_data.data()), n) != n) { + if (read(fd, const_cast(config_data.data()), n) != static_cast(n)) { close(fd); return success; } From 51225741fe58c6e274e879585643dbd895b8b945 Mon Sep 17 00:00:00 2001 From: "Daniel Morilha (netlify)" Date: Mon, 25 Mar 2019 09:52:57 -0700 Subject: [PATCH 397/526] adding a note into storage.config admin doc about cache invalidation when changes are made --- doc/admin-guide/files/storage.config.en.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/admin-guide/files/storage.config.en.rst b/doc/admin-guide/files/storage.config.en.rst index 91b72b5e775..1945dc085ab 100644 --- a/doc/admin-guide/files/storage.config.en.rst +++ b/doc/admin-guide/files/storage.config.en.rst @@ -48,6 +48,10 @@ partitions. :arg:`volume` and arg:`seed` are optional. If the :arg:`id` option is used every use must have a unique value for :arg:`string`. +.. note:: + + Any change to this files can (and almost always will) invalidate the existing cache in its entirety. + You can use any partition of any size. For best performance: - Use raw disk partitions. From aa1098895dc2495796dc25b2519c2a01c5d0c744 Mon Sep 17 00:00:00 2001 From: Aaron Canary Date: Mon, 11 Mar 2019 13:25:08 -0500 Subject: [PATCH 398/526] AcidPtr test persistence --- src/tscore/unit_tests/test_AcidPtr.cc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/tscore/unit_tests/test_AcidPtr.cc b/src/tscore/unit_tests/test_AcidPtr.cc index f33af7bdf50..c8994cfb692 100644 --- a/src/tscore/unit_tests/test_AcidPtr.cc +++ b/src/tscore/unit_tests/test_AcidPtr.cc @@ -95,6 +95,28 @@ TEST_CASE("AcidPtr Isolation") CHECK(*p.getPtr() == 42); } +TEST_CASE("AcidPtr persistence") +{ + AcidPtr p(new int(40)); + std::shared_ptr r1, r2, r3, r4; + REQUIRE(p.getPtr() != nullptr); + r1 = p.getPtr(); + { + AcidCommitPtr w = p; + r2 = p.getPtr(); + *w += 1; // update p at end of scope + } + r3 = p.getPtr(); + { + *AcidCommitPtr(p) += 1; // leaves scope immediately if not named. + r4 = p.getPtr(); + } + CHECK(*r1 == 40); // references to data are still valid, but inconsistent. (todo: rename AcidPtr to AiPtr?) + CHECK(*r2 == 40); + CHECK(*r3 == 41); + CHECK(*r4 == 42); +} + TEST_CASE("AcidPtr Abort") { AcidPtr p; From 0b33066f6977c11c973917ef05eb51e207ca7db7 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Mon, 1 Apr 2019 14:49:44 +0000 Subject: [PATCH 399/526] Fix the hiredis autoconf for the default case --- build/hiredis.m4 | 54 +++++++++++++++++++----------------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/build/hiredis.m4 b/build/hiredis.m4 index e49ad7925ce..871df838a34 100644 --- a/build/hiredis.m4 +++ b/build/hiredis.m4 @@ -24,34 +24,33 @@ dnl AC_DEFUN([TS_CHECK_HIREDIS], [ hiredis_base_dir='/usr' -has_hiredis=0 +has_hiredis=1 AC_ARG_WITH(hiredis, [AC_HELP_STRING([--with-hiredis=DIR],[use a specific hiredis library])], [ - has_hiredis=1 if test "x$withval" != "xyes" && test "x$withval" != "x"; then hiredis_base_dir="$withval" - if test "$withval" != "no"; then - case "$withval" in - *":"*) - hiredis_include="`echo $withval |sed -e 's/:.*$//'`" - hiredis_ldflags="`echo $withval |sed -e 's/^.*://'`" - AC_MSG_CHECKING(checking for hiredis includes in $hiredis_include libs in $hiredis_ldflags ) - ;; - *) - hiredis_include="$withval/include" - hiredis_ldflags="$withval/lib" - AC_MSG_CHECKING(checking for hiredis includes in $withval) - ;; - esac - fi fi +],[]) - if test -d $hiredis_include && test -d $hiredis_ldflags && test -f $hiredis_include/hiredis/hiredis.h; then - AC_MSG_RESULT([ok]) - else - has_hiredis=0 - AC_MSG_RESULT([not found]) - fi +case "$hiredis_base_dir" in +*":"*) + hidredis_include="`echo $hiredis_base_dir |sed -e 's/:.*$//'`" + hiredis_ldflags="`echo $hiredis_base_dir |sed -e 's/^.*://'`" + AC_MSG_CHECKING(checking for hiredis includes in $hiredis_include libs in $hiredis_ldflags ) + ;; +*) + hiredis_include="$hiredis_base_dir/include" + hiredis_ldflags="$hiredis_base_dir/lib" + AC_MSG_CHECKING(checking for hiredis includes in $hiredis_base_dir) + ;; +esac + +if test -d $hiredis_include && test -d $hiredis_ldflags && test -f $hiredis_include/hiredis/hiredis.h; then + AC_MSG_RESULT([ok]) +else + has_hiredis=0 + AC_MSG_RESULT([not found]) +fi if test "$has_hiredis" != "0"; then saved_ldflags=$LDFLAGS @@ -77,17 +76,6 @@ if test "$has_hiredis" != "0"; then LDFLAGS=$saved_ldflags fi fi -], -[ -has_hiredis=1 -AC_CHECK_HEADER([hiredis/hiredis.h], [], [has_hiredis=0]) -AC_CHECK_LIB([hiredis], redisConnect, [], [has_hiredis=0]) - -if test "x$has_hiredis" == "x1"; then - AC_SUBST([LIB_HIREDIS], [-lhiredis]) -fi -]) - ]) From f32e5c1b76e2cc5f4583028a05d03ceb00fa5105 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Mon, 11 Mar 2019 14:50:42 +0900 Subject: [PATCH 400/526] Add nullptr check on ConfigProcessor::release() --- mgmt/ProxyConfig.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mgmt/ProxyConfig.cc b/mgmt/ProxyConfig.cc index 4ea74d623f4..2753ca41380 100644 --- a/mgmt/ProxyConfig.cc +++ b/mgmt/ProxyConfig.cc @@ -188,7 +188,7 @@ ConfigProcessor::release(unsigned int id, ConfigInfo *info) idx = id - 1; - if (info->refcount_dec() == 0) { + if (info && info->refcount_dec() == 0) { // When we release, we should already have replaced this object in the index. Debug("config", "Release config %d 0x%" PRId64, id, (int64_t)info); ink_release_assert(info != this->infos[idx]); From 5f6a278a17172f663e57270d5414fd52270ae4d8 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 2 Apr 2019 15:05:45 +0000 Subject: [PATCH 401/526] Fix compilation error in wccp --- src/wccp/WccpLocal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wccp/WccpLocal.h b/src/wccp/WccpLocal.h index 89dab2ebf2d..dff8f6d9856 100644 --- a/src/wccp/WccpLocal.h +++ b/src/wccp/WccpLocal.h @@ -25,6 +25,7 @@ #include "wccp/Wccp.h" #include "WccpUtil.h" #include "tscore/TsBuffer.h" +#include "ts/apidefs.h" // Needed for template use of byte ordering functions. #include #include From eb8cd942a950a99dc2210476e962f6d0bf55d5a2 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 3 Apr 2019 14:40:11 -0600 Subject: [PATCH 402/526] Normalize on negative value for milsetone metrics Rather than giving (now) arbitrary negative value, or zero as it was before, we now return -1 as an indicator that this metric does not have a sensible value semantically. --- proxy/logging/LogAccess.cc | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index 202cfaab4d0..da106fa3b4d 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -2265,9 +2265,7 @@ int LogAccess::marshal_server_resp_time_ms(char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones[TS_MILESTONE_SERVER_CLOSE] - m_http_sm->milestones[TS_MILESTONE_SERVER_CONNECT]; - int64_t val = (int64_t)ink_hrtime_to_msec(elapsed); - marshal_int(buf, val); + marshal_int(buf, m_http_sm->milestones.difference_msec(TS_MILESTONE_SERVER_CONNECT, TS_MILESTONE_SERVER_CLOSE)); } return INK_MIN_ALIGN; } @@ -2276,9 +2274,8 @@ int LogAccess::marshal_server_resp_time_s(char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones[TS_MILESTONE_SERVER_CLOSE] - m_http_sm->milestones[TS_MILESTONE_SERVER_CONNECT]; - int64_t val = (int64_t)ink_hrtime_to_sec(elapsed); - marshal_int(buf, val); + marshal_int(buf, + static_cast(m_http_sm->milestones.difference_sec(TS_MILESTONE_SERVER_CONNECT, TS_MILESTONE_SERVER_CLOSE))); } return INK_MIN_ALIGN; } @@ -2461,9 +2458,7 @@ int LogAccess::marshal_transfer_time_ms(char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones[TS_MILESTONE_SM_FINISH] - m_http_sm->milestones[TS_MILESTONE_SM_START]; - int64_t val = (int64_t)ink_hrtime_to_msec(elapsed); - marshal_int(buf, val); + marshal_int(buf, m_http_sm->milestones.difference_msec(TS_MILESTONE_SM_START, TS_MILESTONE_SM_FINISH)); } return INK_MIN_ALIGN; } @@ -2472,9 +2467,7 @@ int LogAccess::marshal_transfer_time_s(char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones[TS_MILESTONE_SM_FINISH] - m_http_sm->milestones[TS_MILESTONE_SM_START]; - int64_t val = (int64_t)ink_hrtime_to_sec(elapsed); - marshal_int(buf, val); + marshal_int(buf, static_cast(m_http_sm->milestones.difference_sec(TS_MILESTONE_SM_START, TS_MILESTONE_SM_FINISH))); } return INK_MIN_ALIGN; } @@ -2786,8 +2779,7 @@ int LogAccess::marshal_milestone_diff(TSMilestonesType ms1, TSMilestonesType ms2, char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones.elapsed(ms2, ms1); - int64_t val = (int64_t)ink_hrtime_to_msec(elapsed); + int64_t val = m_http_sm->milestones.difference_msec(ms2, ms1); marshal_int(buf, val); } return INK_MIN_ALIGN; From 089316d6e2926a2bd9c9c4dfd828fe0292bfd7c0 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Mon, 1 Apr 2019 13:13:10 -0700 Subject: [PATCH 403/526] Doc: Add IPv6 CIDR Mask example --- doc/admin-guide/plugins/header_rewrite.en.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index 17f3d7f79df..63df29fc49c 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -307,6 +307,8 @@ e.g.:: cond %{CIDR:8} ="8.0.0.0" set-header X-Is-Eight "Yes" + cond %{CIDR:,8} ="fd00::" #note the IPv6 Mask is in the second position + set-header IPv6Internal "true" This condition has no requirements other than access to the Client IP, hence, it should work in any and all hooks. From 768bcf8cf9b7d6fc36ec9145aed0ae1d9879e056 Mon Sep 17 00:00:00 2001 From: scw00 Date: Fri, 15 Mar 2019 22:21:12 +0800 Subject: [PATCH 404/526] Adds Cache test suits --- .gitignore | 3 + iocore/cache/Makefile.am | 124 +++++ iocore/cache/test/CacheTestHandler.cc | 76 ++++ iocore/cache/test/CacheTestHandler.h | 99 ++++ iocore/cache/test/main.cc | 273 +++++++++++ iocore/cache/test/main.h | 232 ++++++++++ iocore/cache/test/storage.config | Bin 0 -> 24 bytes iocore/cache/test/stub.cc | 264 +++++++++++ iocore/cache/test/test_Alternate_L_to_S.cc | 191 ++++++++ .../test/test_Alternate_L_to_S_remove_L.cc | 260 +++++++++++ .../test/test_Alternate_L_to_S_remove_S.cc | 261 +++++++++++ iocore/cache/test/test_Alternate_S_to_L.cc | 191 ++++++++ .../test/test_Alternate_S_to_L_remove_L.cc | 264 +++++++++++ .../test/test_Alternate_S_to_L_remove_S.cc | 262 +++++++++++ iocore/cache/test/test_Cache.cc | 55 +++ iocore/cache/test/test_RWW.cc | 429 ++++++++++++++++++ iocore/cache/test/var/trafficserver/guard.txt | 24 + 17 files changed, 3008 insertions(+) create mode 100644 iocore/cache/test/CacheTestHandler.cc create mode 100644 iocore/cache/test/CacheTestHandler.h create mode 100644 iocore/cache/test/main.cc create mode 100644 iocore/cache/test/main.h create mode 100644 iocore/cache/test/storage.config create mode 100644 iocore/cache/test/stub.cc create mode 100644 iocore/cache/test/test_Alternate_L_to_S.cc create mode 100644 iocore/cache/test/test_Alternate_L_to_S_remove_L.cc create mode 100644 iocore/cache/test/test_Alternate_L_to_S_remove_S.cc create mode 100644 iocore/cache/test/test_Alternate_S_to_L.cc create mode 100644 iocore/cache/test/test_Alternate_S_to_L_remove_L.cc create mode 100644 iocore/cache/test/test_Alternate_S_to_L_remove_S.cc create mode 100644 iocore/cache/test/test_Cache.cc create mode 100644 iocore/cache/test/test_RWW.cc create mode 100644 iocore/cache/test/var/trafficserver/guard.txt diff --git a/.gitignore b/.gitignore index 0c7f2854e9a..3b85d100b63 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,6 @@ RELEASE # autest tests/env-test/ + +iocore/cache/test_* +iocore/cache/test/var/trafficserver/cache.db diff --git a/iocore/cache/Makefile.am b/iocore/cache/Makefile.am index ed05579e428..a9993a37024 100644 --- a/iocore/cache/Makefile.am +++ b/iocore/cache/Makefile.am @@ -66,6 +66,130 @@ libinkcache_a_SOURCES += \ P_CacheTest.h endif +TESTS = $(check_PROGRAMS) + +test_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + $(iocore_include_dirs) \ + -I$(abs_top_srcdir)/include \ + -I$(abs_top_srcdir)/lib \ + -I$(abs_top_srcdir)/proxy \ + -I$(abs_top_srcdir)/proxy/http \ + -I$(abs_top_srcdir)/proxy/http2 \ + -I$(abs_top_srcdir)/proxy/http3 \ + -I$(abs_top_srcdir)/proxy/logging \ + -I$(abs_top_srcdir)/proxy/http/remap \ + -I$(abs_top_srcdir)/proxy/hdrs \ + -I$(abs_top_srcdir)/proxy/shared \ + -I$(abs_top_srcdir)/mgmt \ + -I$(abs_top_srcdir)/mgmt/utils \ + -I$(abs_top_srcdir)/tests/include \ + $(TS_INCLUDES) \ + @OPENSSL_INCLUDES@ + +test_LDADD = \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/iocore/cache/libinkcache.a \ + $(top_builddir)/proxy/libproxy.a \ + $(top_builddir)/proxy/http/libhttp.a \ + $(top_builddir)/proxy/http/remap/libhttp_remap.a \ + $(top_builddir)/proxy/libproxy.a \ + $(top_builddir)/iocore/net/libinknet.a \ + $(top_builddir)/iocore/dns/libinkdns.a \ + $(top_builddir)/iocore/hostdb/libinkhostdb.a \ + $(top_builddir)/proxy/logging/liblogging.a \ + $(top_builddir)/proxy/hdrs/libhdrs.a \ + $(top_builddir)/proxy/logging/liblogcollation.a \ + $(top_builddir)/proxy/shared/libdiagsconfig.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/iocore/utils/libinkutils.a \ + $(top_builddir)/iocore/aio/libinkaio.a \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/tsconfig/libtsconfig.la \ + @HWLOC_LIBS@ \ + @LIBPCRE@ \ + @LIBRESOLV@ \ + @LIBZ@ \ + @LIBLZMA@ \ + @LIBPROFILER@ \ + @OPENSSL_LIBS@ \ + @YAMLCPP_LIBS@ \ + -lm + + +check_PROGRAMS = \ + test_Cache \ + test_RWW \ + test_Alternate_L_to_S \ + test_Alternate_S_to_L \ + test_Alternate_L_to_S_remove_L \ + test_Alternate_L_to_S_remove_S \ + test_Alternate_S_to_L_remove_S \ + test_Alternate_S_to_L_remove_L + +test_main_SOURCES = \ + ./test/main.cc \ + ./test/stub.cc \ + ./test/CacheTestHandler.cc + +test_Cache_CPPFLAGS = $(test_CPPFLAGS) +test_Cache_LDFLAGS = @AM_LDFLAGS@ +test_Cache_LDADD = $(test_LDADD) +test_Cache_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Cache.cc + +test_RWW_CPPFLAGS = $(test_CPPFLAGS) +test_RWW_LDFLAGS = @AM_LDFLAGS@ +test_RWW_LDADD = $(test_LDADD) +test_RWW_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_RWW.cc + +test_Alternate_L_to_S_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_L_to_S_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_L_to_S_LDADD = $(test_LDADD) +test_Alternate_L_to_S_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_L_to_S.cc + +test_Alternate_S_to_L_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_S_to_L_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_S_to_L_LDADD = $(test_LDADD) +test_Alternate_S_to_L_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_S_to_L.cc + +test_Alternate_L_to_S_remove_L_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_L_to_S_remove_L_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_L_to_S_remove_L_LDADD = $(test_LDADD) +test_Alternate_L_to_S_remove_L_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_L_to_S_remove_L.cc + +test_Alternate_L_to_S_remove_S_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_L_to_S_remove_S_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_L_to_S_remove_S_LDADD = $(test_LDADD) +test_Alternate_L_to_S_remove_S_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_L_to_S_remove_S.cc + +test_Alternate_S_to_L_remove_S_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_S_to_L_remove_S_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_S_to_L_remove_S_LDADD = $(test_LDADD) +test_Alternate_S_to_L_remove_S_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_S_to_L_remove_S.cc + +test_Alternate_S_to_L_remove_L_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_S_to_L_remove_L_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_S_to_L_remove_L_LDADD = $(test_LDADD) +test_Alternate_S_to_L_remove_L_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_S_to_L_remove_L.cc + include $(top_srcdir)/build/tidy.mk clang-tidy-local: $(DIST_SOURCES) diff --git a/iocore/cache/test/CacheTestHandler.cc b/iocore/cache/test/CacheTestHandler.cc new file mode 100644 index 00000000000..72ceceaf934 --- /dev/null +++ b/iocore/cache/test/CacheTestHandler.cc @@ -0,0 +1,76 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + 1 test_Cache.cc + X distributed with this work for additional information regarding copyright ownership. The ASF licenses this + file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" +#include "CacheTestHandler.h" + +TestContChain::TestContChain() : Continuation(new_ProxyMutex()) {} + +CacheTestHandler::CacheTestHandler(size_t size, const char *url) +{ + this->_wt = new CacheWriteTest(size, this, url); + this->_rt = new CacheReadTest(size, this, url); + + this->_wt->mutex = this->mutex; + this->_rt->mutex = this->mutex; + SET_HANDLER(&CacheTestHandler::start_test); +} + +void +CacheTestHandler::handle_cache_event(int event, CacheTestBase *base) +{ + REQUIRE(base != nullptr); + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + break; + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_READ_READY: + case VC_EVENT_WRITE_READY: + REQUIRE(base->vc != nullptr); + REQUIRE(base->vio != nullptr); + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this_ethread()->schedule_imm(this->_rt); + base->close(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + base->close(); + delete this; + break; + } + return; +} + +int +CacheTestHandler::start_test(int event, void *e) +{ + this_ethread()->schedule_imm(this->_wt); + return 0; +} diff --git a/iocore/cache/test/CacheTestHandler.h b/iocore/cache/test/CacheTestHandler.h new file mode 100644 index 00000000000..3d2c1a0b697 --- /dev/null +++ b/iocore/cache/test/CacheTestHandler.h @@ -0,0 +1,99 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +void test_done(); + +#define TEST_DONE() test_done(); +#define T_DONE 1 +#define T_CONT 1 + +#define DEFAULT_URL "http://www.scw00.com/" + +#include + +class CacheTestBase; + +struct TestContChain : public Continuation { + TestContChain(); + virtual ~TestContChain() { this->next_test(); } + + void + add(TestContChain *n) + { + TestContChain *p = this; + while (p->next) { + p = p->next; + } + p->next = n; + } + + bool + next_test() + { + if (!this->next) { + return false; + } + + this_ethread()->schedule_imm(this->next); + return true; + } + + TestContChain *next = nullptr; +}; + +class CacheTestHandler : public TestContChain +{ +public: + CacheTestHandler() = default; + CacheTestHandler(size_t size, const char *url = DEFAULT_URL); + + int start_test(int event, void *e); + + virtual void handle_cache_event(int event, CacheTestBase *base); + +protected: + CacheTestBase *_rt = nullptr; + CacheTestBase *_wt = nullptr; +}; + +class TerminalTest : public CacheTestHandler +{ +public: + TerminalTest() { SET_HANDLER(&TerminalTest::terminal_event); } + ~TerminalTest() { TEST_DONE(); } + + int + terminal_event(int event, void *e) + { + delete this; + return 0; + } + + void + handle_cache_event(int event, CacheTestBase *e) override + { + delete this; + } +}; diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc new file mode 100644 index 00000000000..c62cd3f5fe1 --- /dev/null +++ b/iocore/cache/test/main.cc @@ -0,0 +1,273 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define CATCH_CONFIG_MAIN +#include "main.h" + +#define THREADS 1 +#define DIAGS_LOG_FILE "diags.log" + +char working_dir[1024] = {0}; +void +test_done() +{ + shutdown_event_system = true; +} + +const char *GLOBAL_DATA = (char *)ats_malloc(10 * 1024 * 1024 + 3); // 10M + +struct EventProcessorListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; // inherit constructor + + virtual void + testRunStarting(Catch::TestRunInfo const &testRunInfo) override + { + getcwd(working_dir, 1024); + BaseLogFile *base_log_file = new BaseLogFile("stderr"); + diags = new Diags(testRunInfo.name.c_str(), "*" /* tags */, "" /* actions */, base_log_file); + diags->activate_taglist("cache.*|agg.*|locks", DiagsTagType_Debug); + diags->config.enabled[DiagsTagType_Debug] = true; + diags->show_location = SHOW_LOCATION_DEBUG; + + mime_init(); + Layout::create(); + RecProcessInit(RECM_STAND_ALONE); + LibRecordsConfigInit(); + ink_net_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_assert(GLOBAL_DATA != nullptr); + + netProcessor.init(); + eventProcessor.start(THREADS); + + ink_aio_init(AIO_MODULE_PUBLIC_VERSION); + + EThread *thread = new EThread(); + thread->set_specific(); + init_buffer_allocators(0); + + Layout::get()->sysconfdir = std::string_view("./test"); + Layout::get()->prefix = std::string_view("./test"); + ::remove("./test/var/trafficserver/cache.db"); + } +}; +CATCH_REGISTER_LISTENER(EventProcessorListener); + +extern Store theCacheStore; + +void +init_cache(size_t size, const char *name) +{ + ink_cache_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + cacheProcessor.start(); +} + +void +build_hdrs(HTTPInfo &info, const char *url, const char *content_type) +{ + HTTPHdr req; + HTTPHdr resp; + HTTPParser parser; + int err = -1; + char buf[1024] = {0}; + const char *start = buf; + char *p = buf; + + REQUIRE(url != nullptr); + + p += sprintf(p, "GET %s HTTP/1.1\n", url); + p += sprintf(p, "User-Agent: curl/7.47.0\n"); + p += sprintf(p, "Accept: %s\n", content_type); + p += sprintf(p, "Vary: Content-type\n"); + p += sprintf(p, "Proxy-Connection: Keep-Alive\n\n"); + + req.create(HTTP_TYPE_REQUEST); + http_parser_init(&parser); + + while (true) { + err = req.parse_req(&parser, &start, p, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + + ink_assert(err == PARSE_RESULT_DONE); + + memset(buf, 0, sizeof(buf)); + p = buf; + + if (content_type == nullptr) { + content_type = "application/octet-stream"; + } + + p = buf; + p += sprintf(p, "HTTP/1.1 200 OK\n"); + p += sprintf(p, "Content-Type: %s\n", content_type); + p += sprintf(p, "Expires: Fri, 15 Mar 2219 08:55:45 GMT\n"); + p += sprintf(p, "Last-Modified: Thu, 14 Mar 2019 08:47:40 GMT\n\n"); + + resp.create(HTTP_TYPE_RESPONSE); + http_parser_init(&parser); + start = buf; + + while (true) { + err = resp.parse_resp(&parser, &start, p, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + ink_assert(err == PARSE_RESULT_DONE); + + info.request_set(&req); + info.response_set(&resp); + + req.destroy(); + resp.destroy(); +} + +HttpCacheKey +generate_key(HTTPInfo &info) +{ + HttpCacheKey key; + Cache::generate_key(&key, info.request_get()->url_get(), 1); + return key; +} + +void +CacheWriteTest::fill_data() +{ + size_t size = std::min(WRITE_LIMIT, this->_size); + auto n = this->_write_buffer->write(this->_cursor, size); + this->_size -= n; + this->_cursor += n; +} + +int +CacheWriteTest::write_event(int event, void *e) +{ + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + this->vc = (CacheVC *)e; + /* fall through */ + case CACHE_EVENT_OPEN_WRITE_FAILED: + this->process_event(event); + break; + case VC_EVENT_WRITE_READY: + this->process_event(event); + this->fill_data(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->process_event(event); + break; + default: + this->close(); + CHECK(false); + break; + } + return 0; +} + +void +CacheWriteTest::do_io_write(size_t size) +{ + if (size == 0) { + size = this->_size; + } + this->vc->set_http_info(&this->info); + this->vio = this->vc->do_io_write(this, size, this->_write_buffer->alloc_reader()); +} + +int +CacheWriteTest::start_test(int event, void *e) +{ + Debug("cache test", "start write test"); + + HttpCacheKey key; + key = generate_key(this->info); + + SET_HANDLER(&CacheWriteTest::write_event); + cacheProcessor.open_write(this, 0, &key, (CacheHTTPHdr *)this->info.request_get(), nullptr); + return 0; +} + +int +CacheReadTest::read_event(int event, void *e) +{ + switch (event) { + case CACHE_EVENT_OPEN_READ: + this->vc = (CacheVC *)e; + /* fall through */ + case CACHE_EVENT_OPEN_READ_FAILED: + this->process_event(event); + break; + case VC_EVENT_READ_READY: { + while (this->_reader->block_read_avail()) { + auto str = this->_reader->block_read_view(); + if (memcmp(str.data(), this->_cursor, str.size()) == 0) { + this->_reader->consume(str.size()); + this->_cursor += str.size(); + this->process_event(event); + } else { + CHECK(false); + this->close(); + TEST_DONE(); + break; + } + } + break; + } + case VC_EVENT_ERROR: + case VC_EVENT_EOS: + case VC_EVENT_READ_COMPLETE: + this->process_event(event); + break; + default: + CHECK(false); + this->close(); + break; + } + return 0; +} + +void +CacheReadTest::do_io_read(size_t size) +{ + if (size == 0) { + size = this->_size; + } + this->vc->get_http_info(&this->read_http_info); + this->vio = this->vc->do_io_read(this, size, this->_read_buffer); +} + +int +CacheReadTest::start_test(int event, void *e) +{ + Debug("cache test", "start read test"); + HttpCacheKey key; + key = generate_key(this->info); + + SET_HANDLER(&CacheReadTest::read_event); + cacheProcessor.open_read(this, &key, (CacheHTTPHdr *)this->info.request_get(), &this->params); + return 0; +} + +constexpr size_t WRITE_LIMIT = 1024 * 3; diff --git a/iocore/cache/test/main.h b/iocore/cache/test/main.h new file mode 100644 index 00000000000..b4dc4ee4544 --- /dev/null +++ b/iocore/cache/test/main.h @@ -0,0 +1,232 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include + +#include "catch.hpp" + +#include "tscore/I_Layout.h" +#include "tscore/Diags.h" + +#include "RecordsConfig.h" +#include "records/I_RecProcess.h" +#include "P_AIO.h" +#include "P_CacheDisk.h" +#include "P_Net.h" +#include "test/CacheTestHandler.h" +#include "P_Cache.h" + +#include + +// redefine BUILD PREFIX +#ifdef TS_BUILD_PREFIX +#undef TS_BUILD_PREFIX +#endif +#define TS_BUILD_PREFIX "./test" + +#ifdef TS_BUILD_EXEC_PREFIX +#undef TS_BUILD_EXEC_PREFIX +#endif +#define TS_BUILD_EXEC_PREFIX "./test" + +#ifdef TS_BUILD_SYSCONFDIR +#undef TS_BUILD_SYSCONFDIR +#endif +#define TS_BUILD_SYSCONFDIR "./test" + +#define SLEEP_TIME 20000 + +void init_cache(size_t size, const char *name = "cache.db"); +void build_hdrs(HTTPInfo &info, const char *url, const char *content_type = "text/html;charset=utf-8"); + +HttpCacheKey generate_key(HTTPInfo &info); + +extern const char *GLOBAL_DATA; +extern size_t const WRITE_LIMIT; + +class CacheInit : public Continuation +{ +public: + CacheInit() : Continuation(new_ProxyMutex()) { SET_HANDLER(&CacheInit::init_event); } + + int + start_event(int event, void *e) + { + Debug("cache_test", "cache init successfully"); + this->cache_init_success_callback(event, e); + return 0; + } + + int + init_event(int event, void *e) + { + switch (event) { + case EVENT_INTERVAL: + case EVENT_IMMEDIATE: + if (!CacheProcessor::IsCacheReady(CACHE_FRAG_TYPE_HTTP)) { + this_ethread()->schedule_in(this, SLEEP_TIME); + } else { + SET_HANDLER(&CacheInit::start_event); + this->handleEvent(event, e); + } + return 0; + default: + CHECK(false); + TEST_DONE(); + return 0; + } + + return 0; + } + + virtual int cache_init_success_callback(int event, void *e) = 0; + + virtual ~CacheInit() {} +}; + +class CacheTestBase : public Continuation +{ +public: + CacheTestBase(CacheTestHandler *test_handler) : Continuation(new_ProxyMutex()), test_handler(test_handler) + { + SET_HANDLER(&CacheTestBase::init_handler); + } + + int + init_handler(int event, void *e) + { + this->start_test(event, e); + return 0; + } + + // test entrance + virtual int start_test(int event, void *e) = 0; + + void + process_event(int event) + { + this->test_handler->handle_cache_event(event, this); + } + + virtual void + reenable() + { + if (this->vio) { + this->vio->reenable(); + } + } + + int + terminal_event(int event, void *e) + { + delete this; + return 0; + } + + void + close(int error = -1) + { + if (this->vc) { + this->vc->do_io_close(error); + this->vc = nullptr; + this->vio = nullptr; + } + + SET_HANDLER(&CacheTestBase::terminal_event); + if (!this->terminal) { + this->terminal = this_ethread()->schedule_imm(this); + } + } + + virtual void + do_io_read(size_t size = 0) + { + REQUIRE(!"should not be called"); + } + + virtual void + do_io_write(size_t size = 0) + { + REQUIRE(!"should not be called"); + } + + Event *terminal = nullptr; + CacheVC *vc = nullptr; + VIO *vio = nullptr; + CacheTestHandler *test_handler = nullptr; +}; + +class CacheWriteTest : public CacheTestBase +{ +public: + CacheWriteTest(size_t size, CacheTestHandler *cont, const char *url = "http://www.scw00.com/") : CacheTestBase(cont), _size(size) + { + this->_cursor = (char *)GLOBAL_DATA; + this->_write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + + this->info.create(); + build_hdrs(this->info, url); + } + + int start_test(int event, void *e) override; + int write_event(int event, void *e); + void fill_data(); + void do_io_write(size_t size = 0) override; + + HTTPInfo info; + +private: + size_t _size = 0; + char *_cursor = nullptr; + MIOBuffer *_write_buffer = nullptr; +}; + +class CacheReadTest : public CacheTestBase +{ +public: + CacheReadTest(size_t size, CacheTestHandler *cont, const char *url = "http://www.scw00.com/") : CacheTestBase(cont), _size(size) + { + this->_cursor = (char *)GLOBAL_DATA; + this->_read_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + this->_reader = this->_read_buffer->alloc_reader(); + + this->info.create(); + build_hdrs(this->info, url); + } + + int start_test(int event, void *e) override; + int read_event(int event, void *e); + void do_io_read(size_t size = 0) override; + + HTTPInfo info; + HTTPInfo *read_http_info = nullptr; + +private: + size_t _size = 0; + char *_cursor = nullptr; + MIOBuffer *_read_buffer = nullptr; + IOBufferReader *_reader = nullptr; + OverridableHttpConfigParams params; +}; diff --git a/iocore/cache/test/storage.config b/iocore/cache/test/storage.config new file mode 100644 index 0000000000000000000000000000000000000000..d86334e356c2d92d5962228d3fe1485382e59044 GIT binary patch literal 24 fcmXR*EYdG2N=!@3OfF6>DoZU=FfujsW#9q;a-|3Z literal 0 HcmV?d00001 diff --git a/iocore/cache/test/stub.cc b/iocore/cache/test/stub.cc new file mode 100644 index 00000000000..d6710ecc823 --- /dev/null +++ b/iocore/cache/test/stub.cc @@ -0,0 +1,264 @@ +/** @file + + Stub file for linking libinknet.a from unit tests + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "HttpSessionManager.h" +#include "HttpBodyFactory.h" +#include "DiagsConfig.h" +#include "ts/InkAPIPrivateIOCore.h" + +void +initialize_thread_for_http_sessions(EThread *, int) +{ + ink_assert(false); +} + +#include "InkAPIInternal.h" +void +APIHooks::append(INKContInternal *cont) +{ +} + +int +APIHook::invoke(int, void *) +{ + ink_assert(false); + return 0; +} + +APIHook * +APIHook::next() const +{ + ink_assert(false); + return nullptr; +} + +APIHook * +APIHooks::get() const +{ + return nullptr; +} + +void +APIHooks::prepend(INKContInternal *cont) +{ +} + +void +APIHooks::clear() +{ +} + +void +ConfigUpdateCbTable::invoke(const char * /* name ATS_UNUSED */) +{ + ink_release_assert(false); +} + +HttpAPIHooks *http_global_hooks = nullptr; +SslAPIHooks *ssl_hooks = nullptr; +LifecycleAPIHooks *lifecycle_hooks = nullptr; +ConfigUpdateCbTable *global_config_cbs = nullptr; + +HttpBodyFactory *body_factory = nullptr; + +intmax_t +ts::svtoi(TextView src, TextView *out, int base) +{ + intmax_t zret = 0; + + if (out) { + out->clear(); + } + if (!(0 <= base && base <= 36)) { + return 0; + } + if (src.ltrim_if(&isspace) && src) { + const char *start = src.data(); + int8_t v; + bool neg = false; + if ('-' == *src) { + ++src; + neg = true; + } + // If base is 0, it wasn't specified - check for standard base prefixes + if (0 == base) { + base = 10; + if ('0' == *src) { + ++src; + base = 8; + if (src && ('x' == *src || 'X' == *src)) { + ++src; + base = 16; + } + } + } + + // For performance in common cases, use the templated conversion. + switch (base) { + case 8: + zret = svto_radix<8>(src); + break; + case 10: + zret = svto_radix<10>(src); + break; + case 16: + zret = svto_radix<16>(src); + break; + default: + while (src.size() && (0 <= (v = svtoi_convert[static_cast(*src)])) && v < base) { + auto n = zret * base + v; + if (n < zret) { + zret = std::numeric_limits::max(); + break; // overflow, stop parsing. + } + zret = n; + ++src; + } + break; + } + + if (out && (src.data() > (neg ? start + 1 : start))) { + out->assign(start, src.data()); + } + + if (neg) { + zret = -zret; + } + } + return zret; +} + +void +HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned int down_time, const char *reason) +{ +} + +HostStatus_t +HostStatus::getHostStatus(const char *name) +{ + return (HostStatus_t)0; +} + +void +HostStatus::createHostStat(const char *name) +{ +} + +HostStatus::HostStatus() {} + +HostStatus::~HostStatus() {} + +int auto_clear_hostdb_flag = 0; +bool ts_is_draining = false; + +void +INKVConnInternal::do_io_close(int error) +{ +} + +void +INKVConnInternal::do_io_shutdown(ShutdownHowTo_t howto) +{ +} + +VIO * +INKVConnInternal::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) +{ + return nullptr; +} + +VIO * +INKVConnInternal::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) +{ + return nullptr; +} + +void +INKVConnInternal::destroy() +{ +} + +void +INKVConnInternal::free() +{ +} + +void +INKVConnInternal::clear() +{ +} + +void +INKVConnInternal::reenable(VIO * /* vio ATS_UNUSED */) +{ +} + +bool +INKVConnInternal::get_data(int id, void *data) +{ + return false; +} + +bool +INKVConnInternal::set_data(int id, void *data) +{ + return false; +} + +void +INKVConnInternal::do_io_transform(VConnection *vc) +{ +} + +void +INKContInternal::handle_event_count(int event) +{ +} + +void +INKVConnInternal::retry(unsigned int delay) +{ +} + +INKContInternal::INKContInternal(TSEventFunc funcp, TSMutex mutexp) : DummyVConnection((ProxyMutex *)mutexp) {} + +INKContInternal::INKContInternal() : DummyVConnection(nullptr) {} + +void +INKContInternal::destroy() +{ +} + +void +INKContInternal::clear() +{ +} + +void +INKContInternal::free() +{ +} + +INKVConnInternal::INKVConnInternal() : INKContInternal() {} + +INKVConnInternal::INKVConnInternal(TSEventFunc funcp, TSMutex mutexp) : INKContInternal(funcp, mutexp) {} diff --git a/iocore/cache/test/test_Alternate_L_to_S.cc b/iocore/cache/test/test_Alternate_L_to_S.cc new file mode 100644 index 00000000000..6d4656a1acf --- /dev/null +++ b/iocore/cache/test/test_Alternate_L_to_S.cc @@ -0,0 +1,191 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_L_to_S : public CacheTestHandler +{ +public: + CacheAltTest_L_to_S(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_L_to_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + this_ethread()->schedule_imm(this->_rt); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + +private: + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE, "http://www.scw11.com"); + CacheAltTest_L_to_S *ls = new CacheAltTest_L_to_S(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(LARGE_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc b/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc new file mode 100644 index 00000000000..f1167c7002b --- /dev/null +++ b/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc @@ -0,0 +1,260 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +// delete dir +Dir dir = {}; + +class CacheAltReadAgain2 : public CacheTestHandler +{ +public: + CacheAltReadAgain2(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + + rt->info.destroy(); + + rt->info.create(); + build_hdrs(rt->info, url, "application/x-javascript"); + + this->_rt = rt; + + SET_HANDLER(&CacheAltReadAgain2::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } +}; + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ_FAILED: + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_L_to_S_remove_L : public CacheTestHandler +{ +public: + CacheAltTest_L_to_S_remove_L(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_L_to_S_remove_L::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + this_ethread()->schedule_imm(this->_rt); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + delete_earliest_dir(base->vc); + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } + + void + delete_earliest_dir(CacheVC *vc) + { + CacheKey key = {}; + Dir *last_collision = nullptr; + SCOPED_MUTEX_LOCK(lock, vc->vol->mutex, this->mutex->thread_holding); + vc->vector.data[0].alternate.object_key_get(&key); + REQUIRE(dir_probe(&key, vc->vol, &dir, &last_collision) != 0); + REQUIRE(dir_delete(&key, vc->vol, &dir)); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE, "http://www.scw11.com"); + CacheAltTest_L_to_S_remove_L *ls = new CacheAltTest_L_to_S_remove_L(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain2 *read2 = new CacheAltReadAgain2(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(read2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc b/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc new file mode 100644 index 00000000000..f2a4844d21a --- /dev/null +++ b/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc @@ -0,0 +1,261 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +// delete dir +Dir dir = {}; + +class CacheAltReadAgain2 : public CacheTestHandler +{ +public: + CacheAltReadAgain2(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + this->_rt = rt; + SET_HANDLER(&CacheAltReadAgain2::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + + rt->info.destroy(); + + rt->info.create(); + build_hdrs(rt->info, url, "application/x-javascript"); + + this->_rt = rt; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ_FAILED: + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_L_to_S_remove_S : public CacheTestHandler +{ +public: + CacheAltTest_L_to_S_remove_S(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_L_to_S_remove_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + this_ethread()->schedule_imm(this->_rt); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + delete_earliest_dir(base->vc); + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } + + void + delete_earliest_dir(CacheVC *vc) + { + CacheKey key = {}; + Dir *last_collision = nullptr; + SCOPED_MUTEX_LOCK(lock, vc->vol->mutex, this->mutex->thread_holding); + vc->vector.data[1].alternate.object_key_get(&key); + REQUIRE(dir_probe(&key, vc->vol, &dir, &last_collision) != 0); + REQUIRE(dir_delete(&key, vc->vol, &dir)); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE, "http://www.scw11.com"); + CacheAltTest_L_to_S_remove_S *ls = new CacheAltTest_L_to_S_remove_S(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain2 *read2 = new CacheAltReadAgain2(LARGE_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(read2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_S_to_L.cc b/iocore/cache/test/test_Alternate_S_to_L.cc new file mode 100644 index 00000000000..097b61b59b0 --- /dev/null +++ b/iocore/cache/test/test_Alternate_S_to_L.cc @@ -0,0 +1,191 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_L_to_S : public CacheTestHandler +{ +public: + CacheAltTest_L_to_S(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_L_to_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + this_ethread()->schedule_imm(this->_rt); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + +private: + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + CacheAltTest_L_to_S *ls = new CacheAltTest_L_to_S(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc b/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc new file mode 100644 index 00000000000..282224baabe --- /dev/null +++ b/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc @@ -0,0 +1,264 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +// delete dir +Dir dir = {}; + +class CacheAltReadAgain2 : public CacheTestHandler +{ +public: + CacheAltReadAgain2(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + this->_rt = rt; + SET_HANDLER(&CacheAltReadAgain2::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + + rt->info.destroy(); + + rt->info.create(); + build_hdrs(rt->info, url, "application/x-javascript"); + + this->_rt = rt; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + // sleep for a while to wait for writer close + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ_FAILED: + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_S_to_L_remove_L : public CacheTestHandler +{ +public: + CacheAltTest_S_to_L_remove_L(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_S_to_L_remove_L::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + // to make sure writer successfully write the final doc done. we need to schedule + // to wait for some while. This time should be large than cache_config_mutex_retry_delay + this_ethread()->schedule_in(this->_rt, HRTIME_SECONDS(1)); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + delete_earliest_dir(base->vc); + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } + + void + delete_earliest_dir(CacheVC *vc) + { + CacheKey key = {}; + Dir *last_collision = nullptr; + SCOPED_MUTEX_LOCK(lock, vc->vol->mutex, this->mutex->thread_holding); + vc->vector.data[1].alternate.object_key_get(&key); + REQUIRE(dir_probe(&key, vc->vol, &dir, &last_collision) != 0); + REQUIRE(dir_delete(&key, vc->vol, &dir)); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + CacheAltTest_S_to_L_remove_L *ls = new CacheAltTest_S_to_L_remove_L(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain2 *read2 = new CacheAltReadAgain2(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(read2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc b/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc new file mode 100644 index 00000000000..96a1e7cce77 --- /dev/null +++ b/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc @@ -0,0 +1,262 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +// delete dir +Dir dir = {}; + +class CacheAltReadAgain2 : public CacheTestHandler +{ +public: + CacheAltReadAgain2(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + + rt->info.destroy(); + + rt->info.create(); + build_hdrs(rt->info, url, "application/x-javascript"); + + this->_rt = rt; + + SET_HANDLER(&CacheAltReadAgain2::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } +}; + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ_FAILED: + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class test_Alternate_S_to_L_remove_S : public CacheTestHandler +{ +public: + test_Alternate_S_to_L_remove_S(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&test_Alternate_S_to_L_remove_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + // to make sure writer successfully write the final doc done. we need to schedule + // to wait for some while. This time should be large than cache_config_mutex_retry_delay + this_ethread()->schedule_in(this->_rt, HRTIME_SECONDS(1)); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + delete_earliest_dir(base->vc); + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } + + void + delete_earliest_dir(CacheVC *vc) + { + CacheKey key = {}; + Dir *last_collision = nullptr; + SCOPED_MUTEX_LOCK(lock, vc->vol->mutex, this->mutex->thread_holding); + vc->vector.data[0].alternate.object_key_get(&key); + REQUIRE(dir_probe(&key, vc->vol, &dir, &last_collision) != 0); + REQUIRE(dir_delete(&key, vc->vol, &dir)); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + test_Alternate_S_to_L_remove_S *ls = new test_Alternate_S_to_L_remove_S(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain2 *read2 = new CacheAltReadAgain2(LARGE_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(read2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Cache.cc b/iocore/cache/test/test_Cache.cc new file mode 100644 index 00000000000..63284f51c54 --- /dev/null +++ b/iocore/cache/test/test_Cache.cc @@ -0,0 +1,55 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +class CacheCommInit : public CacheInit +{ +public: + CacheCommInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE); + CacheTestHandler *h2 = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + h->add(h2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheCommInit *init = new CacheCommInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_RWW.cc b/iocore/cache/test/test_RWW.cc new file mode 100644 index 00000000000..167cf1ee7b4 --- /dev/null +++ b/iocore/cache/test/test_RWW.cc @@ -0,0 +1,429 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#define DEFAULT_URL "http://www.scw00.com/" + +#include "main.h" + +class CacheRWWTest; + +struct SimpleCont : public Continuation { + SimpleCont(CacheTestBase *base) : base(base) + { + REQUIRE(base != nullptr); + SET_HANDLER(&SimpleCont::handle_event); + this->mutex = base->mutex; + } + + int + handle_event(int event, void *data) + { + Debug("cache_rww_test", "cache write reenable"); + base->reenable(); + delete this; + return 0; + } + + CacheTestBase *base = nullptr; +}; + +class CacheRWWTest : public CacheTestHandler +{ +public: + CacheRWWTest(size_t size, const char *url = DEFAULT_URL) : CacheTestHandler(), _size(size) + { + if (size != LARGE_FILE && size != SMALL_FILE) { + REQUIRE(!"size should be LARGE_FILE or SMALL_FILE"); + } + + this->_rt = new CacheReadTest(size, this, url); + this->_wt = new CacheWriteTest(size, this, url); + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheRWWTest::start_test); + } + + void handle_cache_event(int event, CacheTestBase *e) override; + int start_test(int event, void *e); + + virtual void process_read_event(int event, CacheTestBase *base); + virtual void process_write_event(int event, CacheTestBase *base); + + void + close_write(int error = -1) + { + if (!this->_wt) { + return; + } + + this->_wt->close(error); + this->_wt = nullptr; + } + + void + close_read(int error = -1) + { + if (!this->_rt) { + return; + } + + this->_rt->close(error); + this->_rt = nullptr; + } + +protected: + size_t _size = 0; + Event *_read_event = nullptr; + bool _is_read_start = false; + CacheTestBase *_rt = nullptr; + CacheTestBase *_wt = nullptr; +}; + +int +CacheRWWTest::start_test(int event, void *e) +{ + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; +} + +void +CacheRWWTest::process_write_event(int event, CacheTestBase *base) +{ + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + // schedule read test imm + if (this->_size != SMALL_FILE && !this->_wt->vc->fragment) { + Debug("cache_rww_test", "cache write reenable"); + base->reenable(); + return; + } + + if (!this->_is_read_start) { + if (!this->_read_event) { + this->_read_event = this_ethread()->schedule_imm(this->_rt); + } + return; + } + + // stop writing for a while and wait for reading + // data->vio->reenable(); + this_ethread()->schedule_imm(new SimpleCont(base)); + break; + case VC_EVENT_WRITE_COMPLETE: + this->close_write(); + + break; + default: + REQUIRE(event == 0); + REQUIRE(false); + this->close_write(); + this->close_read(); + return; + } + + if (this->_rt) { + this->_rt->reenable(); + } +} + +void +CacheRWWTest::process_read_event(int event, CacheTestBase *base) +{ + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + break; + case VC_EVENT_READ_READY: + Debug("cache_rww_test", "cache read reenable"); + this->_read_event = nullptr; + this->_is_read_start = true; + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + this->close_read(); + return; + + default: + REQUIRE(false); + this->close_write(); + this->close_read(); + return; + } + + if (this->_wt) { + this->_wt->reenable(); + } +} + +void +CacheRWWTest::handle_cache_event(int event, CacheTestBase *base) +{ + REQUIRE(base != nullptr); + + switch (event) { + case CACHE_EVENT_OPEN_WRITE_FAILED: + case CACHE_EVENT_OPEN_WRITE: + case VC_EVENT_WRITE_READY: + case VC_EVENT_WRITE_COMPLETE: + this->process_write_event(event, base); + break; + case CACHE_EVENT_OPEN_READ: + case CACHE_EVENT_OPEN_READ_FAILED: + case VC_EVENT_ERROR: + case VC_EVENT_EOS: + case VC_EVENT_READ_READY: + case VC_EVENT_READ_COMPLETE: + this->process_read_event(event, base); + break; + default: + REQUIRE(false); + this->close_write(); + this->close_read(); + break; + } + + if (this->_wt == nullptr && this->_rt == nullptr) { + delete this; + } + + return; +} + +class CacheRWWErrorTest : public CacheRWWTest +{ +public: + CacheRWWErrorTest(size_t size, const char *url = DEFAULT_URL) : CacheRWWTest(size, url) {} + void + process_write_event(int event, CacheTestBase *base) override + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + if (this->_size != SMALL_FILE && !this->_wt->vc->fragment) { + Debug("cache_rww_test", "cache write reenable"); + base->reenable(); + return; + } + + if (!this->_is_read_start) { + if (!this->_read_event) { + this->_read_event = this_ethread()->schedule_imm(this->_rt); + } + return; + } else { + this->close_write(100); + return; + } + this_ethread()->schedule_imm(new SimpleCont(base)); + break; + + case VC_EVENT_WRITE_COMPLETE: + REQUIRE(!"should not happen because the writter aborted"); + this->close_read(); + this->close_write(); + break; + default: + REQUIRE(false); + delete this; + return; + } + } + + void + process_read_event(int event, CacheTestBase *base) override + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + this->_read_event = nullptr; + this->_is_read_start = true; + base->do_io_read(); + break; + case CACHE_EVENT_OPEN_READ_FAILED: + REQUIRE(this->_size == SMALL_FILE); + this->close_read(); + return; + case VC_EVENT_READ_READY: + base->reenable(); + if (this->_wt) { + this->_wt->reenable(); + } + return; + + case VC_EVENT_READ_COMPLETE: + REQUIRE(!"should not happen because the writter aborted"); + this->close_read(); + this->close_write(); + break; + case VC_EVENT_ERROR: + case VC_EVENT_EOS: + if (this->_size == LARGE_FILE) { + REQUIRE(base->vio->ndone >= 1 * 1024 * 1024 - sizeof(Doc)); + } else { + REQUIRE(base->vio->ndone == 0); + } + this->close_read(); + break; + default: + REQUIRE(event == 0); + this->close_read(); + this->close_write(); + break; + } + } + +private: + bool _is_read_start = false; +}; + +class CacheRWWEOSTest : public CacheRWWTest +{ +public: + CacheRWWEOSTest(size_t size, const char *url = DEFAULT_URL) : CacheRWWTest(size, url) {} + /* + * test this code in openReadMain + * if (writer_done()) { + last_collision = nullptr; + while (dir_probe(&earliest_key, vol, &dir, &last_collision)) { + // write complete. this could be reached the size we set in do_io_write or someone call do_io_close with -1 (-1 means write + success) flag if (dir_offset(&dir) == dir_offset(&earliest_dir)) { DDebug("cache_read_agg", "%p: key: %X ReadMain complete: %d", + this, first_key.slice32(1), (int)vio.ndone); doc_len = vio.ndone; goto Leos; + } + } + // writer abort. server crash. someone call do_io_close() with error flag + DDebug("cache_read_agg", "%p: key: %X ReadMain writer aborted: %d", this, first_key.slice32(1), (int)vio.ndone); + goto Lerror; + } + * + */ + + void + process_write_event(int event, CacheTestBase *base) override + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + if (this->_size != SMALL_FILE && !this->_wt->vc->fragment) { + Debug("cache_rww_test", "cache write reenable"); + base->reenable(); + return; + } + + if (!this->_is_read_start) { + if (!this->_read_event) { + this->_read_event = this_ethread()->schedule_imm(this->_rt); + } + return; + } + this_ethread()->schedule_imm(new SimpleCont(base)); + break; + + case VC_EVENT_WRITE_COMPLETE: + this->close_write(); + break; + default: + REQUIRE(false); + delete this; + return; + } + } + + void + process_read_event(int event, CacheTestBase *base) override + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + this->_read_event = nullptr; + this->_is_read_start = true; + base->do_io_read(UINT32_MAX); + break; + case VC_EVENT_READ_READY: + base->reenable(); + if (this->_wt) { + this->_wt->reenable(); + } + return; + + case VC_EVENT_READ_COMPLETE: + REQUIRE(!"should not happen because the writter aborted"); + this->close_read(); + this->close_write(); + break; + case VC_EVENT_EOS: + this->close_write(); + this->close_read(); + break; + default: + REQUIRE(event == 0); + this->close_read(); + this->close_write(); + break; + } + } + +private: + bool _is_read_start = false; +}; + +class CacheRWWCacheInit : public CacheInit +{ +public: + CacheRWWCacheInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheRWWTest *crww = new CacheRWWTest(LARGE_FILE); + CacheRWWErrorTest *crww_l = new CacheRWWErrorTest(LARGE_FILE, "http://www.scw22.com/"); + CacheRWWEOSTest *crww_eos = new CacheRWWEOSTest(LARGE_FILE, "ttp://www.scw44.com/"); + TerminalTest *tt = new TerminalTest(); + + crww->add(crww_l); + crww->add(crww_eos); + crww->add(tt); + this_ethread()->schedule_imm(crww); + delete this; + return 0; + } +}; + +TEST_CASE("cache rww", "cache") +{ + init_cache(256 * 1024 * 1024); + cache_config_target_fragment_size = 1 * 1024 * 1024; + CacheRWWCacheInit *init = new CacheRWWCacheInit(); + + this_ethread()->schedule_imm(init); + this_ethread()->execute(); +} diff --git a/iocore/cache/test/var/trafficserver/guard.txt b/iocore/cache/test/var/trafficserver/guard.txt new file mode 100644 index 00000000000..8fe36290b11 --- /dev/null +++ b/iocore/cache/test/var/trafficserver/guard.txt @@ -0,0 +1,24 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +This is guard file for this empty dir From 3d6e63d764bb8741d4f6a30227f651f0e7c6118a Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Tue, 2 Apr 2019 13:25:48 +0000 Subject: [PATCH 405/526] url_sig debug fix for when url is missing the signature query string --- plugins/experimental/url_sig/README | 2 +- plugins/experimental/url_sig/url_sig.c | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/plugins/experimental/url_sig/README b/plugins/experimental/url_sig/README index a8b4360d634..9dc4dee3f31 100644 --- a/plugins/experimental/url_sig/README +++ b/plugins/experimental/url_sig/README @@ -133,7 +133,7 @@ Example Add the remap rule like - map http://test-remap.domain.com http://google.com @plugin=url_sig.so @pparam=sign_test.config + map http://test-remap.domain.com http://google.com @plugin=url_sig.so @pparam=sign_test.config @pparam=pristineurl Restart traffic server or traffic_ctl config reload - verify there are no errors in diags.log diff --git a/plugins/experimental/url_sig/url_sig.c b/plugins/experimental/url_sig/url_sig.c index 762a7da3ad4..a21861e1753 100644 --- a/plugins/experimental/url_sig/url_sig.c +++ b/plugins/experimental/url_sig/url_sig.c @@ -23,9 +23,9 @@ _a < _b ? _a : _b; \ }) -#include "tscore/ink_defs.h" #include "url_sig.h" +#include #include #include #include @@ -538,7 +538,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) char sig_string[2 * MAX_SIG_SIZE + 1]; if (current_url_len >= MAX_REQ_LEN - 1) { - err_log(url, "URL string too long"); + err_log(current_url, "Request Url string too long"); goto deny; } @@ -555,15 +555,12 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) err_log(url, "Pristine URL string too long."); goto deny; } - } else { url_len = current_url_len; } TSDebug(PLUGIN_NAME, "%s", url); - const char *query = strchr(url, '?'); - if (cfg->regex) { const int offset = 0, options = 0; int ovector[30]; @@ -580,12 +577,17 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) } } + const char *query = strchr(url, '?'); + // check for path params. if (query == NULL || strstr(query, "E=") == NULL) { - if ((url = urlParse(url, cfg->sig_anchor, new_path, 8192, path_params, 8192)) == NULL) { + char *const parsed = urlParse(url, cfg->sig_anchor, new_path, 8192, path_params, 8192); + if (NULL == parsed) { err_log(url, "Has no signing query string or signing path parameters."); goto deny; } + + url = parsed; has_path_params = true; query = strstr(url, ";"); From 334d6c736c31fb132db8bc30350942c600b53ff3 Mon Sep 17 00:00:00 2001 From: Valentin Gutierrez Date: Wed, 3 Apr 2019 10:07:54 +0200 Subject: [PATCH 406/526] Fix mysql-remap compilation error It looks like d77cd73166c1b27cde61ffb3c8a1545c437fad83 messed with the fix applied in 6e35780e43d7313d18b7ee260e3e0c239d917104 --- plugins/experimental/mysql_remap/lib/iniparser.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/experimental/mysql_remap/lib/iniparser.h b/plugins/experimental/mysql_remap/lib/iniparser.h index 546a41fbea9..ca415becc42 100644 --- a/plugins/experimental/mysql_remap/lib/iniparser.h +++ b/plugins/experimental/mysql_remap/lib/iniparser.h @@ -42,9 +42,9 @@ Includes ---------------------------------------------------------------------------*/ -#include -#include -#include +#include // NOLINT(modernize-deprecated-headers) +#include // NOLINT(modernize-deprecated-headers) +#include // NOLINT(modernize-deprecated-headers) /* * The following #include is necessary on many Unixes but not Linux. From cdd6481aabb97bc1b620f5d668f04a168c246ea1 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Mon, 11 Feb 2019 18:03:19 -0600 Subject: [PATCH 407/526] Some tidying up of the global namespace. --- include/tscore/TSSystemState.h | 92 ++++++++++++++++++++++++++++ iocore/aio/AIO.cc | 4 +- iocore/aio/test_AIO.cc | 3 +- iocore/eventsystem/I_EThread.h | 2 - iocore/eventsystem/UnixEThread.cc | 6 +- iocore/eventsystem/test_Event.cc | 3 +- iocore/net/SSLNetVConnection.cc | 5 +- iocore/net/UnixNetAccept.cc | 4 +- iocore/net/UnixNetProcessor.cc | 3 +- mgmt/ProcessManager.cc | 5 +- proxy/ProxyClientSession.h | 7 +-- proxy/logging/Log.cc | 5 +- src/traffic_server/traffic_server.cc | 22 +++---- 13 files changed, 126 insertions(+), 35 deletions(-) create mode 100644 include/tscore/TSSystemState.h diff --git a/include/tscore/TSSystemState.h b/include/tscore/TSSystemState.h new file mode 100644 index 00000000000..cc5bf3d1229 --- /dev/null +++ b/include/tscore/TSSystemState.h @@ -0,0 +1,92 @@ +/** + @file TSSystemState.h + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +#pragma once + +#include +#include + +// Global status information for trafficserver +// +class TSSystemState +{ +private: + struct Data { + bool ssl_handshaking_stopped; + bool event_system_shut_down; + bool draining; + }; + +public: + static bool + is_ssl_handshaking_stopped() + { + return unlikely(_instance().ssl_handshaking_stopped); + } + + static bool + is_event_system_shut_down() + { + return unlikely(_instance().event_system_shut_down); + } + + // Keeps track if the server is in draining state, follows the proxy.node.config.draining metric. + // + static bool + is_draining() + { + return unlikely(_instance().draining); + } + + static void + stop_ssl_handshaking() + { + ink_assert(!_instance().ssl_handshaking_stopped); + + _instance().ssl_handshaking_stopped = true; + } + + static void + shut_down_event_system() + { + // For some reason this is triggered by the regression testing. + // ink_assert(_instance().ssl_handshaking_stopped && !_instance().event_system_shut_down); + + _instance().event_system_shut_down = true; + } + + static void + drain(bool enable) + { + _instance().draining = enable; + } + +private: + static Data & + _instance() + { + static Data d; + + return d; + } +}; diff --git a/iocore/aio/AIO.cc b/iocore/aio/AIO.cc index ca7860cd72b..29deaf65e87 100644 --- a/iocore/aio/AIO.cc +++ b/iocore/aio/AIO.cc @@ -25,6 +25,8 @@ * Async Disk IO operations. */ +#include + #include "P_AIO.h" #if AIO_MODE == AIO_MODE_NATIVE @@ -457,7 +459,7 @@ aio_thread_main(void *arg) ink_mutex_acquire(&my_aio_req->aio_mutex); for (;;) { do { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { ink_mutex_release(&my_aio_req->aio_mutex); return nullptr; } diff --git a/iocore/aio/test_AIO.cc b/iocore/aio/test_AIO.cc index 4292e6de55a..4bc2c237bf0 100644 --- a/iocore/aio/test_AIO.cc +++ b/iocore/aio/test_AIO.cc @@ -24,6 +24,7 @@ #include "P_AIO.h" #include "InkAPIInternal.h" #include "tscore/I_Layout.h" +#include "tscore/TSSystemState.h" #include #include @@ -469,7 +470,7 @@ main(int /* argc ATS_UNUSED */, char *argv[]) } } - while (!shutdown_event_system) { + while (!TSSystemState::is_event_system_shut_down()) { sleep(1); } delete main_thread; diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index 9a1bbcd0511..a0518fd24c6 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -51,8 +51,6 @@ enum ThreadType { DEDICATED, }; -extern bool shutdown_event_system; - /** Event System specific type of thread. diff --git a/iocore/eventsystem/UnixEThread.cc b/iocore/eventsystem/UnixEThread.cc index 06920dbe5b7..d0f896d3412 100644 --- a/iocore/eventsystem/UnixEThread.cc +++ b/iocore/eventsystem/UnixEThread.cc @@ -21,6 +21,8 @@ limitations under the License. */ +#include + ////////////////////////////////////////////////////////////////////// // // The EThread Class @@ -45,8 +47,6 @@ char const *const EThread::STAT_NAME[] = {"proxy.process.eventloop.count", int const EThread::SAMPLE_COUNT[N_EVENT_TIMESCALES] = {10, 100, 1000}; -bool shutdown_event_system = false; - int thread_max_heartbeat_mseconds = THREAD_MAX_HEARTBEAT_MSECONDS; EThread::EThread() @@ -209,7 +209,7 @@ EThread::execute_regular() // give priority to immediate events for (;;) { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return; } diff --git a/iocore/eventsystem/test_Event.cc b/iocore/eventsystem/test_Event.cc index 714aff534b9..78c47bb3624 100644 --- a/iocore/eventsystem/test_Event.cc +++ b/iocore/eventsystem/test_Event.cc @@ -23,6 +23,7 @@ #include "I_EventSystem.h" #include "tscore/I_Layout.h" +#include "tscore/TSSystemState.h" #include "diags.i" @@ -75,7 +76,7 @@ main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) process_killer *killer = new process_killer(new_ProxyMutex()); eventProcessor.schedule_in(killer, HRTIME_SECONDS(10)); eventProcessor.schedule_every(alrm, HRTIME_SECONDS(1)); - while (!shutdown_event_system) { + while (!TSSystemState::is_event_system_shut_down()) { sleep(1); } return 0; diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index bc3cd8ec459..070e4141ab2 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -24,6 +24,7 @@ #include "tscore/ink_config.h" #include "tscore/EventNotify.h" #include "tscore/I_Layout.h" +#include "tscore/TSSystemState.h" #include "records/I_RecHttp.h" #include "InkAPIInternal.h" // Added to include the ssl_hook definitions @@ -77,8 +78,6 @@ void SSL_set0_rbio(SSL *ssl, BIO *rbio); ClassAllocator sslNetVCAllocator("sslNetVCAllocator"); -bool stop_ssl_handshake = false; - namespace { /// Callback to get two locks. @@ -961,7 +960,7 @@ SSLNetVConnection::free(EThread *t) int SSLNetVConnection::sslStartHandShake(int event, int &err) { - if (stop_ssl_handshake) { + if (TSSystemState::is_ssl_handshaking_stopped()) { Debug("ssl", "Stopping handshake due to server shutting down."); return EVENT_ERROR; } diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index a1141e4c48b..2a2c2f7f5fc 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -21,6 +21,8 @@ limitations under the License. */ +#include + #include "P_Net.h" #ifdef ROUNDUP @@ -333,7 +335,7 @@ NetAccept::do_blocking_accept(EThread *t) return -1; } - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return -1; } diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index e9896d52a6e..488f6db6178 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -24,6 +24,7 @@ #include "P_Net.h" #include "tscore/InkErrno.h" #include "tscore/ink_sock.h" +#include "tscore/TSSystemState.h" #include "P_SSLNextProtocolAccept.h" // For Stat Pages @@ -205,7 +206,7 @@ NetProcessor::stop_accept() Action * UnixNetProcessor::connect_re_internal(Continuation *cont, sockaddr const *target, NetVCOptions *opt) { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return ACTION_RESULT_NONE; } EThread *t = cont->mutex->thread_holding; diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc index 35d2b3cbfc3..abfcb9e48c2 100644 --- a/mgmt/ProcessManager.cc +++ b/mgmt/ProcessManager.cc @@ -25,6 +25,7 @@ #include "ProcessManager.h" #include "tscore/ink_apidefs.h" +#include "tscore/TSSystemState.h" #include "MgmtSocket.h" #include "tscore/I_Layout.h" @@ -166,13 +167,13 @@ ProcessManager::processManagerThread(void *arg) if (pmgmt->require_lm) { ret = pmgmt->pollLMConnection(); - if (ret < 0 && pmgmt->running && !shutdown_event_system) { + if (ret < 0 && pmgmt->running && !TSSystemState::is_event_system_shut_down()) { Alert("exiting with read error from process manager: %s", strerror(-ret)); } } ret = pmgmt->processSignalQueue(); - if (ret < 0 && pmgmt->running && !shutdown_event_system) { + if (ret < 0 && pmgmt->running && !TSSystemState::is_event_system_shut_down()) { Alert("exiting with write error from process manager: %s", strerror(-ret)); } } diff --git a/proxy/ProxyClientSession.h b/proxy/ProxyClientSession.h index 241da9b205f..7c6acf9f6a9 100644 --- a/proxy/ProxyClientSession.h +++ b/proxy/ProxyClientSession.h @@ -25,6 +25,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_resolver.h" +#include "tscore/TSSystemState.h" #include #include "P_Net.h" #include "InkAPIInternal.h" @@ -69,9 +70,7 @@ struct ProxyError { uint32_t code = 0; }; -// A little ugly, but this global is tracked by traffic_server. -extern bool ts_is_draining; - +/// Abstract class for HttpSM to interface with any session class ProxyClientSession : public VConnection { public: @@ -149,7 +148,7 @@ class ProxyClientSession : public VConnection bool is_draining() const { - return ts_is_draining; + return TSSystemState::is_draining(); } // Initiate an API hook invocation. diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc index 04038f19747..e3870ac447a 100644 --- a/proxy/logging/Log.cc +++ b/proxy/logging/Log.cc @@ -33,6 +33,7 @@ ***************************************************************************/ #include "tscore/ink_platform.h" +#include "tscore/TSSystemState.h" #include "P_EventSystem.h" #include "P_Net.h" #include "I_Machine.h" @@ -1248,7 +1249,7 @@ Log::preproc_thread_main(void *args) Log::preproc_notify[idx].lock(); while (true) { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return nullptr; } size_t buffers_preproced = 0; @@ -1291,7 +1292,7 @@ Log::flush_thread_main(void * /* args ATS_UNUSED */) Log::flush_notify->lock(); while (true) { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return nullptr; } fdata = (LogFlushData *)ink_atomiclist_popall(flush_data_list); diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 092745010dc..689a2eec34c 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -171,12 +171,6 @@ static bool signal_received[NSIG]; // -1: cache is already initialized, don't delay. static int delay_listen_for_cache_p; -// Keeps track if the server is in draining state, follows the proxy.node.config.draining metric -bool ts_is_draining = false; - -// Flag to stop ssl handshakes during shutdown. -extern bool stop_ssl_handshake; - AppVersionInfo appVersionInfo; // Build info for this application static ArgumentDescription argument_descriptions[] = { @@ -226,7 +220,7 @@ struct AutoStopCont : public Continuation { int mainEvent(int /* event */, Event * /* e */) { - stop_ssl_handshake = true; + TSSystemState::stop_ssl_handshaking(); APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_SHUTDOWN_HOOK); while (hook) { @@ -236,7 +230,7 @@ struct AutoStopCont : public Continuation { } pmgmt->stop(); - shutdown_event_system = true; + TSSystemState::shut_down_event_system(); delete this; return EVENT_CONT; } @@ -294,7 +288,7 @@ class SignalContinuation : public Continuation RecInt timeout = 0; if (RecGetRecordInt("proxy.config.stop.shutdown_timeout", &timeout) == REC_ERR_OKAY && timeout) { RecSetRecordInt("proxy.node.config.draining", 1, REC_SOURCE_DEFAULT); - ts_is_draining = true; + TSSystemState::drain(true); if (!remote_management_flag) { // Close listening sockets here only if TS is running standalone RecInt close_sockets = 0; @@ -1365,7 +1359,7 @@ struct RegressionCont : public Continuation { return EVENT_CONT; } - shutdown_event_system = true; + TSSystemState::shut_down_event_system(); fprintf(stderr, "REGRESSION_TEST DONE: %s\n", regression_status_string(res)); ::exit(res == REGRESSION_TEST_PASSED ? 0 : 1); return EVENT_CONT; @@ -2002,7 +1996,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) } #endif - while (!shutdown_event_system) { + while (!TSSystemState::is_event_system_shut_down()) { sleep(1); } @@ -2032,9 +2026,9 @@ static void mgmt_restart_shutdown_callback(ts::MemSpan) static void mgmt_drain_callback(ts::MemSpan span) { - char *arg = static_cast(span.data()); - ts_is_draining = (span.size() == 2 && arg[0] == '1'); - RecSetRecordInt("proxy.node.config.draining", ts_is_draining ? 1 : 0, REC_SOURCE_DEFAULT); + char *arg = static_cast(span.data()); + TSSystemState::drain(span.size() == 2 && arg[0] == '1'); + RecSetRecordInt("proxy.node.config.draining", TSSystemState::is_draining() ? 1 : 0, REC_SOURCE_DEFAULT); } static void From 8ec536bf5a2c35a95bea9e8c5028f554e192f973 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Fri, 5 Apr 2019 17:10:36 -0700 Subject: [PATCH 408/526] Setting the correct directory for test_Cache --- include/tscore/ink_config.h.in | 2 ++ iocore/cache/test/main.cc | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/include/tscore/ink_config.h.in b/include/tscore/ink_config.h.in index 3e3aa288780..53b02db899b 100644 --- a/include/tscore/ink_config.h.in +++ b/include/tscore/ink_config.h.in @@ -118,6 +118,8 @@ #define TS_BUILD_CACHEDIR "@rel_cachedir@" #define TS_BUILD_INFODIR "@rel_infodir@" +#define TS_ABS_TOP_SRCDIR "@abs_top_srcdir@" + #define TS_BUILD_CANONICAL_HOST "@host@" #define TS_BUILD_DEFAULT_LOOPBACK_IFACE "@default_loopback_iface@" diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc index c62cd3f5fe1..777d9cabde5 100644 --- a/iocore/cache/test/main.cc +++ b/iocore/cache/test/main.cc @@ -65,8 +65,9 @@ struct EventProcessorListener : Catch::TestEventListenerBase { thread->set_specific(); init_buffer_allocators(0); - Layout::get()->sysconfdir = std::string_view("./test"); - Layout::get()->prefix = std::string_view("./test"); + std::string src_dir = std::string(TS_ABS_TOP_SRCDIR) + "/iocore/cache/test"; + Layout::get()->sysconfdir = src_dir; + Layout::get()->prefix = src_dir; ::remove("./test/var/trafficserver/cache.db"); } }; From 233aa44353c90a94830455f68bb7ad18635e80ac Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Mon, 25 Feb 2019 14:57:50 -0800 Subject: [PATCH 409/526] url_sig: fixed unit-test for remapped url. When the plugin does not use @pparam=pristineurl it should work with the remapped url. The code already does this but the unit-test still tests the signing of the pristine url in those specific use cases. This is related to a behavior change with v9.0 where the 1st plugin gets the remapped url as a remap API requestUrl regardless of its possition in the plugin chain (PR #4964). --- .../pluginTest/url_sig/url_sig.test.py | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py index e4a5819a36e..3b83837e2b0 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib +import hmac import os import subprocess Test.Summary = ''' @@ -194,16 +196,6 @@ # Success tests. -# No client / SHA1 / P=1 / URL not pristine / URL not altered. -# -tr = Test.AddTestRun() -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Command = ( - "curl --verbose --proxy http://127.0.0.1:{} 'http://one.two.three/".format(ts.Variables.port) + - "foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=acae22b0e1ba6ea6fbb5d26018dbf152558e98cb'" + - LogTee -) - # With client / SHA1 / P=1 / URL pristine / URL not altered. # tr = Test.AddTestRun() @@ -244,13 +236,34 @@ LogTee ) +def sign(payload, key): + secret=bytes(key,'utf-8') + data=bytes(payload, 'utf-8') + md=bytes(hmac.new(secret, data, digestmod=hashlib.sha1).digest().hex(), 'utf-8') + return md.decode("utf-8") + +# No client / SHA1 / P=1 / URL not pristine / URL not altered. +# +path="foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=" +to_sign="127.0.0.1:{}/".format(server.Variables.Port) + path +url="http://one.two.three/" + path + sign(to_sign, "dqsgopTSM_doT6iAysasQVUKaPykyb6e") + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = ( + "curl --verbose --proxy http://127.0.0.1:{} '{}'".format(ts.Variables.port, url) + LogTee +) + # No client / SHA1 / P=1 / URL not pristine / URL not altered -- HTTPS. # +path="foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=" +to_sign="127.0.0.1:{}/".format(server.Variables.Port) + path +url="https://127.0.0.1:{}/".format(ts.Variables.ssl_port) + path + sign(to_sign, "dqsgopTSM_doT6iAysasQVUKaPykyb6e") + tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Command = ( - "curl --verbose --http1.1 --insecure --header 'Host: one.two.three' 'https://127.0.0.1:{}/".format(ts.Variables.ssl_port) + - "foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=acae22b0e1ba6ea6fbb5d26018dbf152558e98cb'" + + "curl --verbose --http1.1 --insecure --header 'Host: one.two.three' '{}'".format(url) + LogTee + " ; grep -F -e '< HTTP' -e Authorization {0}/url_sig_long.log > {0}/url_sig_short.log ".format(ts.RunDirectory) ) From cc7e10282b3f15c3b8cf8da1a1450ff304d125db Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Mon, 8 Apr 2019 10:31:40 -0700 Subject: [PATCH 410/526] Fixed cache test, using updated shutdown --- iocore/cache/test/main.cc | 2 +- iocore/cache/test/main.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc index 777d9cabde5..578314dd3d0 100644 --- a/iocore/cache/test/main.cc +++ b/iocore/cache/test/main.cc @@ -31,7 +31,7 @@ char working_dir[1024] = {0}; void test_done() { - shutdown_event_system = true; + TSSystemState::shut_down_event_system(); } const char *GLOBAL_DATA = (char *)ats_malloc(10 * 1024 * 1024 + 3); // 10M diff --git a/iocore/cache/test/main.h b/iocore/cache/test/main.h index b4dc4ee4544..e661faaca38 100644 --- a/iocore/cache/test/main.h +++ b/iocore/cache/test/main.h @@ -29,6 +29,7 @@ #include "tscore/I_Layout.h" #include "tscore/Diags.h" +#include "tscore/TSSystemState.h" #include "RecordsConfig.h" #include "records/I_RecProcess.h" From b20ba1e2d3d3a76829070ee9802cfc3d9cb1cd09 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Sun, 7 Apr 2019 12:32:31 -0600 Subject: [PATCH 411/526] Adds a missing header field for the tcpinfo log --- plugins/tcpinfo/tcpinfo.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/tcpinfo/tcpinfo.cc b/plugins/tcpinfo/tcpinfo.cc index dccb4c22300..588ba3424f5 100644 --- a/plugins/tcpinfo/tcpinfo.cc +++ b/plugins/tcpinfo/tcpinfo.cc @@ -57,7 +57,7 @@ static const char *tcpi_headers[] = { "timestamp event client server rtt", "timestamp event client server rtt rttvar last_sent last_recv " - "snd_ssthresh rcv_ssthresh unacked sacked lost retrans fackets all_retrans", + "snd_cwnd snd_ssthresh rcv_ssthresh unacked sacked lost retrans fackets all_retrans", }; struct Config { From 2c3dd8dd62116c0df3326b69d045a30daa47aad0 Mon Sep 17 00:00:00 2001 From: ezelko260 Date: Sat, 6 Apr 2019 00:31:36 +0000 Subject: [PATCH 412/526] Changed how current age is determined to age out documents. Guaranteed freshness was being used innapropriately when the docs age was already beyond that value --- proxy/http/HttpTransact.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index a85263d3959..ed30844b4c4 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -7211,11 +7211,16 @@ HttpTransact::what_is_document_freshness(State *s, HTTPHdr *client_request, HTTP current_age = HttpTransactHeaders::calculate_document_age(s->request_sent_time, s->response_received_time, cached_obj_response, response_date, s->current.now); - // Overflow ? + // First check overflow status + // Second if current_age is under the max, use the smaller value + // Finally we take the max of current age or guaranteed max, this ensures it will + // age out properly, otherwise a doc will never expire if guaranteed < document max-age if (current_age < 0) { current_age = s->txn_conf->cache_guaranteed_max_lifetime; - } else { + } else if (current_age < s->txn_conf->cache_guaranteed_max_lifetime) { current_age = std::min((time_t)s->txn_conf->cache_guaranteed_max_lifetime, current_age); + } else { + current_age = std::max((time_t)s->txn_conf->cache_guaranteed_max_lifetime, current_age); } TxnDebug("http_match", "[what_is_document_freshness] fresh_limit: %d current_age: %" PRId64, fresh_limit, (int64_t)current_age); From 5c643b6b6d66c528066fa426719c4fd788743920 Mon Sep 17 00:00:00 2001 From: John Rushford Date: Mon, 8 Apr 2019 22:32:25 +0000 Subject: [PATCH 413/526] DRAFT: This PR fixes #5248 - parent host status is not persistent accross restarts. --- src/traffic_server/HostStatus.cc | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index 993feb363b3..d13e6fe3d86 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -94,6 +94,30 @@ mgmt_host_status_down_callback(ts::MemSpan span) } } +static void +handle_record_read(const RecRecord *rec, void *edata) +{ + HostStatus &hs = HostStatus::instance(); + std::string hostname; + std::string reason; + + if (rec) { + // parse the hostname from the stat name + + char *s = const_cast(rec->name); + // 1st move the pointer past the stat prefix. + s += strlen(stat_prefix.c_str()); + hostname = s; + reason = hostname.substr(hostname.find('_')); + reason.erase(0, 1); + // erase the reason tag + hostname.erase(hostname.find('_')); + if (Reasons::validReason(reason.c_str())) { + hs.setHostStatus(hostname.c_str(), HOST_STATUS_DOWN, 0, reason.c_str()); + } + } +} + HostStatus::HostStatus() { ink_rwlock_init(&host_status_rwlock); @@ -101,6 +125,10 @@ HostStatus::HostStatus() pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, &mgmt_host_status_up_callback); pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, &mgmt_host_status_down_callback); host_status_rsb = RecAllocateRawStatBlock((int)TS_MAX_API_STATS); + + if (RecLookupMatchingRecords(RECT_ALL, stat_prefix.c_str(), handle_record_read, nullptr) != REC_ERR_OKAY) { + Error("[HostStatus] - Error reading stats."); + } } HostStatus::~HostStatus() @@ -203,7 +231,7 @@ HostStatus::createHostStat(const char *name) std::string reason_stat; getStatName(reason_stat, name, i); if (hosts_stats_ids.find(reason_stat) == hosts_stats_ids.end()) { - RecRegisterRawStat(host_status_rsb, RECT_PROCESS, (reason_stat).c_str(), RECD_INT, RECP_NON_PERSISTENT, (int)next_stat_id, + RecRegisterRawStat(host_status_rsb, RECT_PROCESS, (reason_stat).c_str(), RECD_INT, RECP_PERSISTENT, (int)next_stat_id, RecRawStatSyncSum); RecSetRawStatCount(host_status_rsb, next_stat_id, 1); RecSetRawStatSum(host_status_rsb, next_stat_id, 1); From c1f8435ef12623a9a9f5b57432a518867a39a7ff Mon Sep 17 00:00:00 2001 From: John Rushford Date: Tue, 9 Apr 2019 17:19:35 +0000 Subject: [PATCH 414/526] Adds a HostStatus::loadStats() function used to load persisted HostStatus stats after a restart of trafficserver. --- proxy/HostStatus.h | 1 + proxy/ParentSelection.cc | 6 ++++++ src/traffic_server/HostStatus.cc | 22 +++++++++++++++------- src/traffic_server/traffic_server.cc | 2 +- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/proxy/HostStatus.h b/proxy/HostStatus.h index 486b8f3c36a..f349b903f8b 100644 --- a/proxy/HostStatus.h +++ b/proxy/HostStatus.h @@ -86,6 +86,7 @@ struct HostStatus { void setHostStatus(const char *name, const HostStatus_t status, const unsigned int down_time, const char *reason); HostStatus_t getHostStatus(const char *name); void createHostStat(const char *name); + void loadHostStatusFromStats(); int getHostStatId(const char *name); private: diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index d8b64d43dc4..6d3d0e55992 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -1782,6 +1782,12 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, FP; RE(verify(result, PARENT_SPECIFIED, "carol", 80), 211); + // cleanup, allow changes to be persisted to records.snap + // so that subsequent test runs do not fail unexpectedly. + _st.setHostStatus("curly", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); + sleep(2); + delete request; delete result; delete params; diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index d13e6fe3d86..fdbd6b9b08d 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -103,16 +103,20 @@ handle_record_read(const RecRecord *rec, void *edata) if (rec) { // parse the hostname from the stat name - char *s = const_cast(rec->name); // 1st move the pointer past the stat prefix. s += strlen(stat_prefix.c_str()); hostname = s; - reason = hostname.substr(hostname.find('_')); + // parse the reason from the stat name. + reason = hostname.substr(hostname.find('_')); reason.erase(0, 1); // erase the reason tag hostname.erase(hostname.find('_')); - if (Reasons::validReason(reason.c_str())) { + + // if the data loaded from stats indicates that the host was down, + // then update the state so that the host remains down until + // specificcaly marked up using traffic_ctl. + if (rec->data.rec_int == 0 && Reasons::validReason(reason.c_str())) { hs.setHostStatus(hostname.c_str(), HOST_STATUS_DOWN, 0, reason.c_str()); } } @@ -125,10 +129,6 @@ HostStatus::HostStatus() pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, &mgmt_host_status_up_callback); pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, &mgmt_host_status_down_callback); host_status_rsb = RecAllocateRawStatBlock((int)TS_MAX_API_STATS); - - if (RecLookupMatchingRecords(RECT_ALL, stat_prefix.c_str(), handle_record_read, nullptr) != REC_ERR_OKAY) { - Error("[HostStatus] - Error reading stats."); - } } HostStatus::~HostStatus() @@ -141,6 +141,14 @@ HostStatus::~HostStatus() ink_rwlock_destroy(&host_statids_rwlock); } +void +HostStatus::loadHostStatusFromStats() +{ + if (RecLookupMatchingRecords(RECT_ALL, stat_prefix.c_str(), handle_record_read, nullptr) != REC_ERR_OKAY) { + Error("[HostStatus] - While loading HostStatus stats, there was an Error reading HostStatus stats."); + } +} + void HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned int down_time, const char *reason) { diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 689a2eec34c..9c3c4bcf452 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -1864,8 +1864,8 @@ main(int /* argc ATS_UNUSED */, const char **argv) RecProcessStart(); initCacheControl(); IpAllow::startup(); + HostStatus::instance().loadHostStatusFromStats(); ParentConfig::startup(); - HostStatus::instance(); #ifdef SPLIT_DNS SplitDNSConfig::startup(); #endif From d8191b6e6f6e149c149347244ef38a50d77d2f21 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Tue, 9 Apr 2019 10:51:30 -0700 Subject: [PATCH 415/526] Remove unused variable in cache test --- iocore/cache/test/main.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc index 578314dd3d0..f0425d9147f 100644 --- a/iocore/cache/test/main.cc +++ b/iocore/cache/test/main.cc @@ -27,7 +27,6 @@ #define THREADS 1 #define DIAGS_LOG_FILE "diags.log" -char working_dir[1024] = {0}; void test_done() { @@ -42,7 +41,6 @@ struct EventProcessorListener : Catch::TestEventListenerBase { virtual void testRunStarting(Catch::TestRunInfo const &testRunInfo) override { - getcwd(working_dir, 1024); BaseLogFile *base_log_file = new BaseLogFile("stderr"); diags = new Diags(testRunInfo.name.c_str(), "*" /* tags */, "" /* actions */, base_log_file); diags->activate_taglist("cache.*|agg.*|locks", DiagsTagType_Debug); From b5fd8ac20fedb275cd76a1b03ba6625a173738e3 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Wed, 10 Apr 2019 10:44:27 -0700 Subject: [PATCH 416/526] Fixed pthread mutex init issue with cache test --- iocore/cache/test/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc index f0425d9147f..a55394037b4 100644 --- a/iocore/cache/test/main.cc +++ b/iocore/cache/test/main.cc @@ -54,6 +54,7 @@ struct EventProcessorListener : Catch::TestEventListenerBase { ink_net_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); ink_assert(GLOBAL_DATA != nullptr); + statPagesManager.init(); // mutex needs to be initialized before calling netProcessor.init netProcessor.init(); eventProcessor.start(THREADS); From 6406ef8c66b3dd009af3e2f2b8f3cd6e409fa2cf Mon Sep 17 00:00:00 2001 From: scw00 Date: Wed, 10 Apr 2019 01:46:00 +0000 Subject: [PATCH 417/526] Adds cache alterante update tests --- iocore/cache/Makefile.am | 18 ++- iocore/cache/test/main.cc | 7 +- iocore/cache/test/main.h | 1 + iocore/cache/test/test_Update_L_to_S.cc | 155 ++++++++++++++++++++++++ iocore/cache/test/test_Update_S_to_L.cc | 155 ++++++++++++++++++++++++ 5 files changed, 334 insertions(+), 2 deletions(-) create mode 100644 iocore/cache/test/test_Update_L_to_S.cc create mode 100644 iocore/cache/test/test_Update_S_to_L.cc diff --git a/iocore/cache/Makefile.am b/iocore/cache/Makefile.am index a9993a37024..b5d7c5d9729 100644 --- a/iocore/cache/Makefile.am +++ b/iocore/cache/Makefile.am @@ -127,7 +127,9 @@ check_PROGRAMS = \ test_Alternate_L_to_S_remove_L \ test_Alternate_L_to_S_remove_S \ test_Alternate_S_to_L_remove_S \ - test_Alternate_S_to_L_remove_L + test_Alternate_S_to_L_remove_L \ + test_Update_L_to_S \ + test_Update_S_to_L test_main_SOURCES = \ ./test/main.cc \ @@ -190,6 +192,20 @@ test_Alternate_S_to_L_remove_L_SOURCES = \ $(test_main_SOURCES) \ ./test/test_Alternate_S_to_L_remove_L.cc +test_Update_L_to_S_CPPFLAGS = $(test_CPPFLAGS) +test_Update_L_to_S_LDFLAGS = @AM_LDFLAGS@ +test_Update_L_to_S_LDADD = $(test_LDADD) +test_Update_L_to_S_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Update_L_to_S.cc + +test_Update_S_to_L_CPPFLAGS = $(test_CPPFLAGS) +test_Update_S_to_L_LDFLAGS = @AM_LDFLAGS@ +test_Update_S_to_L_LDADD = $(test_LDADD) +test_Update_S_to_L_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Update_S_to_L.cc + include $(top_srcdir)/build/tidy.mk clang-tidy-local: $(DIST_SOURCES) diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc index a55394037b4..c9427c464f0 100644 --- a/iocore/cache/test/main.cc +++ b/iocore/cache/test/main.cc @@ -204,8 +204,13 @@ CacheWriteTest::start_test(int event, void *e) HttpCacheKey key; key = generate_key(this->info); + HTTPInfo *old_info = &this->old_info; + if (!old_info->valid()) { + old_info = nullptr; + } + SET_HANDLER(&CacheWriteTest::write_event); - cacheProcessor.open_write(this, 0, &key, (CacheHTTPHdr *)this->info.request_get(), nullptr); + cacheProcessor.open_write(this, 0, &key, (CacheHTTPHdr *)this->info.request_get(), old_info); return 0; } diff --git a/iocore/cache/test/main.h b/iocore/cache/test/main.h index e661faaca38..5bcc4f82a2f 100644 --- a/iocore/cache/test/main.h +++ b/iocore/cache/test/main.h @@ -197,6 +197,7 @@ class CacheWriteTest : public CacheTestBase void do_io_write(size_t size = 0) override; HTTPInfo info; + HTTPInfo old_info; private: size_t _size = 0; diff --git a/iocore/cache/test/test_Update_L_to_S.cc b/iocore/cache/test/test_Update_L_to_S.cc new file mode 100644 index 00000000000..1172c816218 --- /dev/null +++ b/iocore/cache/test/test_Update_L_to_S.cc @@ -0,0 +1,155 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +class CacheUpdateReadAgain : public CacheTestHandler +{ +public: + CacheUpdateReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheUpdateReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } +}; + +class CacheUpdate_L_to_S : public CacheTestHandler +{ +public: + CacheUpdate_L_to_S(size_t read_size, size_t write_size, const char *url) + { + this->_rt = new CacheReadTest(read_size, this, url); + this->_wt = new CacheWriteTest(write_size, this, url); + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheUpdate_L_to_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + CacheWriteTest *wt = static_cast(this->_wt); + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + delete this; + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + wt->old_info.copy(static_cast(&base->vc->alternate)); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + this->_rt->close(); + this->_rt = nullptr; + this_ethread()->schedule_imm(this->_wt); + break; + default: + REQUIRE(false); + break; + } + } +}; + +class CacheUpdateInit : public CacheInit +{ +public: + CacheUpdateInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE, "http://www.scw11.com"); + CacheUpdate_L_to_S *update = new CacheUpdate_L_to_S(LARGE_FILE, SMALL_FILE, "http://www.scw11.com"); + CacheUpdateReadAgain *read = new CacheUpdateReadAgain(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(update); + h->add(read); // read again + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheUpdateInit *init = new CacheUpdateInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Update_S_to_L.cc b/iocore/cache/test/test_Update_S_to_L.cc new file mode 100644 index 00000000000..2944c33f4f3 --- /dev/null +++ b/iocore/cache/test/test_Update_S_to_L.cc @@ -0,0 +1,155 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +class CacheUpdateReadAgain : public CacheTestHandler +{ +public: + CacheUpdateReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheUpdateReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } +}; + +class CacheUpdate_S_to_L : public CacheTestHandler +{ +public: + CacheUpdate_S_to_L(size_t read_size, size_t write_size, const char *url) + { + this->_rt = new CacheReadTest(read_size, this, url); + this->_wt = new CacheWriteTest(write_size, this, url); + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheUpdate_S_to_L::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + CacheWriteTest *wt = static_cast(this->_wt); + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + delete this; + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + wt->old_info.copy(static_cast(&base->vc->alternate)); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + this->_rt->close(); + this->_rt = nullptr; + this_ethread()->schedule_imm(this->_wt); + break; + default: + REQUIRE(false); + break; + } + } +}; + +class CacheUpdateInit : public CacheInit +{ +public: + CacheUpdateInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + CacheUpdate_S_to_L *update = new CacheUpdate_S_to_L(SMALL_FILE, LARGE_FILE, "http://www.scw11.com"); + CacheUpdateReadAgain *read = new CacheUpdateReadAgain(LARGE_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(update); + h->add(read); // read again + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheUpdateInit *init = new CacheUpdateInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} From 076e1f3d85296fff7c25f60ede5a6b9c6a928c0d Mon Sep 17 00:00:00 2001 From: scw00 Date: Wed, 10 Apr 2019 04:54:46 +0000 Subject: [PATCH 418/526] Fixed cache RWW test crash --- iocore/cache/test/test_RWW.cc | 72 ++++++++++++++++------------------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/iocore/cache/test/test_RWW.cc b/iocore/cache/test/test_RWW.cc index 167cf1ee7b4..ee86681d58b 100644 --- a/iocore/cache/test/test_RWW.cc +++ b/iocore/cache/test/test_RWW.cc @@ -30,26 +30,6 @@ class CacheRWWTest; -struct SimpleCont : public Continuation { - SimpleCont(CacheTestBase *base) : base(base) - { - REQUIRE(base != nullptr); - SET_HANDLER(&SimpleCont::handle_event); - this->mutex = base->mutex; - } - - int - handle_event(int event, void *data) - { - Debug("cache_rww_test", "cache write reenable"); - base->reenable(); - delete this; - return 0; - } - - CacheTestBase *base = nullptr; -}; - class CacheRWWTest : public CacheTestHandler { public: @@ -97,11 +77,13 @@ class CacheRWWTest : public CacheTestHandler } protected: - size_t _size = 0; - Event *_read_event = nullptr; - bool _is_read_start = false; - CacheTestBase *_rt = nullptr; - CacheTestBase *_wt = nullptr; + // start at 1 framgents + int64_t _latest_fragments = 1; + size_t _size = 0; + Event *_read_event = nullptr; + bool _is_read_start = false; + CacheTestBase *_rt = nullptr; + CacheTestBase *_wt = nullptr; }; int @@ -122,7 +104,6 @@ CacheRWWTest::process_write_event(int event, CacheTestBase *base) case VC_EVENT_WRITE_READY: // schedule read test imm if (this->_size != SMALL_FILE && !this->_wt->vc->fragment) { - Debug("cache_rww_test", "cache write reenable"); base->reenable(); return; } @@ -134,13 +115,17 @@ CacheRWWTest::process_write_event(int event, CacheTestBase *base) return; } - // stop writing for a while and wait for reading - // data->vio->reenable(); - this_ethread()->schedule_imm(new SimpleCont(base)); + // write at least one fragment before read it + if (this->_latest_fragments == this->_wt->vc->fragment) { + base->reenable(); + return; + } + + this->_latest_fragments = this->_wt->vc->fragment; + this->_rt->reenable(); break; case VC_EVENT_WRITE_COMPLETE: this->close_write(); - break; default: REQUIRE(event == 0); @@ -149,10 +134,6 @@ CacheRWWTest::process_write_event(int event, CacheTestBase *base) this->close_read(); return; } - - if (this->_rt) { - this->_rt->reenable(); - } } void @@ -166,7 +147,6 @@ CacheRWWTest::process_read_event(int event, CacheTestBase *base) Debug("cache_rww_test", "cache read reenable"); this->_read_event = nullptr; this->_is_read_start = true; - base->reenable(); break; case VC_EVENT_READ_COMPLETE: this->close_read(); @@ -245,7 +225,15 @@ class CacheRWWErrorTest : public CacheRWWTest this->close_write(100); return; } - this_ethread()->schedule_imm(new SimpleCont(base)); + + // write at least one fragment before read it + if (this->_latest_fragments == this->_wt->vc->fragment) { + base->reenable(); + return; + } + + this->_latest_fragments = this->_wt->vc->fragment; + this->_rt->reenable(); break; case VC_EVENT_WRITE_COMPLETE: @@ -274,7 +262,6 @@ class CacheRWWErrorTest : public CacheRWWTest this->close_read(); return; case VC_EVENT_READ_READY: - base->reenable(); if (this->_wt) { this->_wt->reenable(); } @@ -347,7 +334,15 @@ class CacheRWWEOSTest : public CacheRWWTest } return; } - this_ethread()->schedule_imm(new SimpleCont(base)); + + // write at least one fragment before read it + if (this->_latest_fragments == this->_wt->vc->fragment) { + base->reenable(); + return; + } + + this->_latest_fragments = this->_wt->vc->fragment; + this->_rt->reenable(); break; case VC_EVENT_WRITE_COMPLETE: @@ -370,7 +365,6 @@ class CacheRWWEOSTest : public CacheRWWTest base->do_io_read(UINT32_MAX); break; case VC_EVENT_READ_READY: - base->reenable(); if (this->_wt) { this->_wt->reenable(); } From fd8d9acf22ef5fc8c7ec44050b5a90a1a05484e6 Mon Sep 17 00:00:00 2001 From: Xavier Chi Date: Thu, 11 Apr 2019 14:08:32 -0500 Subject: [PATCH 419/526] Fix reason tag of traffic_ctl host --- src/traffic_ctl/traffic_ctl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/traffic_ctl/traffic_ctl.cc b/src/traffic_ctl/traffic_ctl.cc index 8efb92a8112..a826abb0cbf 100644 --- a/src/traffic_ctl/traffic_ctl.cc +++ b/src/traffic_ctl/traffic_ctl.cc @@ -211,10 +211,10 @@ main(int argc, const char **argv) host_command.add_command("down", "Set down one or more host(s)", "", MORE_THAN_ONE_ARG_N, [&]() { engine.status_down(); }) .add_example_usage("traffic_ctl host down HOST [OPTIONS]") .add_option("--time", "-I", "number of seconds that a host is marked down", "", 1) - .add_option("--reason", "", "reason for marking the host down, one of 'manual|active|local"); + .add_option("--reason", "", "reason for marking the host down, one of 'manual|active|local", "", 1); host_command.add_command("up", "Set up one or more host(s)", "", MORE_THAN_ONE_ARG_N, [&]() { engine.status_up(); }) .add_example_usage("traffic_ctl host up METRIC value") - .add_option("--reason", "", "reason for marking the host up, one of 'manual|active|local"); + .add_option("--reason", "", "reason for marking the host up, one of 'manual|active|local", "", 1); // metric commands metric_command.add_command("get", "Get one or more metric values", "", MORE_THAN_ONE_ARG_N, [&]() { engine.metric_get(); }) From 8f93bcafedce9f0abd80762acc4b42e5a10ca4dd Mon Sep 17 00:00:00 2001 From: dyrock Date: Wed, 10 Apr 2019 13:22:30 -0500 Subject: [PATCH 420/526] Corrected usage of OpenSSL API for ec and ecpf list retrieval. Now the signature is valid without additional length bytes accounted for signature. Added OPENSSL_free after call to SSL_client_hello_get1_extensions_present returned success --- plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc index 0c91b83dff5..b94c835ead6 100644 --- a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc +++ b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc @@ -227,14 +227,21 @@ custom_get_ja3(SSL *s) // Get cipher suites len = SSL_client_hello_get0_ciphers(s, &p); custom_get_ja3_prefixed(2, p, len, ja3); + ja3 += ','; // Get extenstions int *o; std::string eclist, ecpflist; if (SSL_client_hello_get0_ext(s, 0x0a, &p, &len) == 1) { + // Skip first 2 bytes since we already have length + p += 2; + len -= 2; custom_get_ja3_prefixed(2, p, len, eclist); } if (SSL_client_hello_get0_ext(s, 0x0b, &p, &len) == 1) { + // Skip first byte since we already have length + ++p; + --len; custom_get_ja3_prefixed(1, p, len, ecpflist); } if (SSL_client_hello_get1_extensions_present(s, &o, &len) == 1) { @@ -249,6 +256,7 @@ custom_get_ja3(SSL *s) ja3 += std::to_string(type); } } + OPENSSL_free(o); } ja3 += "," + eclist + "," + ecpflist; return ja3; From 8871845c3f0606fb991a954aa7d35a72aa128e82 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 11 Apr 2019 19:04:38 -0500 Subject: [PATCH 421/526] Fix IntrusiveHashMap active bucket list corruption issue during expansion. --- include/tscore/IntrusiveHashMap.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/tscore/IntrusiveHashMap.h b/include/tscore/IntrusiveHashMap.h index c787e4b358b..3e2b5c9d13d 100644 --- a/include/tscore/IntrusiveHashMap.h +++ b/include/tscore/IntrusiveHashMap.h @@ -343,6 +343,9 @@ IntrusiveHashMap::Bucket::clear() _v = nullptr; _count = 0; _mixed_p = false; + // These can be left set during an expansion, when the bucket did have elements before but not + // after. Therefore make sure they are cleared. + _link._next = _link._prev = nullptr; } template From 23a855fedcbd1a286877e801e7aa2bcfda16ca1a Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Fri, 1 Feb 2019 09:37:10 -0600 Subject: [PATCH 422/526] Update plugin API type conversions to support enums automatically. --- src/traffic_server/InkAPI.cc | 69 +++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index fb1e0d0ac00..6945a96a969 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -95,10 +95,6 @@ extern AppVersionInfo appVersionInfo; static int api_rsb_index; static RecRawStatBlock *api_rsb; -static std::type_info const &TYPE_INFO_MGMT_INT = typeid(MgmtInt); -static std::type_info const &TYPE_INFO_MGMT_BYTE = typeid(MgmtByte); -static std::type_info const &TYPE_INFO_MGMT_FLOAT = typeid(MgmtFloat); - /** Reservation for a user arg. */ struct UserArg { @@ -7944,32 +7940,63 @@ TSSkipRemappingSet(TSHttpTxn txnp, int flag) * to this API handling, with the rest of the code base using the natural types. */ +/// Unhandled API conversions. +/// Because the code around the specially handled types still uses this in the default case, +/// it must compile for those cases. To indicate unhandled, return @c nullptr for @a conv. +/// @internal This should be a temporary state, eventually the other cases should be handled +/// via specializations here. +/// @internal C++ note - THIS MUST BE FIRST IN THE DECLARATIONS or it might be falsely used. template inline void * _memberp_to_generic(T *ptr, MgmtConverter const *&conv) { - static const MgmtConverter IntConverter([](void *data) -> MgmtInt { return *static_cast(data); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }); + conv = nullptr; + return ptr; +} - static const MgmtConverter ByteConverter{[](void *data) -> MgmtInt { return *static_cast(data); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }}; +/// API conversion for @c MgmtInt, identify conversion as integer. +inline void * +_memberp_to_generic(MgmtInt *ptr, MgmtConverter const *&conv) +{ + static const MgmtConverter converter([](void *data) -> MgmtInt { return *static_cast(data); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }); - static const MgmtConverter FloatConverter{[](void *data) -> MgmtFloat { return *static_cast(data); }, - [](void *data, MgmtFloat f) -> void { *static_cast(data) = f; }}; + conv = &converter; + return ptr; +} - // For now, strings are special. +/// API conversion for @c MgmtByte, handles integer / byte size differences. +inline void * +_memberp_to_generic(MgmtByte *ptr, MgmtConverter const *&conv) +{ + static const MgmtConverter converter{[](void *data) -> MgmtInt { return *static_cast(data); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }}; - auto type = &typeid(T); - if (*type == TYPE_INFO_MGMT_INT) { - conv = &IntConverter; - } else if (*type == TYPE_INFO_MGMT_BYTE) { - conv = &ByteConverter; - } else if (*type == TYPE_INFO_MGMT_FLOAT) { - conv = &FloatConverter; - } else { - conv = nullptr; - } + conv = &converter; + return ptr; +} + +/// API conversion for @c MgmtFloat, identity conversion as float. +inline void * +_memberp_to_generic(MgmtFloat *ptr, MgmtConverter const *&conv) +{ + static const MgmtConverter converter{[](void *data) -> MgmtFloat { return *static_cast(data); }, + [](void *data, MgmtFloat f) -> void { *static_cast(data) = f; }}; + + conv = &converter; + return ptr; +} + +/// API conversion for arbitrary enum. +/// Handle casting to and from the enum type @a E. +template +inline auto +_memberp_to_generic(MgmtFloat *ptr, MgmtConverter const *&conv) -> typename std::enable_if::value, void *>::type +{ + static const MgmtConverter converter{[](void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }}; + conv = &converter; return ptr; } From 6ae8c58cccf4b2ed425c025df99830fd4532f5a8 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Fri, 12 Apr 2019 09:28:38 +0900 Subject: [PATCH 423/526] Ignore unsupported HTTP/2 settings parameters --- proxy/http2/Http2ConnectionState.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index 50c2b395b50..57cd3685bc4 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -85,7 +85,7 @@ class Http2ConnectionSettings if (0 < id && id < HTTP2_SETTINGS_MAX) { return this->settings[indexof(id)] = value; } else { - ink_assert(!"Bad Settings Identifier"); + // Do nothing - 6.5.3 Unsupported parameters MUST be ignored. } return 0; From 192dc8300209ed17b0ff1c96aafba0f4096b27b2 Mon Sep 17 00:00:00 2001 From: Dylan Souza Date: Tue, 15 Jan 2019 18:50:55 +0000 Subject: [PATCH 424/526] Strip token from upstream if conifigured and dynamically allocate string buffers Adds a configuration option to strip uri signing tokens from both the cache key URL and the upstream URL. Additionally it was pointed out that some statically allocated buffers were too small in some of the string manipulating functions (normalize and strip token). These buffers are now dynamically allocated since the maximum buffer size is known for these. --- plugins/experimental/uri_signing/README.md | 27 +++--- plugins/experimental/uri_signing/common.h | 4 + plugins/experimental/uri_signing/config.c | 16 +++- plugins/experimental/uri_signing/config.h | 1 + plugins/experimental/uri_signing/cookie.c | 1 - plugins/experimental/uri_signing/jwt.c | 21 +++-- plugins/experimental/uri_signing/match.c | 1 - plugins/experimental/uri_signing/parse.c | 1 - .../unit_tests/uri_signing_test.cc | 12 ++- .../experimental/uri_signing/uri_signing.c | 83 ++++++++++++++++--- 10 files changed, 130 insertions(+), 37 deletions(-) diff --git a/plugins/experimental/uri_signing/README.md b/plugins/experimental/uri_signing/README.md index 8180586bf9a..398b9869958 100644 --- a/plugins/experimental/uri_signing/README.md +++ b/plugins/experimental/uri_signing/README.md @@ -1,8 +1,7 @@ URI Signing Plugin ================== -This remap plugin implements the draft URI Signing protocol documented here: -https://tools.ietf.org/html/draft-ietf-cdni-uri-signing-16 . +This remap plugin implements the draft URI Signing protocol documented [here](https://tools.ietf.org/html/draft-ietf-cdni-uri-signing-16): It takes a single argument: the name of a config file that contains key information. @@ -77,16 +76,25 @@ It's worth noting that multiple issuers can provide `auth_directives`. Each issuer will be processed in order and any issuer can provide access to a path. -### Token Stripping +### More Configuration Options -When The boolean strip_token parameter is set to true, the plugin removes the +**Strip Token** +When the strip_token parameter is set to true, the plugin removes the token from both the url that is sent upstream to the origin and the url that -is used as the cache key. It can be set like this: +is used as the cache key. The strip_token parameter defaults to false and should +be set by only one issuer. +**ID** +The id field takes a string indicating the identification of the entity processing the request. +This is used in aud claim checks to ensure that the receiver is the intended audience of a +tokenized request. The id parameter can only be set by one issuer. + +Example: { "Kabletown URI Authority": { "renewal_kid": "Second Key", "strip_token" : true, + "id" : "mycdn", "auth_directives": [ ⋮ ] @@ -95,8 +103,6 @@ is used as the cache key. It can be set like this: ] } -The strip_token parameter defaults to false and should be set by only one issuer. - Usage ----- @@ -107,10 +113,9 @@ will receive a 403 Forbidden response, instead of receiving content. Tokens will be found in either of these places: - A query parameter named `URISigningPackage`. The value must be the JWT. + - A path parameter named `URISigningPackage`. The value must be the JWT. - A cookie named `URISigningPackage`. The value of the cookie must be the JWT. -Path parameters will not be searched for JWTs. - ### Supported Claims The following claims are understood: @@ -118,6 +123,8 @@ The following claims are understood: - `iss`: Must be present. The issuer is used to locate the key for verification. - `sub`: May be present, but is not validated. - `exp`: Expired tokens are not valid. + - `nbf`: Tokens processed before this time are not valid. + - `aud`: Token aud claim strings must match the configured id to be considered valid. - `iat`: May be present, but is not validated. - `cdniv`: Must be missing or 1. - `cdniuc`: Validated last, after key verificationD. **Only `regex` is supported!** @@ -129,8 +136,6 @@ The following claims are understood: These claims are not supported. If they are present, the token will not validate: - - `aud` - - `nbf` - `jti` - `cdnicrit` - `cdniip` diff --git a/plugins/experimental/uri_signing/common.h b/plugins/experimental/uri_signing/common.h index 10505fa8d8a..467d0cee257 100644 --- a/plugins/experimental/uri_signing/common.h +++ b/plugins/experimental/uri_signing/common.h @@ -16,6 +16,8 @@ * limitations under the License. */ +#pragma once + #define PLUGIN_NAME "uri_signing" #ifdef URI_SIGNING_UNIT_TEST @@ -24,6 +26,8 @@ #define PluginDebug(fmt, ...) PrintToStdErr("(%s) %s:%d:%s() " fmt "\n", PLUGIN_NAME, __FILE__, __LINE__, __func__, ##__VA_ARGS__) #define PluginError(fmt, ...) PrintToStdErr("(%s) %s:%d:%s() " fmt "\n", PLUGIN_NAME, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define TSmalloc(x) malloc(x) +#define TSfree(p) free(p) void PrintToStdErr(const char *fmt, ...); #else diff --git a/plugins/experimental/uri_signing/config.c b/plugins/experimental/uri_signing/config.c index 96429148e90..cdf2f2db4ce 100644 --- a/plugins/experimental/uri_signing/config.c +++ b/plugins/experimental/uri_signing/config.c @@ -21,8 +21,6 @@ #include "timing.h" #include "jwt.h" -#include - #include #include @@ -46,6 +44,7 @@ struct config { struct signer signer; struct auth_directive *auth_directives; char *id; + bool strip_token; }; cjose_jwk_t ** @@ -87,6 +86,12 @@ config_get_id(struct config *cfg) return cfg->id; } +bool +config_strip_token(struct config *cfg) +{ + return cfg->strip_token; +} + struct config * config_new(size_t n) { @@ -114,6 +119,8 @@ config_new(size_t n) cfg->auth_directives = NULL; cfg->id = NULL; + cfg->strip_token = false; + PluginDebug("New config object created at %p", cfg); return cfg; } @@ -284,6 +291,11 @@ read_config(const char *path) } json_decref(id_json); + json_t *strip_json = json_object_get(jwks, "strip_token"); + if (strip_json) { + cfg->strip_token = json_boolean_value(strip_json); + } + size_t jwks_ct = json_array_size(key_ary); cjose_jwk_t **jwks = (*jwkis++ = malloc((jwks_ct + 1) * sizeof *jwks)); PluginDebug("Created table with size %d", cfg->issuers->size); diff --git a/plugins/experimental/uri_signing/config.h b/plugins/experimental/uri_signing/config.h index a22ec5d273d..87688374c85 100644 --- a/plugins/experimental/uri_signing/config.h +++ b/plugins/experimental/uri_signing/config.h @@ -34,3 +34,4 @@ struct _cjose_jwk_int **find_keys(struct config *cfg, const char *issuer); struct _cjose_jwk_int *find_key_by_kid(struct config *cfg, const char *issuer, const char *kid); bool uri_matches_auth_directive(struct config *cfg, const char *uri, size_t uri_ct); const char *config_get_id(struct config *cfg); +bool config_strip_token(struct config *cfg); diff --git a/plugins/experimental/uri_signing/cookie.c b/plugins/experimental/uri_signing/cookie.c index 45a2e43b961..1e9fc7f9a30 100644 --- a/plugins/experimental/uri_signing/cookie.c +++ b/plugins/experimental/uri_signing/cookie.c @@ -18,7 +18,6 @@ #include "cookie.h" #include "common.h" -#include #include const char * diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index 020d0cac0f8..f14ecb6e289 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -20,7 +20,6 @@ #include "jwt.h" #include "match.h" #include "normalize.h" -#include "ts/ts.h" #include #include #include @@ -206,13 +205,13 @@ jwt_check_uri(const char *cdniuc, const char *uri) int uri_ct = strlen(uri); int buff_ct = uri_ct + 2; int err; - char normal_uri[buff_ct]; - + char *normal_uri = (char *)TSmalloc(buff_ct); memset(normal_uri, 0, buff_ct); + err = normalize_uri(uri, uri_ct, normal_uri, buff_ct); if (err) { - return false; + goto fail_jwt; } const char *kind = cdniuc, *container = cdniuc; @@ -220,27 +219,35 @@ jwt_check_uri(const char *cdniuc, const char *uri) ++container; } if (!*container) { - return false; + goto fail_jwt; } ++container; size_t len = container - kind; + bool status; PluginDebug("Comparing with match kind \"%.*s\" on \"%s\" to normalized URI \"%s\"", (int)len - 1, kind, container, normal_uri); switch (len) { case sizeof CONT_URI_HASH_STR: if (!strncmp(CONT_URI_HASH_STR, kind, len - 1)) { - return match_hash(container, normal_uri); + status = match_hash(container, normal_uri); + TSfree(normal_uri); + return status; } PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_HASH_STR, (int)len - 1, kind); break; case sizeof CONT_URI_REGEX_STR: if (!strncmp(CONT_URI_REGEX_STR, kind, len - 1)) { - return match_regex(container, normal_uri); + status = match_regex(container, normal_uri); + TSfree(normal_uri); + return status; } PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_REGEX_STR, (int)len - 1, kind); break; } PluginDebug("Unknown match kind \"%.*s\"", (int)len - 1, kind); + +fail_jwt: + TSfree(normal_uri); return false; } diff --git a/plugins/experimental/uri_signing/match.c b/plugins/experimental/uri_signing/match.c index 40d4d472e93..faea8953dfc 100644 --- a/plugins/experimental/uri_signing/match.c +++ b/plugins/experimental/uri_signing/match.c @@ -18,7 +18,6 @@ #include #include "common.h" -#include "ts/ts.h" #include #include diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index 43667659922..3ca10b21d23 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -25,7 +25,6 @@ #include #include #include -#include #include cjose_jws_t * diff --git a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc index fe938160ccf..522db9999fd 100644 --- a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc +++ b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc @@ -58,19 +58,22 @@ normalize_uri_helper(const char *uri, const char *expected_normal) size_t uri_ct = strlen(uri); int buff_size = uri_ct + 2; int err; - char uri_normal[buff_size]; + char *uri_normal = (char *)malloc(buff_size); memset(uri_normal, 0, buff_size); err = normalize_uri(uri, uri_ct, uri_normal, buff_size); if (err) { + free(uri_normal); return false; } if (expected_normal && strcmp(expected_normal, uri_normal) == 0) { + free(uri_normal); return true; } + free(uri_normal); return false; } @@ -101,8 +104,10 @@ jws_parsing_helper(const char *uri, const char *paramName, const char *expected_ bool resp; size_t uri_ct = strlen(uri); size_t strip_ct = 0; - char uri_strip[uri_ct + 1]; - memset(uri_strip, 0, sizeof uri_strip); + + char *uri_strip = (char *)malloc(uri_ct + 1); + memset(uri_strip, 0, uri_ct + 1); + cjose_jws_t *jws = get_jws_from_uri(uri, uri_ct, paramName, uri_strip, uri_ct, &strip_ct); if (jws) { resp = true; @@ -114,6 +119,7 @@ jws_parsing_helper(const char *uri, const char *paramName, const char *expected_ resp = false; } cjose_jws_release(jws); + free(uri_strip); return resp; } diff --git a/plugins/experimental/uri_signing/uri_signing.c b/plugins/experimental/uri_signing/uri_signing.c index fadd269e540..f85a87b5ed3 100644 --- a/plugins/experimental/uri_signing/uri_signing.c +++ b/plugins/experimental/uri_signing/uri_signing.c @@ -22,10 +22,10 @@ #include "jwt.h" #include "timing.h" -#include #include #include +#include #include #include @@ -157,6 +157,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) int cpi = 0; int url_ct = 0; const char *url = NULL; + char *strip_uri = NULL; + TSRemapStatus status = TSREMAP_NO_REMAP; const char *package = "URISigningPackage"; @@ -176,9 +178,12 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) checkpoints[cpi++] = mark_timer(&t); } - char strip_uri[2000] = {0}; + int strip_size = url_ct + 1; + strip_uri = (char *)TSmalloc(strip_size); + memset(strip_uri, 0, strip_size); + size_t strip_ct; - cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, strip_uri, 2000, &strip_ct); + cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, strip_uri, strip_size, &strip_ct); if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); @@ -186,6 +191,9 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) int checked_cookies = 0; if (!jws) { check_cookies: + /* There is no valid token in the url */ + strncpy(strip_uri, url, url_ct); + strip_ct = url_ct; ++checked_cookies; TSMLoc field; @@ -218,6 +226,55 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) checkpoints[cpi++] = mark_timer(&t); } jws = get_jws_from_cookie(&client_cookie, &client_cookie_sz_ct, package); + } else { + /* There has been a JWS found in the url */ + /* Strip the token from the URL for upstream if configured to do so */ + if (config_strip_token((struct config *)ih)) { + if ((int)strip_ct != url_ct) { + int map_url_ct = 0; + char *map_url = NULL; + char *map_strip_uri = NULL; + map_url = TSUrlStringGet(rri->requestBufp, rri->requestUrl, &map_url_ct); + + PluginDebug("Stripping Token from requestUrl: %s", map_url); + + int map_strip_size = map_url_ct + 1; + map_strip_uri = (char *)TSmalloc(map_strip_size); + memset(map_strip_uri, 0, map_strip_size); + size_t map_strip_ct = 0; + + cjose_jws_t *map_jws = get_jws_from_uri(map_url, map_url_ct, package, map_strip_uri, map_strip_size, &map_strip_ct); + cjose_jws_release(map_jws); + + char *strip_uri_start = &map_strip_uri[0]; + char *strip_uri_end = &map_strip_uri[map_strip_ct]; + PluginDebug("Stripping token from upstream url to: %s", strip_uri_start); + + TSParseResult parse_rc = TSUrlParse(rri->requestBufp, rri->requestUrl, (const char **)&strip_uri_start, strip_uri_end); + if (map_url != NULL) { + TSfree(map_url); + } + if (map_strip_uri != NULL) { + TSfree(map_strip_uri); + } + + if (parse_rc != TS_PARSE_DONE) { + PluginDebug("Error in TSUrlParse"); + goto fail; + } + status = TSREMAP_DID_REMAP; + } + } + } + /* Check auth_dir and pass through if configured */ + if (uri_matches_auth_directive((struct config *)ih, url, url_ct)) { + if (url != NULL) { + TSfree((void *)url); + } + if (strip_uri != NULL) { + TSfree(strip_uri); + } + return TSREMAP_NO_REMAP; } if (!jws) { goto fail; @@ -226,8 +283,10 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } + struct jwt *jwt = validate_jws(jws, (struct config *)ih, strip_uri, strip_ct); cjose_jws_release(jws); + if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } @@ -239,6 +298,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) } } + /* There has been a validated JWT found in either the cookie or url */ + struct signer *signer = config_signer((struct config *)ih); char *cookie = renew(jwt, signer->issuer, signer->jwk, signer->alg, package); jwt_delete(jwt); @@ -260,16 +321,13 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) last_mark = checkpoints[i]; } PluginDebug("Spent %" PRId64 " ns uri_signing verification of %.*s.", mark_timer(&t), url_ct, url); + TSfree((void *)url); - return TSREMAP_NO_REMAP; -fail: - if (uri_matches_auth_directive((struct config *)ih, url, url_ct)) { - if (url != NULL) { - TSfree((void *)url); - } - return TSREMAP_NO_REMAP; + if (strip_uri != NULL) { + TSfree(strip_uri); } - + return status; +fail: PluginDebug("Invalid JWT for %.*s", url_ct, url); TSHttpTxnStatusSet(txnp, TS_HTTP_STATUS_FORBIDDEN); PluginDebug("Spent %" PRId64 " ns uri_signing verification of %.*s.", mark_timer(&t), url_ct, url); @@ -277,6 +335,9 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (url != NULL) { TSfree((void *)url); } + if (strip_uri != NULL) { + TSfree(strip_uri); + } return TSREMAP_DID_REMAP; } From 21dc79abb5a5e2a74b47ba4541c761bec372e602 Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Tue, 9 Apr 2019 14:59:39 -0500 Subject: [PATCH 425/526] docs for schedule api --- .../api/functions/TSContScheduleOnPool.en.rst | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst b/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst index 462f6a9aec9..f35ddb95fa9 100644 --- a/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst +++ b/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst @@ -55,6 +55,70 @@ called and continuations that use them have the same restrictions. ``TS_THREAD_P are threads that exist to perform long or blocking actions, although sufficiently long operation can impact system performance by blocking other continuations on the threads. +Example Scenarios +================= + +Scenario 1 (no thread affinity info, different types of threads) +---------------------------------------------------------------- + +When thread affinity is not set, a plugin calls the API on thread "A" (which is an "ET_TASK" type), and +wants to schedule on an "ET_NET" type thread provided in "tp", the system would see there is no thread +affinity information stored in "contp." + +In this situation, system sees there is no thread affinity information stored in "contp". It then +checks whether the type of thread "A" is the same as provided in "tp", and sees that "A" is "ET_TASK", +but "tp" says "ET_NET". So "contp" gets scheduled on the next available "ET_NET" thread provided by a +round robin list, which we will call thread "B". Since "contp" doesn't have thread affinity information, +thread "B" will be assigned as the affinity thread for it automatically. + +The reason for doing this is most of the time people want to schedule the same things on the same type +of thread, so logically it is better to default the first thread that it is scheduled on as the affinity +thread. + +Scenario 2 (no thread affinity info, same types of threads) +----------------------------------------------------------- + +Slight variation of scenario 1, instead of scheduling on a "ET_NET" thread, the plugin wants to schedule +on a "ET_TASK" thread (i.e. "tp" contains "ET_TASK" now), all other conditions stays the same. + +This time since the type of the desired thread for scheduling and thread "A" are the same, the system +schedules "contp" on thread "A", and assigns thread "A" as the affinity thread for "contp". + +The reason behind this choice is that we are trying to keep things simple such that lock contention +problems happens less. And for the most part, there is no point of scheduling the same thing on several +different threads of the same type, because there is no parallelism between them (a thread will have to +wait for the previous thread to finish, either because locking or the nature of the job it’s handling is +serialized since its on the same continuation). + +Scenario 3 (has thread affinity info, different types of threads) +----------------------------------------------------------------- + +Slight variation of scenario 1, thread affinity is set for continuation "contp" to thread "A", all other +conditions stays the same. + +In this situation, the system sees that the "tp" has "ET_NET", but the type of thread "A" is "ET_TASK". +So even though "contp" has an affinity thread, the system will not use that information since the type is +different, instead it schedules "contp" on the next available "ET_NET" thread provided by a round robin +list, which we will call thread "B". The difference with scenario 1 is that since thread "A" is set to +be the affinity thread for "contp" already, the system will NOT overwrite that information with thread "B". + +Most of the time, a continuation will be scheduled on one type of threads, and rarely gets scheduled on +a different type. But when that happens, we want it to return to the thread it was previously on, so it +won’t have any lock contention problems. And that’s also why "thread_affinity" is not a hashmap of thread +types and thread pointers. + +Scenario 4 (has thread affinity info, same types of threads) +------------------------------------------------------------ + +Slight variation of scenario 3, the only difference is "tp" now says "ET_TASK". + +This is the easiest scenario since the type of thread "A" and "tp" are the same, so the system schedules +"contp" on thread "A". And, as discussed, there is really no reason why one may want to schedule +the same continuation on two different threads of the same type. + +.. note:: In scenario 3 & 4, it doesn't matter which thread the plugin is calling the API from. + + See Also ======== From 5a0f7e93bb03a0fcae178f2d9a37c605d029618c Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Wed, 10 Apr 2019 17:00:06 -0500 Subject: [PATCH 426/526] Fix HostDBReverseTest unitilization sa_family, remove LRAND48 and SRAND48 for C++11 random. --- configure.ac | 2 +- iocore/hostdb/HostDB.cc | 34 ++++++++++++--------------------- iocore/hostdb/P_RefCountCache.h | 4 ++-- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/configure.ac b/configure.ac index 75aebc581ff..9328124e753 100644 --- a/configure.ac +++ b/configure.ac @@ -1262,7 +1262,7 @@ TS_CHECK_ZLIB TS_CHECK_LZMA AC_CHECK_FUNCS([clock_gettime kqueue epoll_ctl posix_fadvise posix_madvise posix_fallocate inotify_init]) -AC_CHECK_FUNCS([lrand48_r srand48_r port_create strlcpy strlcat sysconf sysctlbyname getpagesize]) +AC_CHECK_FUNCS([port_create strlcpy strlcat sysconf sysctlbyname getpagesize]) AC_CHECK_FUNCS([getreuid getresuid getresgid setreuid setresuid getpeereid getpeerucred]) AC_CHECK_FUNCS([strsignal psignal psiginfo accept4]) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index d32f87a5f97..8a097d8fbd0 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -32,6 +32,8 @@ #include #include #include +#include +#include HostDBProcessor hostDBProcessor; int HostDBProcessor::hostdb_strict_round_robin = 0; @@ -2055,8 +2057,8 @@ register_ShowHostDB(Continuation *c, HTTPHdr *h) return &s->action; } -#define HOSTDB_TEST_MAX_OUTSTANDING 100 -#define HOSTDB_TEST_LENGTH 100000 +static constexpr int HOSTDB_TEST_MAX_OUTSTANDING = 20; +static constexpr int HOSTDB_TEST_LENGTH = 200; struct HostDBTestReverse; using HostDBTestReverseHandler = int (HostDBTestReverse::*)(int, void *); @@ -2065,34 +2067,26 @@ struct HostDBTestReverse : public Continuation { int type; int *status; - int outstanding; - int total; -#if HAVE_LRAND48_R - struct drand48_data dr; -#endif + int outstanding = 0; + int total = 0; + std::ranlux48 randu; int mainEvent(int event, Event *e) { if (event == EVENT_HOST_DB_LOOKUP) { - HostDBInfo *i = (HostDBInfo *)e; + HostDBInfo *i = reinterpret_cast(e); if (i) { rprintf(test, "HostDBTestReverse: reversed %s\n", i->hostname()); } outstanding--; } while (outstanding < HOSTDB_TEST_MAX_OUTSTANDING && total < HOSTDB_TEST_LENGTH) { - long l = 0; -#if HAVE_LRAND48_R - lrand48_r(&dr, &l); -#else - l = lrand48(); -#endif IpEndpoint ip; - ip.sin.sin_addr.s_addr = static_cast(l); + ip.assign(IpAddr(static_cast(randu()))); outstanding++; total++; - if (!(outstanding % 1000)) { + if (!(outstanding % 100)) { rprintf(test, "HostDBTestReverse: %d\n", total); } hostDBProcessor.getbyaddr_re(this, &ip.sa); @@ -2105,14 +2099,10 @@ struct HostDBTestReverse : public Continuation { return EVENT_CONT; } HostDBTestReverse(RegressionTest *t, int atype, int *astatus) - : Continuation(new_ProxyMutex()), test(t), type(atype), status(astatus), outstanding(0), total(0) + : Continuation(new_ProxyMutex()), test(t), type(atype), status(astatus) { SET_HANDLER((HostDBTestReverseHandler)&HostDBTestReverse::mainEvent); -#if HAVE_SRAND48_R - srand48_r(time(nullptr), &dr); -#else - srand48(time(nullptr)); -#endif + randu.seed(std::chrono::system_clock::now().time_since_epoch().count()); } }; diff --git a/iocore/hostdb/P_RefCountCache.h b/iocore/hostdb/P_RefCountCache.h index 60feab110ea..98073825888 100644 --- a/iocore/hostdb/P_RefCountCache.h +++ b/iocore/hostdb/P_RefCountCache.h @@ -290,9 +290,9 @@ RefCountCachePartition::clear() // Hence, this monstrosity. auto it = this->item_map.begin(); while (it != this->item_map.end()) { - auto cur = it; + auto cur = it++; - it = this->item_map.erase(it); + this->item_map.erase(cur); this->dealloc_entry(cur); } } From 7fb29ab0671552a656cddb34999f1568a1e49252 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 15 Apr 2019 10:09:08 -0500 Subject: [PATCH 427/526] Fix false collapsing of reverse DNS requests. --- iocore/dns/DNS.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 38e1e2efedc..ac9d4e1f239 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -138,7 +138,7 @@ HostEnt::free() dnsBufAllocator.free(this); } -void +size_t make_ipv4_ptr(in_addr_t addr, char *buffer) { char *p = buffer; @@ -176,10 +176,10 @@ make_ipv4_ptr(in_addr_t addr, char *buffer) } *p++ = u[0] % 10 + '0'; *p++ = '.'; - ink_strlcpy(p, "in-addr.arpa", MAXDNAME - (p - buffer + 1)); + return ink_strlcpy(p, "in-addr.arpa", MAXDNAME - (p - buffer + 1)); } -void +size_t make_ipv6_ptr(in6_addr const *addr, char *buffer) { const char hex_digit[] = "0123456789abcdef"; @@ -194,7 +194,7 @@ make_ipv6_ptr(in6_addr const *addr, char *buffer) *p++ = '.'; } - ink_strlcpy(p, "ip6.arpa", MAXDNAME - (p - buffer + 1)); + return ink_strlcpy(p, "ip6.arpa", MAXDNAME - (p - buffer + 1)); } // Public functions @@ -437,9 +437,9 @@ DNSEntry::init(const char *x, int len, int qtype_arg, Continuation *acont, DNSPr } else { // T_PTR IpAddr const *ip = reinterpret_cast(x); if (ip->isIp6()) { - make_ipv6_ptr(&ip->_addr._ip6, qname); + orig_qname_len = qname_len = make_ipv6_ptr(&ip->_addr._ip6, qname); } else if (ip->isIp4()) { - make_ipv4_ptr(ip->_addr._ip4, qname); + orig_qname_len = qname_len = make_ipv4_ptr(ip->_addr._ip4, qname); } else { ink_assert(!"T_PTR query to DNS must be IP address."); } From 574cf2bfb6602184f18f4fe05f4d4ad4b603cef0 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Mon, 15 Apr 2019 09:04:14 +0900 Subject: [PATCH 428/526] Cleanup: Use internal linkage for functions which are only needed in SSLUtils.cc --- iocore/net/SSLUtils.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 38009d5bc90..d2e2e97ae2b 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -169,7 +169,7 @@ SSL_CTX_add_extra_chain_cert_file(SSL_CTX *ctx, const char *chainfile) return SSL_CTX_add_extra_chain_cert_bio(ctx, bio); } -bool +static bool ssl_session_timed_out(SSL_SESSION *session) { return SSL_SESSION_get_timeout(session) < (long)(time(nullptr) - SSL_SESSION_get_time(session)); @@ -273,7 +273,7 @@ ssl_rm_cached_session(SSL_CTX *ctx, SSL_SESSION *sess) session_cache->removeSession(sid); } -int +static int set_context_cert(SSL *ssl) { SSL_CTX *ctx = nullptr; @@ -344,7 +344,7 @@ set_context_cert(SSL *ssl) } // Callback function for verifying client certificate -int +static int ssl_verify_client_callback(int preverify_ok, X509_STORE_CTX *ctx) { Debug("ssl", "Callback: verify client cert"); From 03490d09912e045e7a11abada651da82975af485 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 5 Apr 2019 17:51:44 -0600 Subject: [PATCH 429/526] Adds basic version feature for traffic_layout info --- src/traffic_layout/Makefile.inc | 2 +- src/traffic_layout/engine.cc | 2 + src/traffic_layout/info.cc | 65 ++++++++++++++++++++++++++-- src/traffic_layout/info.h | 2 +- src/traffic_layout/traffic_layout.cc | 1 + 5 files changed, 67 insertions(+), 5 deletions(-) diff --git a/src/traffic_layout/Makefile.inc b/src/traffic_layout/Makefile.inc index 7664fe3d790..afdc3141825 100644 --- a/src/traffic_layout/Makefile.inc +++ b/src/traffic_layout/Makefile.inc @@ -47,4 +47,4 @@ traffic_layout_traffic_layout_LDADD = \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @HWLOC_LIBS@ @YAMLCPP_LIBS@ + @HWLOC_LIBS@ @YAMLCPP_LIBS@ @LIBLZMA@ diff --git a/src/traffic_layout/engine.cc b/src/traffic_layout/engine.cc index e6a2703afcd..3838d63acd8 100644 --- a/src/traffic_layout/engine.cc +++ b/src/traffic_layout/engine.cc @@ -162,6 +162,8 @@ LayoutEngine::info() if (arguments.get("features")) { produce_features(json); + } else if (arguments.get("versions")) { + produce_versions(json); } else { produce_layout(json); } diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 94060cfbf00..74af91fcb9e 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -21,11 +21,25 @@ limitations under the License. */ +#include #include "tscore/I_Layout.h" +#include "tscore/BufferWriter.h" #include "records/I_RecProcess.h" #include "RecordsConfig.h" #include "info.h" +#if HAVE_ZLIB_H +#include +#endif + +#if HAVE_LZMA_H +#include +#endif + +#if HAVE_BROTLI_ENCODE_H +#include +#endif + // Produce output about compile time features, useful for checking how things were built, as well // as for our TSQA test harness. static void @@ -59,17 +73,17 @@ produce_features(bool json) print_feature("BUILD_PERSON", BUILD_PERSON, json); print_feature("BUILD_GROUP", BUILD_GROUP, json); print_feature("BUILD_NUMBER", BUILD_NUMBER, json); -#ifdef HAVE_ZLIB_H +#if HAVE_ZLIB_H print_feature("TS_HAS_LIBZ", 1, json); #else print_feature("TS_HAS_LIBZ", 0, json); #endif -#ifdef HAVE_LZMA_H +#if HAVE_LZMA_H print_feature("TS_HAS_LZMA", 1, json); #else print_feature("TS_HAS_LZMA", 0, json); #endif -#ifdef HAVE_BROTLI_ENCODE_H +#if HAVE_BROTLI_ENCODE_H print_feature("TS_HAS_BROTLI", 1, json); #else print_feature("TS_HAS_BROTLI", 0, json); @@ -152,3 +166,48 @@ produce_layout(bool json) printf("}\n"); } } + +void +produce_versions(bool json) +{ + using LBW = ts::LocalBufferWriter<128>; + static const std::string_view undef{"undef"}; + + if (json) { + printf("{\n"); + } + + print_var("openssl", LBW().print("{:#x}", OPENSSL_VERSION_NUMBER).view(), json); + print_var("openssl_str", LBW().print(OPENSSL_VERSION_TEXT).view(), json); + print_var("pcre", LBW().print("{}.{}", PCRE_MAJOR, PCRE_MINOR).view(), json); + // These are optional, for now at least. +#if TS_USE_HWLOC + print_var("hwloc", LBW().print("{:#x}", HWLOC_API_VERSION).view(), json); + print_var("hwloc.run", LBW().print("{:#x}", hwloc_get_api_version()).view(), json); +#else + print_var("hwloc", undef, json); +#endif +#if HAVE_ZLIB_H + print_var("libz", LBW().print("{}", ZLIB_VERSION).view(), json); +#else + print_var("libz", undef, json); +#endif +#if HAVE_LZMA_H + print_var("lzma", LBW().print("{}", LZMA_VERSION_STRING).view(), json); + print_var("lzma.run", LBW().print("{}", lzma_version_string()).view(), json); +#else + print_var("lzma", undef, json); +#endif +#if HAVE_BROTLI_ENCODE_H + print_var("lzma", LBW().print("{}", BrotliEncoderVersion).view(), json); +#else + print_var("brotli", undef, json); +#endif + + // This should always be last + print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true); + + if (json) { + printf("}\n"); + } +} diff --git a/src/traffic_layout/info.h b/src/traffic_layout/info.h index c70f0ec8756..fc3831c0454 100644 --- a/src/traffic_layout/info.h +++ b/src/traffic_layout/info.h @@ -24,5 +24,5 @@ // the original traffic_layout void produce_features(bool json); - void produce_layout(bool json); +void produce_versions(bool json); diff --git a/src/traffic_layout/traffic_layout.cc b/src/traffic_layout/traffic_layout.cc index fb26e343c3c..935a6a77c89 100644 --- a/src/traffic_layout/traffic_layout.cc +++ b/src/traffic_layout/traffic_layout.cc @@ -47,6 +47,7 @@ main(int argc, const char **argv) // info command engine.parser.add_command("info", "Show the layout as default", [&]() { engine.info(); }) .add_option("--features", "", "Show the compiled features") + .add_option("--versions", "", "Show various library and other versioning information") .add_option("--json", "-j", "Produce output in JSON format (when supported)") .set_default(); // init command From af308e54db7f18cdb92dd4700bdff37af97ea45e Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 26 Mar 2019 21:21:26 +0100 Subject: [PATCH 430/526] Ran CPP check on a few files while prodding around Another example of some of the changes it recommends, I think it would be useful to try to work on getting us CPPCheck "free". --- proxy/IPAllow.h | 6 +-- proxy/InkAPIInternal.h | 23 +++----- proxy/hdrs/HTTP.h | 2 +- proxy/hdrs/HdrToken.h | 3 +- proxy/hdrs/MIME.h | 108 +++++++++++++++++++------------------ proxy/http/HttpSM.cc | 61 ++++++++++----------- proxy/http/HttpSM.h | 2 +- proxy/http/HttpTunnel.h | 5 +- proxy/logging/Log.cc | 25 +++++---- proxy/logging/Log.h | 2 +- proxy/logging/LogAccess.cc | 7 ++- proxy/logging/LogAccess.h | 2 +- proxy/logging/LogField.h | 2 +- proxy/logging/LogObject.h | 4 +- proxy/logging/LogSock.h | 2 +- 15 files changed, 125 insertions(+), 129 deletions(-) diff --git a/proxy/IPAllow.h b/proxy/IPAllow.h index 2fedf29f87f..112d9774d2d 100644 --- a/proxy/IPAllow.h +++ b/proxy/IPAllow.h @@ -98,8 +98,8 @@ class IpAllow : public ConfigInfo using self_type = ACL; ///< Self reference type. public: ACL() = default; - ACL(const self_type &) = delete; // no copies. - ACL(self_type &&that) noexcept; // move allowed. + ACL(const self_type &) = delete; // no copies. + explicit ACL(self_type &&that) noexcept; // move allowed. ~ACL(); self_type &operator=(const self_type &) = delete; @@ -131,7 +131,7 @@ class IpAllow : public ConfigInfo IpAllow *_config{nullptr}; ///< The backing configuration. }; - IpAllow(const char *config_var); + explicit IpAllow(const char *config_var); void Print(); diff --git a/proxy/InkAPIInternal.h b/proxy/InkAPIInternal.h index ee516e238df..17ba528defb 100644 --- a/proxy/InkAPIInternal.h +++ b/proxy/InkAPIInternal.h @@ -53,20 +53,13 @@ enum CacheInfoMagic { struct CacheInfo { CryptoHash cache_key; - CacheFragType frag_type; - char *hostname; - int len; - time_t pin_in_cache; - CacheInfoMagic magic; + CacheFragType frag_type = CACHE_FRAG_TYPE_NONE; + char *hostname = nullptr; + int len = 0; + time_t pin_in_cache = 0; + CacheInfoMagic magic = CACHE_INFO_MAGIC_ALIVE; - CacheInfo() - { - frag_type = CACHE_FRAG_TYPE_NONE; - hostname = nullptr; - len = 0; - pin_in_cache = 0; - magic = CACHE_INFO_MAGIC_ALIVE; - } + CacheInfo() {} }; class FileImpl @@ -268,7 +261,7 @@ class HttpAPIHooks : public FeatureAPIHooks class TSSslHookInternalID { public: - constexpr TSSslHookInternalID(TSHttpHookID id) : _id(id - TS_SSL_FIRST_HOOK) {} + explicit constexpr TSSslHookInternalID(TSHttpHookID id) : _id(id - TS_SSL_FIRST_HOOK) {} constexpr operator int() const { return _id; } @@ -295,7 +288,7 @@ class LifecycleAPIHooks : public FeatureAPIHooksmutex.get()), m_cont(contp) + explicit ConfigUpdateCallback(INKContInternal *contp) : Continuation(contp->mutex.get()), m_cont(contp) { SET_HANDLER(&ConfigUpdateCallback::event_handler); } diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h index 94e1a103365..8636cf92d65 100644 --- a/proxy/hdrs/HTTP.h +++ b/proxy/hdrs/HTTP.h @@ -1236,7 +1236,7 @@ HTTPHdr::is_cache_control_set(const char *cc_directive_wks) ink_assert(valid()); ink_assert(hdrtoken_is_wks(cc_directive_wks)); - HdrTokenHeapPrefix *prefix = hdrtoken_wks_to_prefix(cc_directive_wks); + const HdrTokenHeapPrefix *prefix = hdrtoken_wks_to_prefix(cc_directive_wks); ink_assert(prefix->wks_token_type == HDRTOKEN_TYPE_CACHE_CONTROL); uint32_t cc_mask = prefix->wks_type_specific.u.cache_control.cc_mask; diff --git a/proxy/hdrs/HdrToken.h b/proxy/hdrs/HdrToken.h index ab5f4fb5beb..8a50a16f122 100644 --- a/proxy/hdrs/HdrToken.h +++ b/proxy/hdrs/HdrToken.h @@ -137,11 +137,12 @@ hdrtoken_is_valid_wks_idx(int wks_idx) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +// ToDo: This, and dependencies / users should probalby be const HdrTokenHeapPrefix * IMO. inline HdrTokenHeapPrefix * hdrtoken_wks_to_prefix(const char *wks) { ink_assert(hdrtoken_is_wks(wks)); - return ((HdrTokenHeapPrefix *)(wks - sizeof(HdrTokenHeapPrefix))); + return reinterpret_cast(const_cast(wks) - sizeof(HdrTokenHeapPrefix)); } /*------------------------------------------------------------------------- diff --git a/proxy/hdrs/MIME.h b/proxy/hdrs/MIME.h index 311b5cce39a..6ff23f77220 100644 --- a/proxy/hdrs/MIME.h +++ b/proxy/hdrs/MIME.h @@ -117,7 +117,7 @@ struct MIMEField { bool is_cooked() { - return (m_flags & MIME_FIELD_SLOT_FLAGS_COOKED); + return (m_flags & MIME_FIELD_SLOT_FLAGS_COOKED) ? true : false; } bool @@ -135,10 +135,10 @@ struct MIMEField { bool supports_commas() const { - if (m_wks_idx >= 0) + if (m_wks_idx >= 0) { return (hdrtoken_index_to_flags(m_wks_idx) & MIME_FLAGS_COMMAS); - else - return true; // by default, assume supports commas + } + return true; // by default, assume supports commas } /// @return The name of @a this field. @@ -814,11 +814,10 @@ MIMEField::name_get(int *length) const inline void MIMEField::name_set(HdrHeap *heap, MIMEHdrImpl *mh, const char *name, int length) { - int16_t name_wks_idx; const char *name_wks; if (hdrtoken_is_wks(name)) { - name_wks_idx = hdrtoken_wks_to_index(name); + int16_t name_wks_idx = hdrtoken_wks_to_index(name); mime_field_name_set(heap, mh, this, name_wks_idx, name, length, true); } else { int field_name_wks_idx = hdrtoken_tokenize(name, length, &name_wks); @@ -857,31 +856,31 @@ MIMEField::value_get(int *length) const inline int32_t MIMEField::value_get_int() const { - return (mime_field_value_get_int(this)); + return mime_field_value_get_int(this); } inline uint32_t MIMEField::value_get_uint() const { - return (mime_field_value_get_uint(this)); + return mime_field_value_get_uint(this); } inline int64_t MIMEField::value_get_int64() const { - return (mime_field_value_get_int64(this)); + return mime_field_value_get_int64(this); } inline time_t MIMEField::value_get_date() const { - return (mime_field_value_get_date(this)); + return mime_field_value_get_date(this); } inline int MIMEField::value_get_comma_list(StrList *list) const { - return (mime_field_value_get_comma_list(this, list)); + return mime_field_value_get_comma_list(this, list); } /*------------------------------------------------------------------------- @@ -1169,7 +1168,7 @@ MIMEHdr::field_create(const char *name, int length) mime_field_name_set(m_heap, m_mime, field, field_name_wks_idx, name, length, true); } - return (field); + return field; } /*------------------------------------------------------------------------- @@ -1304,10 +1303,11 @@ inline int MIMEHdr::value_get_index(const char *name, int name_length, const char *value, int value_length) const { const MIMEField *field = field_find(name, name_length); - if (field) + + if (field) { return field->value_get_index(value, value_length); - else - return -1; + } + return -1; } /*------------------------------------------------------------------------- @@ -1316,7 +1316,9 @@ MIMEHdr::value_get_index(const char *name, int name_length, const char *value, i inline const char * MIMEHdr::value_get(const char *name, int name_length, int *value_length_return) const { - if (const MIMEField *field = field_find(name, name_length); field) { + const MIMEField *field = field_find(name, name_length); + + if (field) { return field->value_get(value_length_return); } return nullptr; @@ -1325,7 +1327,9 @@ MIMEHdr::value_get(const char *name, int name_length, int *value_length_return) inline std::string_view MIMEHdr::value_get(std::string_view const &name) const { - if (MIMEField const *field = field_find(name.data(), name.size()); field) { + MIMEField const *field = field_find(name.data(), name.size()); + + if (field) { return field->value_get(); } return {}; @@ -1336,10 +1340,10 @@ MIMEHdr::value_get_int(const char *name, int name_length) const { const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get_int(field)); - else - return (0); + if (field) { + return mime_field_value_get_int(field); + } + return 0; } inline uint32_t @@ -1347,10 +1351,10 @@ MIMEHdr::value_get_uint(const char *name, int name_length) const { const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get_uint(field)); - else - return (0); + if (field) { + return mime_field_value_get_uint(field); + } + return 0; } inline int64_t @@ -1358,10 +1362,10 @@ MIMEHdr::value_get_int64(const char *name, int name_length) const { const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get_int64(field)); - else - return (0); + if (field) { + return mime_field_value_get_int64(field); + } + return 0; } inline time_t @@ -1369,10 +1373,10 @@ MIMEHdr::value_get_date(const char *name, int name_length) const { const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get_date(field)); - else - return (0); + if (field) { + return mime_field_value_get_date(field); + } + return 0; } inline int @@ -1380,10 +1384,10 @@ MIMEHdr::value_get_comma_list(const char *name, int name_length, StrList *list) { const MIMEField *field = field_find(name, name_length); - if (field) - return (field->value_get_comma_list(list)); - else - return (0); + if (field) { + return field->value_get_comma_list(list); + } + return 0; } /*------------------------------------------------------------------------- @@ -1547,7 +1551,7 @@ MIMEHdr::get_age() inline int64_t MIMEHdr::get_content_length() const { - return (value_get_int64(MIME_FIELD_CONTENT_LENGTH, MIME_LEN_CONTENT_LENGTH)); + return value_get_int64(MIME_FIELD_CONTENT_LENGTH, MIME_LEN_CONTENT_LENGTH); } /*------------------------------------------------------------------------- @@ -1556,7 +1560,7 @@ MIMEHdr::get_content_length() const inline time_t MIMEHdr::get_date() { - return (value_get_date(MIME_FIELD_DATE, MIME_LEN_DATE)); + return value_get_date(MIME_FIELD_DATE, MIME_LEN_DATE); } /*------------------------------------------------------------------------- @@ -1565,7 +1569,7 @@ MIMEHdr::get_date() inline time_t MIMEHdr::get_expires() { - return (value_get_date(MIME_FIELD_EXPIRES, MIME_LEN_EXPIRES)); + return value_get_date(MIME_FIELD_EXPIRES, MIME_LEN_EXPIRES); } /*------------------------------------------------------------------------- @@ -1574,7 +1578,7 @@ MIMEHdr::get_expires() inline time_t MIMEHdr::get_if_modified_since() { - return (value_get_date(MIME_FIELD_IF_MODIFIED_SINCE, MIME_LEN_IF_MODIFIED_SINCE)); + return value_get_date(MIME_FIELD_IF_MODIFIED_SINCE, MIME_LEN_IF_MODIFIED_SINCE); } /*------------------------------------------------------------------------- @@ -1583,7 +1587,7 @@ MIMEHdr::get_if_modified_since() inline time_t MIMEHdr::get_if_unmodified_since() { - return (value_get_date(MIME_FIELD_IF_UNMODIFIED_SINCE, MIME_LEN_IF_UNMODIFIED_SINCE)); + return value_get_date(MIME_FIELD_IF_UNMODIFIED_SINCE, MIME_LEN_IF_UNMODIFIED_SINCE); } /*------------------------------------------------------------------------- @@ -1592,7 +1596,7 @@ MIMEHdr::get_if_unmodified_since() inline time_t MIMEHdr::get_last_modified() { - return (value_get_date(MIME_FIELD_LAST_MODIFIED, MIME_LEN_LAST_MODIFIED)); + return value_get_date(MIME_FIELD_LAST_MODIFIED, MIME_LEN_LAST_MODIFIED); } /*------------------------------------------------------------------------- @@ -1601,7 +1605,7 @@ MIMEHdr::get_last_modified() inline time_t MIMEHdr::get_if_range_date() { - return (value_get_date(MIME_FIELD_IF_RANGE, MIME_LEN_IF_RANGE)); + return value_get_date(MIME_FIELD_IF_RANGE, MIME_LEN_IF_RANGE); } /*------------------------------------------------------------------------- @@ -1610,7 +1614,7 @@ MIMEHdr::get_if_range_date() inline int32_t MIMEHdr::get_max_forwards() { - return (value_get_int(MIME_FIELD_MAX_FORWARDS, MIME_LEN_MAX_FORWARDS)); + return value_get_int(MIME_FIELD_MAX_FORWARDS, MIME_LEN_MAX_FORWARDS); } /*------------------------------------------------------------------------- @@ -1622,7 +1626,7 @@ MIMEHdr::get_warning(int idx) (void)idx; // FIXME: what do we do here? ink_release_assert(!"unimplemented"); - return (0); + return 0; } /*------------------------------------------------------------------------- @@ -1631,7 +1635,7 @@ MIMEHdr::get_warning(int idx) inline uint32_t MIMEHdr::get_cooked_cc_mask() { - return (m_mime->m_cooked_stuff.m_cache_control.m_mask); + return m_mime->m_cooked_stuff.m_cache_control.m_mask; } /*------------------------------------------------------------------------- @@ -1640,7 +1644,7 @@ MIMEHdr::get_cooked_cc_mask() inline int32_t MIMEHdr::get_cooked_cc_max_age() { - return (m_mime->m_cooked_stuff.m_cache_control.m_secs_max_age); + return m_mime->m_cooked_stuff.m_cache_control.m_secs_max_age; } /*------------------------------------------------------------------------- @@ -1649,7 +1653,7 @@ MIMEHdr::get_cooked_cc_max_age() inline int32_t MIMEHdr::get_cooked_cc_s_maxage() { - return (m_mime->m_cooked_stuff.m_cache_control.m_secs_s_maxage); + return m_mime->m_cooked_stuff.m_cache_control.m_secs_s_maxage; } /*------------------------------------------------------------------------- @@ -1658,7 +1662,7 @@ MIMEHdr::get_cooked_cc_s_maxage() inline int32_t MIMEHdr::get_cooked_cc_max_stale() { - return (m_mime->m_cooked_stuff.m_cache_control.m_secs_max_stale); + return m_mime->m_cooked_stuff.m_cache_control.m_secs_max_stale; } /*------------------------------------------------------------------------- @@ -1667,7 +1671,7 @@ MIMEHdr::get_cooked_cc_max_stale() inline int32_t MIMEHdr::get_cooked_cc_min_fresh() { - return (m_mime->m_cooked_stuff.m_cache_control.m_secs_min_fresh); + return m_mime->m_cooked_stuff.m_cache_control.m_secs_min_fresh; } /*------------------------------------------------------------------------- @@ -1676,7 +1680,7 @@ MIMEHdr::get_cooked_cc_min_fresh() inline bool MIMEHdr::get_cooked_pragma_no_cache() { - return (m_mime->m_cooked_stuff.m_pragma.m_no_cache); + return m_mime->m_cooked_stuff.m_pragma.m_no_cache; } /*------------------------------------------------------------------------- diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 3b9ae712d05..533db3f4c87 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -992,9 +992,6 @@ HttpSM::state_read_push_response_header(int event, void *data) ink_assert(ua_entry->read_vio == (VIO *)data); ink_assert(t_state.current.server == nullptr); - int64_t data_size = 0; - int64_t bytes_used = 0; - switch (event) { case VC_EVENT_EOS: ua_entry->eos = true; @@ -1018,17 +1015,16 @@ HttpSM::state_read_push_response_header(int event, void *data) while (ua_buffer_reader->read_avail() && state == PARSE_RESULT_CONT) { const char *start = ua_buffer_reader->start(); const char *tmp = start; - data_size = ua_buffer_reader->block_read_avail(); + int64_t data_size = ua_buffer_reader->block_read_avail(); ink_assert(data_size >= 0); ///////////////////// // tokenize header // ///////////////////// - state = - t_state.hdr_info.server_response.parse_resp(&http_parser, &tmp, tmp + data_size, false // Only call w/ eof when data exhausted - ); + state = t_state.hdr_info.server_response.parse_resp(&http_parser, &tmp, tmp + data_size, + false); // Only call w/ eof when data exhausted - bytes_used = tmp - start; + int64_t bytes_used = tmp - start; ink_release_assert(bytes_used <= data_size); ua_buffer_reader->consume(bytes_used); @@ -1719,14 +1715,13 @@ HttpSM::state_http_server_open(int event, void *data) pending_action = nullptr; } milestones[TS_MILESTONE_SERVER_CONNECT_END] = Thread::get_hrtime(); - HttpServerSession *session; - NetVConnection *netvc = nullptr; + NetVConnection *netvc = nullptr; switch (event) { case NET_EVENT_OPEN: { - session = (TS_SERVER_SESSION_SHARING_POOL_THREAD == t_state.http_config_param->server_session_sharing_pool) ? - THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : - httpServerSessionAllocator.alloc(); + HttpServerSession *session = (TS_SERVER_SESSION_SHARING_POOL_THREAD == t_state.http_config_param->server_session_sharing_pool) ? + THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : + httpServerSessionAllocator.alloc(); session->sharing_pool = static_cast(t_state.http_config_param->server_session_sharing_pool); session->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); @@ -4398,8 +4393,6 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) void HttpSM::calculate_output_cl(int64_t num_chars_for_ct, int64_t num_chars_for_cl) { - int i; - if (t_state.range_setup != HttpTransact::RANGE_REQUESTED && t_state.range_setup != HttpTransact::RANGE_NOT_TRANSFORM_REQUESTED) { return; } @@ -4409,7 +4402,7 @@ HttpSM::calculate_output_cl(int64_t num_chars_for_ct, int64_t num_chars_for_cl) if (t_state.num_range_fields == 1) { t_state.range_output_cl = t_state.ranges[0]._end - t_state.ranges[0]._start + 1; } else { - for (i = 0; i < t_state.num_range_fields; i++) { + for (int i = 0; i < t_state.num_range_fields; i++) { if (t_state.ranges[i]._start >= 0) { t_state.range_output_cl += boundary_size; t_state.range_output_cl += sub_header_size + num_chars_for_ct; @@ -4446,9 +4439,6 @@ void HttpSM::do_range_setup_if_necessary() { MIMEField *field; - INKVConnInternal *range_trans; - int field_content_type_len = -1; - const char *content_type; ink_assert(t_state.cache_info.object_read != nullptr); @@ -4492,11 +4482,14 @@ HttpSM::do_range_setup_if_necessary() // We have to do the transform on (allowed) multi-range request, *or* if the VC is not pread capable if (do_transform) { if (api_hooks.get(TS_HTTP_RESPONSE_TRANSFORM_HOOK) == nullptr) { + int field_content_type_len = -1; + const char *content_type = t_state.cache_info.object_read->response_get()->value_get( + MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE, &field_content_type_len); + Debug("http_trans", "Unable to accelerate range request, fallback to transform"); - content_type = t_state.cache_info.object_read->response_get()->value_get(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE, - &field_content_type_len); + // create a Range: transform processor for requests of type Range: bytes=1-2,4-5,10-100 (eg. multiple ranges) - range_trans = transformProcessor.range_transform( + INKVConnInternal *range_trans = transformProcessor.range_transform( mutex.get(), t_state.ranges, t_state.num_range_fields, &t_state.hdr_info.transform_response, content_type, field_content_type_len, t_state.cache_info.object_read->object_size_get()); api_hooks.append(TS_HTTP_RESPONSE_TRANSFORM_HOOK, range_trans); @@ -4621,7 +4614,7 @@ HttpSM::do_cache_prepare_update() void HttpSM::do_cache_prepare_action(HttpCacheSM *c_sm, CacheHTTPInfo *object_read_info, bool retry, bool allow_multiple) { - URL *o_url, *c_url, *s_url; + URL *o_url, *s_url; bool restore_client_request = false; ink_assert(!pending_action); @@ -4643,7 +4636,7 @@ HttpSM::do_cache_prepare_action(HttpCacheSM *c_sm, CacheHTTPInfo *object_read_in // modify client request to make it have the url we are going to // store into the cache if (restore_client_request) { - c_url = t_state.hdr_info.client_request.url_get(); + URL *c_url = t_state.hdr_info.client_request.url_get(); s_url->copy(c_url); } @@ -5856,16 +5849,15 @@ HttpSM::issue_cache_update() int HttpSM::write_header_into_buffer(HTTPHdr *h, MIOBuffer *b) { - int bufindex; int dumpoffset; - int done, tmp; - IOBufferBlock *block; + int done; dumpoffset = 0; do { - bufindex = 0; - tmp = dumpoffset; - block = b->get_current_block(); + IOBufferBlock *block = b->get_current_block(); + int bufindex = 0; + int tmp = dumpoffset; + ink_assert(block->write_avail() > 0); done = h->print(block->start(), block->write_avail(), &bufindex, &tmp); dumpoffset += bufindex; @@ -6330,7 +6322,6 @@ HttpSM::setup_internal_transfer(HttpSMHandler handler_arg) int HttpSM::find_http_resp_buffer_size(int64_t content_length) { - int64_t buf_size; int64_t alloc_index; if (content_length == HTTP_UNDEFINED_CL) { @@ -6341,6 +6332,8 @@ HttpSM::find_http_resp_buffer_size(int64_t content_length) alloc_index = DEFAULT_RESPONSE_BUFFER_SIZE_INDEX; } } else { + int64_t buf_size; + #ifdef WRITE_AND_TRANSFER buf_size = HTTP_HEADER_BUFFER_SIZE + content_length - index_to_buffer_size(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); #else @@ -7796,9 +7789,9 @@ HttpSM::redirect_request(const char *arg_redirect_url, const int arg_redirect_le bool noPortInHost = HttpConfig::m_master.redirection_host_no_port; - bool isRedirectUrlOriginForm = clientUrl.m_url_impl->m_len_scheme <= 0 && clientUrl.m_url_impl->m_len_user <= 0 && - clientUrl.m_url_impl->m_len_password <= 0 && clientUrl.m_url_impl->m_len_host <= 0 && - clientUrl.m_url_impl->m_len_port <= 0; + bool isRedirectUrlOriginForm = !clientUrl.m_url_impl->m_len_scheme && !clientUrl.m_url_impl->m_len_user && + !clientUrl.m_url_impl->m_len_password && !clientUrl.m_url_impl->m_len_host && + !clientUrl.m_url_impl->m_len_port; // check to see if the client request passed a host header, if so copy the host and port from the redirect url and // make a new host header diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index 30ace9ed0b4..fe317290959 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -99,7 +99,7 @@ struct HttpVCTableEntry { struct HttpVCTable { static const int vc_table_max_entries = 4; - HttpVCTable(HttpSM *); + explicit HttpVCTable(HttpSM *); HttpVCTableEntry *new_entry(); HttpVCTableEntry *find_entry(VConnection *); diff --git a/proxy/http/HttpTunnel.h b/proxy/http/HttpTunnel.h index 9a91f5026e6..272b33e674f 100644 --- a/proxy/http/HttpTunnel.h +++ b/proxy/http/HttpTunnel.h @@ -517,7 +517,7 @@ inline bool HttpTunnelConsumer::is_downstream_from(VConnection *vc) { HttpTunnelProducer *p = producer; - HttpTunnelConsumer *c; + while (p) { if (p->vc == vc) { return true; @@ -525,7 +525,8 @@ HttpTunnelConsumer::is_downstream_from(VConnection *vc) // The producer / consumer chain can contain a cycle in the case // of a blind tunnel so give up if we find ourself (the original // consumer). - c = p->self_consumer; + HttpTunnelConsumer *c = p->self_consumer; + p = (c && c != this) ? c->producer : nullptr; } return false; diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc index e3870ac447a..9762dae7ea5 100644 --- a/proxy/logging/Log.cc +++ b/proxy/logging/Log.cc @@ -278,7 +278,10 @@ struct LoggingPreprocContinuation : public Continuation { return 0; } - LoggingPreprocContinuation(int idx) : Continuation(nullptr), m_idx(idx) { SET_HANDLER(&LoggingPreprocContinuation::mainEvent); } + explicit LoggingPreprocContinuation(int idx) : Continuation(nullptr), m_idx(idx) + { + SET_HANDLER(&LoggingPreprocContinuation::mainEvent); + } }; struct LoggingFlushContinuation : public Continuation { @@ -291,7 +294,10 @@ struct LoggingFlushContinuation : public Continuation { return 0; } - LoggingFlushContinuation(int idx) : Continuation(nullptr), m_idx(idx) { SET_HANDLER(&LoggingFlushContinuation::mainEvent); } + explicit LoggingFlushContinuation(int idx) : Continuation(nullptr), m_idx(idx) + { + SET_HANDLER(&LoggingFlushContinuation::mainEvent); + } }; struct LoggingCollateContinuation : public Continuation { @@ -1252,11 +1258,10 @@ Log::preproc_thread_main(void *args) if (TSSystemState::is_event_system_shut_down()) { return nullptr; } - size_t buffers_preproced = 0; - LogConfig *current = (LogConfig *)configProcessor.get(log_configid); + LogConfig *current = static_cast(configProcessor.get(log_configid)); if (likely(current)) { - buffers_preproced = current->log_object_manager.preproc_buffers(idx); + size_t buffers_preproced = current->log_object_manager.preproc_buffers(idx); // config->increment_space_used(bytes_to_disk); // TODO: the bytes_to_disk should be set to Log @@ -1295,7 +1300,7 @@ Log::flush_thread_main(void * /* args ATS_UNUSED */) if (TSSystemState::is_event_system_shut_down()) { return nullptr; } - fdata = (LogFlushData *)ink_atomiclist_popall(flush_data_list); + fdata = static_cast(ink_atomiclist_popall(flush_data_list)); // invert the list // @@ -1312,7 +1317,7 @@ Log::flush_thread_main(void * /* args ATS_UNUSED */) LogFile *logfile = fdata->m_logfile.get(); if (logfile->m_file_format == LOG_FILE_BINARY) { - logbuffer = (LogBuffer *)fdata->m_data; + logbuffer = static_cast(fdata->m_data); LogBufferHeader *buffer_header = logbuffer->header(); buf = (char *)buffer_header; @@ -1475,7 +1480,7 @@ Log::collate_thread_main(void * /* args ATS_UNUSED */) } Debug("log-sock", "pending message ..."); - header = (LogBufferHeader *)sock->read_alloc(sock_id, &bytes_read); + header = static_cast(sock->read_alloc(sock_id, &bytes_read)); if (!header) { Debug("log-sock", "Error reading LogBuffer from collation client"); continue; @@ -1536,9 +1541,9 @@ Log::match_logobject(LogBufferHeader *header) LogFormat fmt("__collation_format__", header->fmt_fieldlist(), header->fmt_printf()); if (fmt.valid()) { - LogFileFormat file_format = header->log_object_flags & LogObject::BINARY ? + LogFileFormat file_format = (header->log_object_flags & LogObject::BINARY) ? LOG_FILE_BINARY : - (header->log_object_flags & LogObject::WRITES_TO_PIPE ? LOG_FILE_PIPE : LOG_FILE_ASCII); + ((header->log_object_flags & LogObject::WRITES_TO_PIPE) ? LOG_FILE_PIPE : LOG_FILE_ASCII); obj = new LogObject(&fmt, Log::config->logfile_dir, header->log_filename(), file_format, nullptr, (Log::RollingEnabledValues)Log::config->rolling_enabled, Log::config->collation_preproc_threads, diff --git a/proxy/logging/Log.h b/proxy/logging/Log.h index 00e0d1bb8f5..d266019a2d6 100644 --- a/proxy/logging/Log.h +++ b/proxy/logging/Log.h @@ -87,7 +87,7 @@ class LogFlushData { switch (m_logfile->m_file_format) { case LOG_FILE_BINARY: - logbuffer = (LogBuffer *)m_data; + logbuffer = static_cast(m_data); LogBuffer::destroy(logbuffer); break; case LOG_FILE_ASCII: diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index da106fa3b4d..43e5c751e86 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -1332,9 +1332,6 @@ LogAccess::validate_unmapped_url() void LogAccess::validate_unmapped_url_path() { - int len; - char *c; - if (m_client_req_unmapped_url_path_str == nullptr && m_client_req_unmapped_url_host_str == nullptr) { // Use unmapped canonical URL as default m_client_req_unmapped_url_path_str = m_client_req_unmapped_url_canon_str; @@ -1343,7 +1340,9 @@ LogAccess::validate_unmapped_url_path() m_client_req_unmapped_url_host_str = INVALID_STR; if (m_client_req_unmapped_url_path_len >= 6) { // xxx:// - minimum schema size - c = (char *)memchr((void *)m_client_req_unmapped_url_path_str, ':', m_client_req_unmapped_url_path_len - 1); + int len; + char *c = (char *)memchr((void *)m_client_req_unmapped_url_path_str, ':', m_client_req_unmapped_url_path_len - 1); + if (c && (len = (int)(c - m_client_req_unmapped_url_path_str)) <= 5) { // 5 - max schema size if (len + 2 <= m_client_req_unmapped_url_canon_len && c[1] == '/' && c[2] == '/') { len += 3; // Skip "://" diff --git a/proxy/logging/LogAccess.h b/proxy/logging/LogAccess.h index c91520412f4..1d35627904d 100644 --- a/proxy/logging/LogAccess.h +++ b/proxy/logging/LogAccess.h @@ -118,7 +118,7 @@ class LogAccess { } - LogAccess(HttpSM *sm); + explicit LogAccess(HttpSM *sm); inkcoreapi ~LogAccess() {} inkcoreapi void init(); diff --git a/proxy/logging/LogField.h b/proxy/logging/LogField.h index 94823e4d8f3..e02dba36ab9 100644 --- a/proxy/logging/LogField.h +++ b/proxy/logging/LogField.h @@ -50,7 +50,7 @@ struct LogSlice { // Initialize LogSlice by slice notation, // the str looks like: "xxx[0:30]". // - LogSlice(char *str); + explicit LogSlice(char *str); // // Convert slice notation to target string's offset, diff --git a/proxy/logging/LogObject.h b/proxy/logging/LogObject.h index 761530c8c16..dff58531f54 100644 --- a/proxy/logging/LogObject.h +++ b/proxy/logging/LogObject.h @@ -234,12 +234,12 @@ class LogObject : public RefCountObj inline bool receives_remote_data() const { - return m_flags & REMOTE_DATA ? true : false; + return (m_flags & REMOTE_DATA) ? true : false; } inline bool writes_to_pipe() const { - return m_flags & WRITES_TO_PIPE ? true : false; + return (m_flags & WRITES_TO_PIPE) ? true : false; } inline bool writes_to_disk() diff --git a/proxy/logging/LogSock.h b/proxy/logging/LogSock.h index d1ad1e9389c..f2eb70e8324 100644 --- a/proxy/logging/LogSock.h +++ b/proxy/logging/LogSock.h @@ -61,7 +61,7 @@ class LogSock LS_N_STATES, }; - LogSock(int max_connects = 1); + explicit LogSock(int max_connects = 1); ~LogSock(); bool pending_any(int *cid, int timeout_msec = 0); From 9a859170e7be826b5de645c66425bd66b315ee08 Mon Sep 17 00:00:00 2001 From: Nozomi Inanami <42267827+nozomi1773@users.noreply.github.com> Date: Tue, 16 Apr 2019 20:41:50 +0900 Subject: [PATCH 431/526] fix proxy.config.hostdb.timeout default --- doc/admin-guide/files/records.config.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 44aedd521d7..6eb2fc06743 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -2577,7 +2577,7 @@ HostDB value become a minimum TTL. ===== ====================================================================== -.. ts:cv:: CONFIG proxy.config.hostdb.timeout INT 1440 +.. ts:cv:: CONFIG proxy.config.hostdb.timeout INT 86400 :units: seconds :reloadable: From 140add06850374eaa2369edd3edf645d12455d75 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Mon, 15 Apr 2019 18:58:46 -0600 Subject: [PATCH 432/526] Fixes the Brotli build issues --- src/traffic_layout/Makefile.inc | 2 +- src/traffic_layout/info.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/traffic_layout/Makefile.inc b/src/traffic_layout/Makefile.inc index afdc3141825..fd37a51123e 100644 --- a/src/traffic_layout/Makefile.inc +++ b/src/traffic_layout/Makefile.inc @@ -29,7 +29,7 @@ traffic_layout_traffic_layout_CPPFLAGS = \ traffic_layout_traffic_layout_LDFLAGS = \ $(AM_LDFLAGS) \ - @YAMLCPP_LDFLAGS@ + @YAMLCPP_LDFLAGS@ $(BROTLIENC_LIB) traffic_layout_traffic_layout_SOURCES = \ traffic_layout/traffic_layout.cc \ diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 74af91fcb9e..184049d1abf 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -199,7 +199,7 @@ produce_versions(bool json) print_var("lzma", undef, json); #endif #if HAVE_BROTLI_ENCODE_H - print_var("lzma", LBW().print("{}", BrotliEncoderVersion).view(), json); + print_var("brotli", LBW().print("{:#x}", BrotliEncoderVersion()).view(), json); #else print_var("brotli", undef, json); #endif From 91b05598bc07d4e8a97281f4d5a556b7956d803e Mon Sep 17 00:00:00 2001 From: chenggang7 Date: Sun, 14 Apr 2019 21:27:12 +0800 Subject: [PATCH 433/526] A recipient MUST ignore If-Modified-Since if the request contains an If-None-Match header field, about rfc https://tools.ietf.org/html/rfc7232#section-3.3 --- proxy/http/HttpTransactCache.cc | 59 +++++++++++++++------------------ 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/proxy/http/HttpTransactCache.cc b/proxy/http/HttpTransactCache.cc index 0b5ecdf92f1..56adb7ba623 100644 --- a/proxy/http/HttpTransactCache.cc +++ b/proxy/http/HttpTransactCache.cc @@ -1288,6 +1288,33 @@ HttpTransactCache::match_response_to_request_conditionals(HTTPHdr *request, HTTP return response->status_get(); } + // If-None-Match: may match weakly // + if (request->presence(MIME_PRESENCE_IF_NONE_MATCH)) { + int raw_etags_len, comma_sep_tag_list_len; + const char *raw_etags = response->value_get(MIME_FIELD_ETAG, MIME_LEN_ETAG, &raw_etags_len); + const char *comma_sep_tag_list = nullptr; + + if (raw_etags) { + comma_sep_tag_list = request->value_get(MIME_FIELD_IF_NONE_MATCH, MIME_LEN_IF_NONE_MATCH, &comma_sep_tag_list_len); + if (!comma_sep_tag_list) { + comma_sep_tag_list = ""; + comma_sep_tag_list_len = 0; + } + + //////////////////////////////////////////////////////////////////////// + // If we have an etag and a if-none-match, we are talking to someone // + // who is doing a 1.1 revalidate. Since this is a GET request with no // + // sub-ranges, we can do a weak validation. // + //////////////////////////////////////////////////////////////////////// + if (do_strings_match_weakly(raw_etags, raw_etags_len, comma_sep_tag_list, comma_sep_tag_list_len)) { + // the response already failed If-modified-since (if one exists) + return HTTP_STATUS_NOT_MODIFIED; + } else { + return response->status_get(); + } + } + } + // If-Modified-Since // if (request->presence(MIME_PRESENCE_IF_MODIFIED_SINCE)) { if (response->presence(MIME_PRESENCE_LAST_MODIFIED)) { @@ -1316,38 +1343,6 @@ HttpTransactCache::match_response_to_request_conditionals(HTTPHdr *request, HTTP response_code = HTTP_STATUS_NOT_MODIFIED; } - - // we cannot return NOT_MODIFIED yet, need to check If-none-match - if (!request->presence(MIME_PRESENCE_IF_NONE_MATCH)) { - return response_code; - } - } - - // If-None-Match: may match weakly // - if (request->presence(MIME_PRESENCE_IF_NONE_MATCH)) { - int raw_etags_len, comma_sep_tag_list_len; - const char *raw_etags = response->value_get(MIME_FIELD_ETAG, MIME_LEN_ETAG, &raw_etags_len); - const char *comma_sep_tag_list = nullptr; - - if (raw_etags) { - comma_sep_tag_list = request->value_get(MIME_FIELD_IF_NONE_MATCH, MIME_LEN_IF_NONE_MATCH, &comma_sep_tag_list_len); - if (!comma_sep_tag_list) { - comma_sep_tag_list = ""; - comma_sep_tag_list_len = 0; - } - - //////////////////////////////////////////////////////////////////////// - // If we have an etag and a if-none-match, we are talking to someone // - // who is doing a 1.1 revalidate. Since this is a GET request with no // - // sub-ranges, we can do a weak validation. // - //////////////////////////////////////////////////////////////////////// - if (do_strings_match_weakly(raw_etags, raw_etags_len, comma_sep_tag_list, comma_sep_tag_list_len)) { - // the response already failed If-modified-since (if one exists) - return HTTP_STATUS_NOT_MODIFIED; - } else { - return response->status_get(); - } - } } // There is no If-none-match, and If-modified-since failed, From 13425d228263c4cba295059a90b5888b892dc057 Mon Sep 17 00:00:00 2001 From: Aaron Canary Date: Wed, 17 Apr 2019 13:40:31 -0500 Subject: [PATCH 434/526] -H "xdebug: probe" injects trace of headers into response body With the intent to debug CDN, peered or multi-tiered proxy behavior, this header will trigger the code to write the request and response headers to the response body at each proxy the request is processed. This has been proven to be significantly easier to use than log-headers. As with all xdebug features some security layer should be applied to control who is allowed to see this type of response data. docs asf license include unistd xdebug probe in json using std::string_view --- doc/admin-guide/plugins/xdebug.en.rst | 19 ++- plugins/xdebug/xdebug.cc | 130 +++++++++------------ plugins/xdebug/xdebug_headers.cc | 159 ++++++++++++++++++++++++++ plugins/xdebug/xdebug_transforms.cc | 158 +++++++++++++++++++++++++ 4 files changed, 382 insertions(+), 84 deletions(-) create mode 100644 plugins/xdebug/xdebug_headers.cc create mode 100644 plugins/xdebug/xdebug_transforms.cc diff --git a/doc/admin-guide/plugins/xdebug.en.rst b/doc/admin-guide/plugins/xdebug.en.rst index 34f77a87189..a9eeac71a14 100644 --- a/doc/admin-guide/plugins/xdebug.en.rst +++ b/doc/admin-guide/plugins/xdebug.en.rst @@ -57,11 +57,20 @@ Diags transaction specific diagnostics for the transaction. This also requires that :ts:cv:`proxy.config.diags.debug.enabled` is set to ``1``. -log-headers - If the ``log-headers`` is requested while :ts:cv:`proxy.config.diags.debug.tags` - is set to ``xdebug.headers`` and :ts:cv:`proxy.config.diags.debug.enabled` is set to ``1``, - then all client and server, request and response headers are logged. - Also, the ``X-Debug: log-headers`` header is always added to the upstream request. +Probe + All request and response headers are written to the response body. Because + the body is altered, it disables writing to cache. + In conjuction with the `fwd` tag, the response body will contain a + chronological log of all headers for all transactions used for this + response. + + Layout: + + - Request Headers from Client -> Proxy A + - Request Headers from Proxy A -> Proxy B + - Original content body + - Response Headers from Proxy B -> Proxy A + - Response Headers from Proxy A -> Client X-Cache-Key The ``X-Cache-Key`` header contains the URL that identifies the HTTP object in the diff --git a/plugins/xdebug/xdebug.cc b/plugins/xdebug/xdebug.cc index c6eefd033db..a2d54b8becf 100644 --- a/plugins/xdebug/xdebug.cc +++ b/plugins/xdebug/xdebug.cc @@ -26,13 +26,15 @@ #include #include #include +#include #include #include "tscore/ink_defs.h" #include "tscpp/util/PostScript.h" #include "tscpp/util/TextView.h" -#define DEBUG_TAG_LOG_HEADERS "xdebug.headers" +#include "xdebug_headers.cc" +#include "xdebug_transforms.cc" static struct { const char *str; @@ -47,9 +49,11 @@ enum { XHEADER_X_TRANSACTION_ID = 1u << 6, XHEADER_X_DUMP_HEADERS = 1u << 7, XHEADER_X_REMAP = 1u << 8, + XHEADER_X_PROBE_HEADERS = 1u << 9, }; static int XArgIndex = 0; +static int BodyBuilderArgIndex = 0; static TSCont XInjectHeadersCont = nullptr; static TSCont XDeleteDebugHdrCont = nullptr; @@ -321,48 +325,6 @@ InjectTxnUuidHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) } } -/////////////////////////////////////////////////////////////////////////// -// Dump a header on stderr, useful together with TSDebug(). -void -log_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, const char *msg_type) -{ - if (!TSIsDebugTagSet(DEBUG_TAG_LOG_HEADERS)) { - return; - } - - TSIOBuffer output_buffer; - TSIOBufferReader reader; - TSIOBufferBlock block; - const char *block_start; - int64_t block_avail; - - std::stringstream ss; - ss << "TxnID:" << TSHttpTxnIdGet(txn) << " " << msg_type << " Headers are..."; - - output_buffer = TSIOBufferCreate(); - reader = TSIOBufferReaderAlloc(output_buffer); - - /* This will print just MIMEFields and not the http request line */ - TSMimeHdrPrint(bufp, hdr_loc, output_buffer); - - /* We need to loop over all the buffer blocks, there can be more than 1 */ - block = TSIOBufferReaderStart(reader); - do { - block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); - if (block_avail > 0) { - ss << "\n" << std::string(block_start, static_cast(block_avail)); - } - TSIOBufferReaderConsume(reader, block_avail); - block = TSIOBufferReaderStart(reader); - } while (block && block_avail != 0); - - /* Free up the TSIOBuffer that we used to print out the header */ - TSIOBufferReaderFree(reader); - TSIOBufferDestroy(output_buffer); - - TSDebug(DEBUG_TAG_LOG_HEADERS, "%s", ss.str().c_str()); -} - static int XInjectResponseHeaders(TSCont /* contp */, TSEvent event, void *edata) { @@ -401,12 +363,25 @@ XInjectResponseHeaders(TSCont /* contp */, TSEvent event, void *edata) InjectTxnUuidHeader(txn, buffer, hdr); } + if (xheaders & XHEADER_X_REMAP) { + InjectRemapHeader(txn, buffer, hdr); + } + + // intentionally placed after all injected headers. + if (xheaders & XHEADER_X_DUMP_HEADERS) { log_headers(txn, buffer, hdr, "ClientResponse"); } - if (xheaders & XHEADER_X_REMAP) { - InjectRemapHeader(txn, buffer, hdr); + if (xheaders & XHEADER_X_PROBE_HEADERS) { + BodyBuilder *data = static_cast(TSHttpTxnArgGet(txn, BodyBuilderArgIndex)); + TSDebug("xdebug_transform", "XInjectResponseHeaders(): client resp header ready"); + if (data == nullptr) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_ERROR); + return TS_ERROR; + } + data->hdr_ready = true; + writePostBody(data); } done: @@ -519,42 +494,36 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) } else if (header_field_eq("diags", value, vsize)) { // Enable diagnostics for DebugTxn()'s only TSHttpTxnDebugSet(txn, 1); - } else if (header_field_eq("log-headers", value, vsize)) { - xheaders |= XHEADER_X_DUMP_HEADERS; - log_headers(txn, buffer, hdr, "ClientRequest"); - - // dump on server request - auto send_req_dump = [](TSCont /* contp */, TSEvent event, void *edata) -> int { - TSHttpTxn txn = (TSHttpTxn)edata; - TSMBuffer buffer; - TSMLoc hdr; - if (TSHttpTxnServerReqGet(txn, &buffer, &hdr) == TS_SUCCESS) { - // re-add header "X-Debug: log-headers", but only once - TSMLoc dst = TSMimeHdrFieldFind(buffer, hdr, xDebugHeader.str, xDebugHeader.len); - if (dst == TS_NULL_MLOC) { - if (TSMimeHdrFieldCreateNamed(buffer, hdr, xDebugHeader.str, xDebugHeader.len, &dst) == TS_SUCCESS) { - TSReleaseAssert(TSMimeHdrFieldAppend(buffer, hdr, dst) == TS_SUCCESS); - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, "log-headers", - lengthof("log-headers")) == TS_SUCCESS); - log_headers(txn, buffer, hdr, "ServerRequest"); - } - } - } - return TS_EVENT_NONE; - }; - TSHttpTxnHookAdd(txn, TS_HTTP_SEND_REQUEST_HDR_HOOK, TSContCreate(send_req_dump, nullptr)); - - // dump on server response - auto read_resp_dump = [](TSCont /* contp */, TSEvent event, void *edata) -> int { - TSHttpTxn txn = (TSHttpTxn)edata; - TSMBuffer buffer; - TSMLoc hdr; - if (TSHttpTxnServerRespGet(txn, &buffer, &hdr) == TS_SUCCESS) { - log_headers(txn, buffer, hdr, "ServerResponse"); - } + + } else if (header_field_eq("probe", value, vsize)) { + xheaders |= XHEADER_X_PROBE_HEADERS; + + // prefix request headers and postfix response headers + BodyBuilder *data = new BodyBuilder(); + data->txn = txn; + + TSVConn connp = TSTransformCreate(body_transform, txn); + TSContDataSet(connp, data); + TSHttpTxnHookAdd(txn, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); + + // store data pointer in txnarg to use in global cont XInjectResponseHeaders + TSHttpTxnArgSet(txn, BodyBuilderArgIndex, data); + + // create a self-cleanup on close + auto cleanupBodyBuilder = [](TSCont /* contp */, TSEvent event, void *edata) -> int { + TSHttpTxn txn = (TSHttpTxn)edata; + BodyBuilder *data = static_cast(TSHttpTxnArgGet(txn, BodyBuilderArgIndex)); + delete data; return TS_EVENT_NONE; }; - TSHttpTxnHookAdd(txn, TS_HTTP_READ_RESPONSE_HDR_HOOK, TSContCreate(read_resp_dump, nullptr)); + TSHttpTxnHookAdd(txn, TS_HTTP_TXN_CLOSE_HOOK, TSContCreate(cleanupBodyBuilder, nullptr)); + + // disable writing to cache because we are injecting data into the body. + TSHttpTxnReqCacheableSet(txn, 0); + TSHttpTxnRespCacheableSet(txn, 0); + TSHttpTxnServerRespNoStoreSet(txn, 1); + TSHttpTxnTransformedRespCache(txn, 0); + TSHttpTxnUntransformedRespCache(txn, 0); } else if (isFwdFieldValue(std::string_view(value, vsize), fwdCnt)) { if (fwdCnt > 0) { @@ -666,7 +635,10 @@ TSPluginInit(int argc, const char *argv[]) // Setup the global hook TSReleaseAssert(TSHttpTxnArgIndexReserve("xdebug", "xdebug header requests", &XArgIndex) == TS_SUCCESS); + TSReleaseAssert(TSHttpTxnArgIndexReserve("bodyTransform", "BodyBuilder*", &XArgIndex) == TS_SUCCESS); TSReleaseAssert(XInjectHeadersCont = TSContCreate(XInjectResponseHeaders, nullptr)); TSReleaseAssert(XDeleteDebugHdrCont = TSContCreate(XDeleteDebugHdr, nullptr)); TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(XScanRequestHeaders, nullptr)); + + gethostname(Hostname, 1024); } diff --git a/plugins/xdebug/xdebug_headers.cc b/plugins/xdebug/xdebug_headers.cc new file mode 100644 index 00000000000..51f5d86c20d --- /dev/null +++ b/plugins/xdebug/xdebug_headers.cc @@ -0,0 +1,159 @@ +/** @file + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG_TAG_LOG_HEADERS "xdebug.headers" + +std::string_view +escape_char_for_json(char const &c, bool &parsing_key) +{ + switch (c) { + case '\'': + return {"\\\'"}; + case '"': + return {"\\\""}; + case '\\': + return {"\\\\"}; + case '\b': + return {"\\b"}; + case '\f': + return {"\\f"}; + case '\t': + return {"\\t"}; + + // Special header reformating + case '\r': + return {""}; + case '\n': + parsing_key = true; + return {"',\r\n\t'"}; // replace new line with pair delemiter + case ':': + if (parsing_key) { + return {"' : "}; // replace colon after keywith quote + colon + } + return {":"}; + case ' ': + if (parsing_key) { + parsing_key = false; + return {"'"}; // replace first space after the key to be a quote + } + return {" "}; + default: + return {&c, 1}; + } +} + +/////////////////////////////////////////////////////////////////////////// +// Dump a header on stderr, useful together with TSDebug(). +void +print_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, std::stringstream &ss) +{ + TSIOBuffer output_buffer; + TSIOBufferReader reader; + TSIOBufferBlock block; + const char *block_start; + int64_t block_avail; + bool parsing_key = true; + size_t print_rewind = ss.str().length(); + output_buffer = TSIOBufferCreate(); + reader = TSIOBufferReaderAlloc(output_buffer); + + ss << "\t'"; + /* This will print just MIMEFields and not the http request line */ + TSMimeHdrPrint(bufp, hdr_loc, output_buffer); + + /* We need to loop over all the buffer blocks, there can be more than 1 */ + block = TSIOBufferReaderStart(reader); + do { + block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); + for (const char *c = block_start; c < block_start + block_avail; ++c) { + bool was_parsing_key = parsing_key; + ss << escape_char_for_json(*c, parsing_key); + if (parsing_key && !was_parsing_key) { + print_rewind = ss.str().length() - 1; + } + } + TSIOBufferReaderConsume(reader, block_avail); + block = TSIOBufferReaderStart(reader); + } while (block && block_avail != 0); + + ss.seekp(print_rewind); + + /* Free up the TSIOBuffer that we used to print out the header */ + TSIOBufferReaderFree(reader); + TSIOBufferDestroy(output_buffer); + + TSDebug(DEBUG_TAG_LOG_HEADERS, "%s", ss.str().c_str()); +} + +void +log_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, char const *type_msg) +{ + if (TSIsDebugTagSet(DEBUG_TAG_LOG_HEADERS)) { + std::stringstream output; + print_headers(txn, bufp, hdr_loc, output); + TSDebug(DEBUG_TAG_LOG_HEADERS, "\n=============\n %s headers are... \n %s", type_msg, output.str().c_str()); + } +} + +void +print_request_headers(TSHttpTxn txn, std::stringstream &output) +{ + TSMBuffer buf_c, buf_s; + TSMLoc hdr_loc; + if (TSHttpTxnClientReqGet(txn, &buf_c, &hdr_loc) == TS_SUCCESS) { + output << "{'type':'request', 'side':'client', 'headers': {\n"; + print_headers(txn, buf_c, hdr_loc, output); + output << "}}"; + TSHandleMLocRelease(buf_c, TS_NULL_MLOC, hdr_loc); + } + if (TSHttpTxnServerReqGet(txn, &buf_s, &hdr_loc) == TS_SUCCESS) { + output << ",{'type':'request', 'side':'server', 'headers': {\n"; + print_headers(txn, buf_s, hdr_loc, output); + output << "}}"; + TSHandleMLocRelease(buf_s, TS_NULL_MLOC, hdr_loc); + } +} + +void +print_response_headers(TSHttpTxn txn, std::stringstream &output) +{ + TSMBuffer buf_c, buf_s; + TSMLoc hdr_loc; + if (TSHttpTxnServerRespGet(txn, &buf_s, &hdr_loc) == TS_SUCCESS) { + output << "{'type':'response', 'side':'server', 'headers': {\n"; + print_headers(txn, buf_s, hdr_loc, output); + output << "}},"; + TSHandleMLocRelease(buf_s, TS_NULL_MLOC, hdr_loc); + } + if (TSHttpTxnClientRespGet(txn, &buf_c, &hdr_loc) == TS_SUCCESS) { + output << "{'type':'response', 'side':'client', 'headers': {\n"; + print_headers(txn, buf_c, hdr_loc, output); + output << "}}"; + TSHandleMLocRelease(buf_c, TS_NULL_MLOC, hdr_loc); + } +} diff --git a/plugins/xdebug/xdebug_transforms.cc b/plugins/xdebug/xdebug_transforms.cc new file mode 100644 index 00000000000..4ceeea1a6c0 --- /dev/null +++ b/plugins/xdebug/xdebug_transforms.cc @@ -0,0 +1,158 @@ +/** @file + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "ts/ts.h" + +static const std::string_view MultipartBoundary{"\r\n--- ATS xDebug Probe Injection Boundary ---\r\n\r\n"}; + +struct BodyBuilder { + TSVIO output_vio = nullptr; + TSIOBuffer output_buffer = nullptr; + TSIOBufferReader output_reader = nullptr; + bool wrote_prebody = false; + bool wrote_body = false; + bool hdr_ready = false; + std::atomic_flag wrote_postbody; + + int64_t nbytes = 0; + TSHttpTxn txn = nullptr; +}; + +static char Hostname[1024]; + +static std::string +getPreBody(TSHttpTxn txn) +{ + std::stringstream output; + output << "{'xDebugProbeAt' : '" << Hostname << "'\n 'captured':["; + print_request_headers(txn, output); + output << "\n ]\n}"; + output << MultipartBoundary; + return output.str(); +} + +static std::string +getPostBody(TSHttpTxn txn) +{ + std::stringstream output; + output << MultipartBoundary; + output << "{'xDebugProbeAt' : '" << Hostname << "'\n 'captured':["; + print_response_headers(txn, output); + output << "\n ]\n}"; + return output.str(); +} + +static void +writePostBody(BodyBuilder *data) +{ + if (data->wrote_body && data->hdr_ready && !data->wrote_postbody.test_and_set()) { + TSDebug("xdebug_transform", "body_transform(): Writing postbody headers..."); + std::string postbody = getPostBody(data->txn); + TSIOBufferWrite(data->output_buffer, postbody.data(), postbody.length()); + data->nbytes += postbody.length(); + TSVIONBytesSet(data->output_vio, data->nbytes); + TSVIOReenable(data->output_vio); + } +} + +static int +body_transform(TSCont contp, TSEvent event, void *edata) +{ + BodyBuilder *data = static_cast(TSContDataGet(contp)); + if (!data) { + TSContDestroy(contp); + return TS_ERROR; + } + if (TSVConnClosedGet(contp)) { + // write connection destoried. cleanup. + delete data; + TSContDestroy(contp); + return 0; + } + + TSVIO src_vio = TSVConnWriteVIOGet(contp); + + switch (event) { + case TS_EVENT_ERROR: { + // Notify input vio of this error event + TSContCall(TSVIOContGet(src_vio), TS_EVENT_ERROR, src_vio); + return 0; + } + case TS_EVENT_VCONN_WRITE_COMPLETE: { + TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1); + return 0; + } + case TS_EVENT_VCONN_WRITE_READY: + TSDebug("xdebug_transform", "body_transform(): Event is TS_EVENT_VCONN_WRITE_READY"); + // fallthru + default: + if (!data->output_buffer) { + data->output_buffer = TSIOBufferCreate(); + data->output_reader = TSIOBufferReaderAlloc(data->output_buffer); + data->output_vio = TSVConnWrite(TSTransformOutputVConnGet(contp), contp, data->output_reader, INT64_MAX); + } + + if (data->wrote_prebody == false) { + TSDebug("xdebug_transform", "body_transform(): Writing prebody headers..."); + std::string prebody = getPreBody(data->txn); + TSIOBufferWrite(data->output_buffer, prebody.data(), prebody.length()); // write prebody + data->wrote_prebody = true; + data->nbytes += prebody.length(); + } + + TSIOBuffer src_buf = TSVIOBufferGet(src_vio); + + if (!src_buf) { + // upstream continuation shuts down write operation. + data->wrote_body = true; + writePostBody(data); + return 0; + } + + int64_t towrite = TSVIONTodoGet(src_vio); + TSDebug("xdebug_transform", "body_transform(): %li bytes of body is expected", towrite); + int64_t avail = TSIOBufferReaderAvail(TSVIOReaderGet(src_vio)); + towrite = towrite > avail ? avail : towrite; + if (towrite > 0) { + TSIOBufferCopy(TSVIOBufferGet(data->output_vio), TSVIOReaderGet(src_vio), towrite, 0); + TSIOBufferReaderConsume(TSVIOReaderGet(src_vio), towrite); + TSVIONDoneSet(src_vio, TSVIONDoneGet(src_vio) + towrite); + TSDebug("xdebug_transform", "body_transform(): writing %li bytes of body", towrite); + } + + if (TSVIONTodoGet(src_vio) > 0) { + TSVIOReenable(data->output_vio); + TSContCall(TSVIOContGet(src_vio), TS_EVENT_VCONN_WRITE_READY, src_vio); + } else { + // End of src vio + // Write post body content and update output VIO + data->wrote_body = true; + data->nbytes += TSVIONDoneGet(src_vio); + writePostBody(data); + TSContCall(TSVIOContGet(src_vio), TS_EVENT_VCONN_WRITE_COMPLETE, src_vio); + } + } + return 0; +} From 153ee3ba78c4b277823b668a964ffa92625772db Mon Sep 17 00:00:00 2001 From: hankai17 <867614535@qq.com> Date: Wed, 17 Apr 2019 23:06:22 +0800 Subject: [PATCH 435/526] Update README.md --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index f16b2932eb9..84fb30f5c30 100644 --- a/tests/README.md +++ b/tests/README.md @@ -66,7 +66,7 @@ tr.Processes.Default.Env=ts.Env These are the current variables that are defined dynamically for Trafficserver port - the ipv4 port to listen on -portv6 - the ipv4 port to listen on +portv6 - the ipv6 port to listen on manager_port - the manager port used. This is set even is select_port is False admin_port - the admin port used. This is set even is select_port is False From 789d7a8b798de5b0dee2c83c7ff27b4b201f8919 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Fri, 15 Mar 2019 13:59:24 -0700 Subject: [PATCH 436/526] traffic_via.pl: Fixed bugs, added tests, and make the output match more like traffic_via. --- tools/traffic_via/test_traffic_via_pl | 47 +++++ .../tests/[u c s f p eS;tNc p s ] | 13 ++ .../tests/[uIcRs f p eN;t cCHp s ] | 13 ++ .../tests/[uIcRs f p eN;t cCNp s ] | 13 ++ .../tests/[uScMsSf pSeN;t cCMp sS] | 13 ++ .../tests/[uScRs f p eN;t cCHp s ] | 13 ++ .../traffic_via/tests/long rubbish via code2 | 15 ++ tools/traffic_via/tests/rubbish | 6 + tools/traffic_via/tests/short | 5 + tools/{ => traffic_via}/traffic_via.pl | 197 +++++++++++------- 10 files changed, 265 insertions(+), 70 deletions(-) create mode 100755 tools/traffic_via/test_traffic_via_pl create mode 100644 tools/traffic_via/tests/[u c s f p eS;tNc p s ] create mode 100644 tools/traffic_via/tests/[uIcRs f p eN;t cCHp s ] create mode 100644 tools/traffic_via/tests/[uIcRs f p eN;t cCNp s ] create mode 100644 tools/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] create mode 100644 tools/traffic_via/tests/[uScRs f p eN;t cCHp s ] create mode 100644 tools/traffic_via/tests/long rubbish via code2 create mode 100644 tools/traffic_via/tests/rubbish create mode 100644 tools/traffic_via/tests/short rename tools/{ => traffic_via}/traffic_via.pl (62%) diff --git a/tools/traffic_via/test_traffic_via_pl b/tools/traffic_via/test_traffic_via_pl new file mode 100755 index 00000000000..ed4c6e1c715 --- /dev/null +++ b/tools/traffic_via/test_traffic_via_pl @@ -0,0 +1,47 @@ +#! /usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e # exit on error + +TMPDIR=${TMPDIR:-/tmp} +tmpfile=$(mktemp "$TMPDIR/via.XXXXXX") +if [ ! -z "$srcdir"]; then + srcdir=$(cd $srcdir && pwd) +else + srcdir=$(pwd) +fi + +# test command line option +echo "Testing the command line option:" +for f in $srcdir/tests/*; do + name=$(basename "$f") + echo "testing $name" + ./traffic_via.pl -s "$name" > "$tmpfile" 2>&1 || true + diff -u "$tmpfile" "$srcdir/tests/$name" +done + +# test stdin +echo -e "\nTesting stdin:" +for f in $srcdir/tests/\[*; do + name=$(basename "$f") + echo "testing $name" + echo "$name" | ./traffic_via.pl > "$tmpfile" 2>&1 || true + diff -u "$tmpfile" "$srcdir/tests/$name" +done + +rm -f $tmpfile diff --git a/tools/traffic_via/tests/[u c s f p eS;tNc p s ] b/tools/traffic_via/tests/[u c s f p eS;tNc p s ] new file mode 100644 index 00000000000..e662dc0e814 --- /dev/null +++ b/tools/traffic_via/tests/[u c s f p eS;tNc p s ] @@ -0,0 +1,13 @@ +Via header is [u c s f p eS;tNc p s ], Length is 22 +Via Header Details: +Request headers received from client :unknown +Result of Traffic Server cache lookup for URL :no cache lookup +Response information received from origin server :no server connection needed +Result of document write-to-cache: :no cache write performed +Proxy operation result :unknown +Error codes (if any) :server related error +Tunnel info :tunneling due to no forward +Cache Type :unknown +Cache Lookup Result :cache miss or no cache lookup +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :no server connection needed diff --git a/tools/traffic_via/tests/[uIcRs f p eN;t cCHp s ] b/tools/traffic_via/tests/[uIcRs f p eN;t cCHp s ] new file mode 100644 index 00000000000..0755883b90f --- /dev/null +++ b/tools/traffic_via/tests/[uIcRs f p eN;t cCHp s ] @@ -0,0 +1,13 @@ +Via header is [uIcRs f p eN;t cCHp s ], Length is 22 +Via Header Details: +Request headers received from client :IMS +Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") +Response information received from origin server :no server connection needed +Result of document write-to-cache: :no cache write performed +Proxy operation result :unknown +Error codes (if any) :no error +Tunnel info :no tunneling +Cache Type :cache +Cache Lookup Result :cache hit +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :no server connection needed diff --git a/tools/traffic_via/tests/[uIcRs f p eN;t cCNp s ] b/tools/traffic_via/tests/[uIcRs f p eN;t cCNp s ] new file mode 100644 index 00000000000..6ecfc64517a --- /dev/null +++ b/tools/traffic_via/tests/[uIcRs f p eN;t cCNp s ] @@ -0,0 +1,13 @@ +Via header is [uIcRs f p eN;t cCNp s ], Length is 22 +Via Header Details: +Request headers received from client :IMS +Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") +Response information received from origin server :no server connection needed +Result of document write-to-cache: :no cache write performed +Proxy operation result :unknown +Error codes (if any) :no error +Tunnel info :no tunneling +Cache Type :cache +Cache Lookup Result :conditional hit (client sent conditional, doc fresh in cache, returned 304) +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :no server connection needed diff --git a/tools/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] b/tools/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] new file mode 100644 index 00000000000..6ee096a1889 --- /dev/null +++ b/tools/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] @@ -0,0 +1,13 @@ +Via header is [uScMsSf pSeN;t cCMp sS], Length is 22 +Via Header Details: +Request headers received from client :simple request (not conditional) +Result of Traffic Server cache lookup for URL :miss (a cache "MISS") +Response information received from origin server :connection opened successfully +Result of document write-to-cache: :no cache write performed +Proxy operation result :served or connection opened successfully +Error codes (if any) :no error +Tunnel info :no tunneling +Cache Type :cache +Cache Lookup Result :cache miss (url not in cache) +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :connection opened successfully diff --git a/tools/traffic_via/tests/[uScRs f p eN;t cCHp s ] b/tools/traffic_via/tests/[uScRs f p eN;t cCHp s ] new file mode 100644 index 00000000000..ea96fdb54fa --- /dev/null +++ b/tools/traffic_via/tests/[uScRs f p eN;t cCHp s ] @@ -0,0 +1,13 @@ +Via header is [uScRs f p eN;t cCHp s ], Length is 22 +Via Header Details: +Request headers received from client :simple request (not conditional) +Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") +Response information received from origin server :no server connection needed +Result of document write-to-cache: :no cache write performed +Proxy operation result :unknown +Error codes (if any) :no error +Tunnel info :no tunneling +Cache Type :cache +Cache Lookup Result :cache hit +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :no server connection needed diff --git a/tools/traffic_via/tests/long rubbish via code2 b/tools/traffic_via/tests/long rubbish via code2 new file mode 100644 index 00000000000..470c5a1b560 --- /dev/null +++ b/tools/traffic_via/tests/long rubbish via code2 @@ -0,0 +1,15 @@ +Via header is [long rubbish via code2], Length is 22 +traffic_via: Invalid VIA header character: l +traffic_via: Invalid VIA header character: o +traffic_via: Invalid VIA header character: n +traffic_via: Invalid VIA header character: g +traffic_via: Invalid VIA header character: r +traffic_via: Invalid VIA header character: b +traffic_via: Invalid VIA header character: b +traffic_via: Invalid VIA header character: i +traffic_via: Invalid VIA header character: h +traffic_via: Invalid VIA header character: v +traffic_via: Invalid VIA header character: i +traffic_via: Invalid VIA header character: a +traffic_via: Invalid VIA header character: o +traffic_via: Invalid VIA header character: d diff --git a/tools/traffic_via/tests/rubbish b/tools/traffic_via/tests/rubbish new file mode 100644 index 00000000000..dbeeab0e892 --- /dev/null +++ b/tools/traffic_via/tests/rubbish @@ -0,0 +1,6 @@ +Via header is [rubbish], Length is 7 +traffic_via: Invalid VIA header character: r +traffic_via: Invalid VIA header character: b +traffic_via: Invalid VIA header character: b +traffic_via: Invalid VIA header character: i +traffic_via: Invalid VIA header character: h diff --git a/tools/traffic_via/tests/short b/tools/traffic_via/tests/short new file mode 100644 index 00000000000..384a48330cc --- /dev/null +++ b/tools/traffic_via/tests/short @@ -0,0 +1,5 @@ +Via header is [short], Length is 5 +traffic_via: Invalid VIA header character: h +traffic_via: Invalid VIA header character: o +traffic_via: Invalid VIA header character: r +traffic_via: Invalid VIA header character: t diff --git a/tools/traffic_via.pl b/tools/traffic_via/traffic_via.pl similarity index 62% rename from tools/traffic_via.pl rename to tools/traffic_via/traffic_via.pl index e5a0f297cac..a135364f75f 100755 --- a/tools/traffic_via.pl +++ b/tools/traffic_via/traffic_via.pl @@ -40,28 +40,28 @@ #Proxy request header flags and titles my @proxy_header_array = ( { - "Request headers received from client:", { - 'I' => "If Modified Since (IMS)", + "Request headers received from client", { + 'I' => "IMS", 'C' => "cookie", 'E' => "error in request", 'S' => "simple request (not conditional)", 'N' => "no-cache", - ' ' => "unknown?", + ' ' => "unknown", }, }, { - "Result of Traffic Server cache lookup for URL:", { + "Result of Traffic Server cache lookup for URL", { 'A' => "in cache, not acceptable (a cache \"MISS\")", 'H' => "in cache, fresh (a cache \"HIT\")", 'S' => "in cache, stale (a cache \"MISS\")", 'R' => "in cache, fresh Ram hit (a cache \"HIT\")", 'M' => "miss (a cache \"MISS\")", - ' ' => "unknown?", + ' ' => "no cache lookup", }, }, { - "Response information received from origin server:", { + "Response information received from origin server", { 'E' => "error in response", ' ' => "no server connection needed", - 'S' => "served", + 'S' => "connection opened successfully", 'N' => "not-modified", } }, { @@ -72,14 +72,14 @@ ' ' => "no cache write performed", }, }, { - "Proxy operation result:", { + "Proxy operation result", { 'R' => "origin server revalidated", - ' ' => "unknown?", - 'S' => "served", + ' ' => "unknown", + 'S' => "served or connection opened successfully", 'N' => "not-modified", }, }, { - "Error codes (if any):", { + "Error codes (if any)", { 'A' => "authorization failure", 'H' => "header syntax unacceptable", 'C' => "connection to server failed", @@ -90,47 +90,42 @@ 'F' => "request forbidden", }, }, { - "Tunnel info:", { + "Tunnel info", { ' ' => "no tunneling", 'U' => "tunneling because of url (url suggests dynamic content)", 'M' => "tunneling due to a method (e.g. CONNECT)", 'O' => "tunneling because cache is turned off", 'F' => "tunneling due to a header field (such as presence of If-Range header)", + 'N' => "tunneling due to no forward", }, }, { - "Cache type:", { + "Cache Type", { + ' ' => "unknown", 'I' => "icp", - ' ' => "cache miss or no cache lookup", 'C' => "cache", }, }, { - "Cache lookup result:", { - ' ' => "no cache lookup", + "Cache Lookup Result", { + ' ' => "cache miss or no cache lookup", 'S' => "cache hit, but expired", 'U' => "cache hit, but client forces revalidate (e.g. Pragma: no-cache)", 'D' => "cache hit, but method forces revalidated (e.g. ftp, not anonymous)", 'I' => "conditional miss (client sent conditional, fresh in cache, returned 412)", 'H' => "cache hit", 'M' => "cache miss (url not in cache)", - 'C' => "cache hit, but config forces revalidate", + 'C' => "cache miss (url not in cache)", 'N' => "conditional hit (client sent conditional, doc fresh in cache, returned 304)", }, }, { - "ICP status:", { - ' ' => "no icp", - 'S' => "connection opened successfully", - 'F' => "connection open failed", - }, - }, { - "Parent proxy connection status:", { - ' ' => "no parent proxy", + "Parent proxy connection status", { + ' ' => "no parent proxy or unknown", 'S' => "connection opened successfully", 'F' => "connection open failed", }, }, { - "Origin server connection status:", { - ' ' => "no server connection", + "Origin server connection status", { + ' ' => "no server connection needed", 'S' => "connection opened successfully", 'F' => "connection open failed", }, @@ -138,7 +133,7 @@ ); ##Print script usage -sub usage +sub usage() { print "\nPass Via Header with -s option \n"; print "Usage: traffic_via [-s viaheader]"; @@ -149,41 +144,10 @@ sub usage exit; } -if (@ARGV == 0) { - #if passed through standard input - my @userinput = ; - my $via_string; - - for my $element (@userinput) { - #Pattern matching for Via - if ($element =~ /Via:(.*)\[(.*)\]/) { - #Search and grep via header - $via_string = $2; - chomp($via_string); - print "Via Header is [$via_string]"; - decode_via_header($via_string); - } - } -} else { - usage() - if ( - !GetOptions( - 's=s' => \$via_header, - 'help|?' => \$help - ) - or defined $help - ); - - if (defined $via_header) { - #if passed through commandline dashed argument - print "Via Header is [$via_header]"; - decode_via_header($via_header); - } -} #Subroutine to decode via header -sub decode_via_header +sub decode_via_header($) { my ($header) = @_; my $hdrLength; @@ -194,45 +158,98 @@ sub decode_via_header #Get via header length $hdrLength = length($header); - # Valid Via header length is 24 or 6. - # When Via header length is 24, it will have both proxy request header result and operational results. - if ($hdrLength == 24) { + # Valid Via header length is 22 or 6. + # When Via header length is 22, it will have both proxy request header result and operational results. + if ($hdrLength == 22) { #Split via header: proxy result and operational result $newHeader = join('', split(':', $header)); } elsif ($hdrLength == 6) { $newHeader = $header; } elsif ($hdrLength == 5) { # When Via header length is 5, it might be missing last field. Fill it and decode header. - my $newHeader = "$header" . " "; + $newHeader = $header . " "; } else { # Invalid header size, come out. - print "\nInvalid VIA header. VIA header length should be 6 or 24 characters\n"; + print "\nInvalid VIA header. VIA header length should be 6 or 22 characters\n"; return; } + convert_header_to_array($newHeader); } } -sub convert_header_to_array +sub convert_header_to_array($) { my ($viaHeader) = @_; my @ResultArray; #Convert string header into character array while ($viaHeader =~ /(.)/g) { #Only capital letters indicate flags - if ($1 !~ m/[a-z]+/) { + if ($1 !~ m/[a-z;]+/) { push(@ResultArray, $1); } } - print "\nVia Header details: \n"; + print "Via Header Details:\n"; for (my $arrayIndex = 0; $arrayIndex < scalar(@ResultArray); $arrayIndex++) { get_via_header_flags(\@proxy_header_array, $arrayIndex, $ResultArray[$arrayIndex]); } } +my %valid_keys = ('main' => { + 'u' => 1, + 'c' => 1, + 's' => 1, + 'f' => 1, + 'p' => 1, + 'e' => 1, + }, + 'detail' => { + 't' => 1, + 'c' => 1, + 'p' => 1, + 's' => 1, + } +); + +sub valid_char ($$) +{ + my($char, $hash) = @_; + + return exists $hash->{$char} +} + +sub validate_keys($) +{ + my($viaHeader) = @_; + my($main, $detail) = split(';', $viaHeader); + my $running_main = 1; + my $return_value_valid = 1; + + foreach my $group ($main, $detail) { + next if !defined $group; + + while ($group =~ /([a-z])/g) { + my $char = $1; + + my $valid = 0; + if ($running_main) { + $valid = valid_char($char, $valid_keys{main}); + } else { + $valid = valid_char($char, $valid_keys{detail}); + } + if (! $valid) { + print "traffic_via: Invalid VIA header character: $char\n"; + $return_value_valid = 0; + } + } + $running_main = 0; + } + return $return_value_valid; +} + #Get values from header arrays -sub get_via_header_flags +sub get_via_header_flags($$$) { my ($arrayName, $inputIndex, $flag) = @_; @@ -253,9 +270,49 @@ sub get_via_header_flags foreach my $key (@keys) { if ($key =~ /$flag/) { #print $flags{$key}; - printf("%s", $flags{$key}); + printf(":%s", $flags{$key}); print "\n"; } } } } + +# main +{ + if (@ARGV == 0) { + #if passed through standard input + my @userinput = ; + + for my $element (@userinput) { + #Pattern matching for Via + if ($element =~ /Via:\s+\[(.+)\]/i || $element =~ /\[(.+)\]/ ) { + #Search and grep via header + my $via_string = $1; + chomp($via_string); + print "Via header is [$via_string], Length is ", length($via_string), "\n"; + last unless validate_keys($via_string); + decode_via_header($via_string); + } + } + } else { + usage() + if ( + !GetOptions( + 's=s' => \$via_header, + 'help|?' => \$help + ) + or defined $help + ); + + if (defined $via_header) { + if ($via_header =~ /Via:\s+\[(.+)\]/i || $via_header =~ /\[(.+)\]/ || $via_header =~ /(.+)/) { + #if passed through commandline dashed argument + my $via_string = $1; + print "Via header is [$via_string], Length is ", length($via_string), "\n"; + last unless validate_keys($via_string); + decode_via_header($via_string); + } + } + + } +} \ No newline at end of file From 671e63994bffc2da6c0b2a89612707f9baf03c1d Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Wed, 10 Apr 2019 17:58:35 +0000 Subject: [PATCH 437/526] Add more information to diags.log error message for block stitching errors. --- plugins/experimental/slice/HttpHeader.cc | 36 +- plugins/experimental/slice/HttpHeader.h | 12 +- plugins/experimental/slice/Range.cc | 8 +- plugins/experimental/slice/client.cc | 6 +- plugins/experimental/slice/response.cc | 9 +- plugins/experimental/slice/server.cc | 139 ++++++-- .../slice/gold_error/crr.stderr.gold | 1 + .../slice/gold_error/crr.stdout.gold | 9 + .../slice/gold_error/etag.stderr.gold | 1 + .../slice/gold_error/etag.stdout.gold | 9 + .../slice/gold_error/lm.stderr.gold | 1 + .../slice/gold_error/lm.stdout.gold | 9 + .../slice/gold_error/non206.stderr.gold | 1 + .../slice/gold_error/non206.stdout.gold | 10 + .../pluginTest/slice/slice_error.test.py | 326 ++++++++++++++++++ 15 files changed, 532 insertions(+), 45 deletions(-) create mode 100644 tests/gold_tests/pluginTest/slice/gold_error/crr.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold_error/etag.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold_error/lm.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold_error/non206.stderr.gold create mode 100644 tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold create mode 100644 tests/gold_tests/pluginTest/slice/slice_error.test.py diff --git a/plugins/experimental/slice/HttpHeader.cc b/plugins/experimental/slice/HttpHeader.cc index 0f1a78f001d..bcf649b761a 100644 --- a/plugins/experimental/slice/HttpHeader.cc +++ b/plugins/experimental/slice/HttpHeader.cc @@ -53,6 +53,24 @@ HttpHeader::setStatus(TSHttpStatus const newstatus) return TS_SUCCESS == TSHttpHdrStatusSet(m_buffer, m_lochdr, newstatus); } +char * +HttpHeader ::urlString(int *const urllen) const +{ + char *urlstr = nullptr; + TSAssert(nullptr != urllen); + + TSMLoc locurl = nullptr; + TSReturnCode const rcode = TSHttpHdrUrlGet(m_buffer, m_lochdr, &locurl); + if (TS_SUCCESS == rcode && nullptr != locurl) { + urlstr = TSUrlStringGet(m_buffer, locurl, urllen); + TSHandleMLocRelease(m_buffer, m_lochdr, locurl); + } else { + *urllen = 0; + } + + return urlstr; +} + bool HttpHeader::setUrl(TSMBuffer const bufurl, TSMLoc const locurl) { @@ -102,6 +120,10 @@ HttpHeader::getCharPtr(CharPtrGetFunc func, int *const len) const } } + if (nullptr == res && nullptr != len) { + *len = 0; + } + return res; } @@ -144,6 +166,7 @@ bool HttpHeader::valueForKey(char const *const keystr, int const keylen, char *const valstr, int *const vallen, int const index) const { if (!isValid()) { + *vallen = 0; return false; } @@ -171,9 +194,6 @@ HttpHeader::valueForKey(char const *const keystr, int const keylen, char *const *vallen = 0; } - if (!status) { - } - return status; } @@ -224,16 +244,12 @@ HttpHeader::toString() const case TS_HTTP_TYPE_REQUEST: { res.append(method()); - TSMLoc locurl = nullptr; - TSReturnCode const rcode = TSHttpHdrUrlGet(m_buffer, m_lochdr, &locurl); - if (TS_SUCCESS == rcode && nullptr != locurl) { - int urllen = 0; - char *const urlstr = TSUrlStringGet(m_buffer, locurl, &urllen); + int urllen = 0; + char *const urlstr = urlString(&urllen); + if (nullptr != urlstr) { res.append(" "); res.append(urlstr, urllen); TSfree(urlstr); - - TSHandleMLocRelease(m_buffer, m_lochdr, locurl); } else { res.append(" UnknownURL"); } diff --git a/plugins/experimental/slice/HttpHeader.h b/plugins/experimental/slice/HttpHeader.h index 35727612335..bfd9c4d6f4e 100644 --- a/plugins/experimental/slice/HttpHeader.h +++ b/plugins/experimental/slice/HttpHeader.h @@ -71,6 +71,16 @@ struct HttpHeader { return getCharPtr(TSHttpHdrMethodGet, len); } + // request method version + int + version() const + { + return TSHttpHdrVersionGet(m_buffer, m_lochdr); + } + + // Returns string representation of the url. Caller gets ownership! + char *urlString(int *const urllen) const; + // host char const * hostname(int *const len) const @@ -95,7 +105,7 @@ struct HttpHeader { bool valueForKey(char const *const keystr, int const keylen, char *const valstr, // <-- return string value int *const vallen, // <-- pass in capacity, returns len of string - int const index = -1 // sets all values + int const index = -1 // retrieves all values ) const; /** diff --git a/plugins/experimental/slice/Range.cc b/plugins/experimental/slice/Range.cc index 1a7e9685d46..c9a9d4dd44a 100644 --- a/plugins/experimental/slice/Range.cc +++ b/plugins/experimental/slice/Range.cc @@ -57,10 +57,10 @@ Range::fromStringClosed(char const *const rangestr) } // rip out any whitespace - static int const RLEN = 1024; - char rangebuf[RLEN]; - char *pbuf = rangebuf; - while ('\0' != *pstr && (pbuf - rangebuf) < RLEN) { + char rangebuf[1024]; + int const rangelen = sizeof(rangebuf); + char *pbuf = rangebuf; + while ('\0' != *pstr && (pbuf - rangebuf) < rangelen) { if (!isblank(*pstr)) { *pbuf++ = *pstr; } diff --git a/plugins/experimental/slice/client.cc b/plugins/experimental/slice/client.cc index d52887e5278..75bbe8e434a 100644 --- a/plugins/experimental/slice/client.cc +++ b/plugins/experimental/slice/client.cc @@ -38,7 +38,7 @@ requestBlock(TSCont contp, Data *const data) Range blockbe(blockbeg, blockbeg + data->m_blockbytes_config); char rangestr[1024]; - int rangelen = 1023; + int rangelen = sizeof(rangestr); bool const rpstat = blockbe.toStringClosed(rangestr, &rangelen); TSAssert(rpstat); @@ -114,7 +114,7 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) Range rangebe; char rangestr[1024]; - int rangelen = 1024; + int rangelen = sizeof(rangestr); bool const hasRange = header.valueForKey(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, &rangelen, 0); // <-- first range only if (hasRange) { @@ -135,7 +135,7 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) } } else { DEBUG_LOG("Full content request"); - static char const *const valstr = "full content request"; + static char const *const valstr = "-"; static size_t const vallen = strlen(valstr); header.setKeyVal(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), valstr, vallen); data->m_statustype = TS_HTTP_STATUS_OK; diff --git a/plugins/experimental/slice/response.cc b/plugins/experimental/slice/response.cc index 4b9f101bde1..410b1507dfe 100644 --- a/plugins/experimental/slice/response.cc +++ b/plugins/experimental/slice/response.cc @@ -68,9 +68,8 @@ string502() bodystr.append("\n"); bodystr.append("\n"); - static int const CLEN = 1024; - char clenstr[CLEN]; - int const clen = snprintf(clenstr, CLEN, "%lu", bodystr.size()); + char clenstr[1024]; + int const clen = snprintf(clenstr, sizeof(clenstr), "%lu", bodystr.size()); msg.append("HTTP/1.1 502 Bad Gateway\r\n"); msg.append("Content-Length: "); @@ -96,9 +95,7 @@ form416HeaderAndBody(HttpHeader &header, int64_t const contentlen, std::string c header.setReason(reason, strlen(reason)); char bufstr[256]; - int buflen = snprintf - // (bufstr, 255, "%" PRId64, bodystr.size()); - (bufstr, 255, "%lu", bodystr.size()); + int buflen = snprintf(bufstr, sizeof(bufstr), "%lu", bodystr.size()); header.setKeyVal(TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, bufstr, buflen); static char const *const ctypestr = "text/html"; diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc index f2f4dee5af3..9371469e367 100644 --- a/plugins/experimental/slice/server.cc +++ b/plugins/experimental/slice/server.cc @@ -22,6 +22,8 @@ #include "response.h" #include "transfer.h" +#include "ts/experimental.h" + #include namespace @@ -42,9 +44,8 @@ contentRangeFrom(HttpHeader const &header) /* Pull content length off the response header and manipulate it into a client response header */ - static int const RLEN = 1024; - char rangestr[RLEN]; - int rangelen = RLEN - 1; + char rangestr[1024]; + int rangelen = sizeof(rangestr); // look for expected Content-Range field bool const hasContentRange(header.valueForKey(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, rangestr, &rangelen)); @@ -63,11 +64,13 @@ handleFirstServerHeader(Data *const data, TSCont const contp) { HttpHeader header(data->m_resp_hdrmgr.m_buffer, data->m_resp_hdrmgr.m_lochdr); + // DEBUG_LOG("First header\n%s", header.toString().c_str()); + data->m_dnstream.setupVioWrite(contp); // only process a 206, everything else gets a pass through if (TS_HTTP_STATUS_PARTIAL_CONTENT != header.status()) { - DEBUG_LOG("Non 206 response from parent: %d", header.status()); + DEBUG_LOG("Initial reponse other than 206: %d", header.status()); data->m_bail = true; TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); @@ -138,7 +141,7 @@ handleFirstServerHeader(Data *const data, TSCont const contp) respcr.m_length = data->m_contentlen; char rangestr[1024]; - int rangelen = 1023; + int rangelen = sizeof(rangestr); bool const crstat = respcr.toStringClosed(rangestr, &rangelen); // corner case, return 500 ?? @@ -163,7 +166,7 @@ handleFirstServerHeader(Data *const data, TSCont const contp) } char bufstr[1024]; - int const buflen = snprintf(bufstr, 1023, "%" PRId64, bodybytes); + int const buflen = snprintf(bufstr, sizeof(bufstr), "%" PRId64, bodybytes); header.setKeyVal(TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, bufstr, buflen); // add the response header length to the total bytes to send @@ -180,29 +183,124 @@ handleFirstServerHeader(Data *const data, TSCont const contp) return true; } +void +logSliceError(char const *const message, Data const *const data, HttpHeader const &header_resp) +{ + HttpHeader const header_req(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); + + TSHRTime const timenowus = TShrtime(); + int64_t const msecs = timenowus / 1000000; + int64_t const secs = msecs / 1000; + int64_t const ms = msecs % 1000; + + // Gather information on the request, must delete urlstr + int urllen = 0; + char *const urlstr = header_req.urlString(&urllen); + + char urlpstr[16384]; + size_t urlplen = sizeof(urlpstr); + TSStringPercentEncode(urlstr, urllen, urlpstr, urlplen, &urlplen, nullptr); + + if (nullptr != urlstr) { + TSfree(urlstr); + } + + // uas + char uasstr[8192]; + int uaslen = sizeof(uasstr); + header_req.valueForKey(TS_MIME_FIELD_USER_AGENT, TS_MIME_LEN_USER_AGENT, uasstr, &uaslen); + + // raw range request + char rangestr[1024]; + int rangelen = sizeof(rangestr); + header_req.valueForKey(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), rangestr, &rangelen); + + // Normalized range request + ContentRange const crange(data->m_req_range.m_beg, data->m_req_range.m_end, data->m_contentlen); + char normstr[1024]; + int normlen = sizeof(normstr); + crange.toStringClosed(normstr, &normlen); + + // block range request + int64_t const blockbeg = data->m_blocknum * data->m_blockbytes_config; + int64_t const blockend = std::min(blockbeg + data->m_blockbytes_config, data->m_contentlen); + + // Block response data + TSHttpStatus const statusgot = header_resp.status(); + + // content range + char crstr[1024]; + int crlen = sizeof(crstr); + header_resp.valueForKey(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, crstr, &crlen); + + // etag + char etagstr[1024]; + int etaglen = sizeof(etagstr); + header_resp.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, etagstr, &etaglen); + + // last modified + char lmstr[1024]; + int lmlen = sizeof(lmstr); + header_resp.valueForKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED, lmstr, &lmlen); + + // cc + char ccstr[2048]; + int cclen = sizeof(ccstr); + header_resp.valueForKey(TS_MIME_FIELD_CACHE_CONTROL, TS_MIME_LEN_CACHE_CONTROL, ccstr, &cclen); + + // via tag + char viastr[8192]; + int vialen = sizeof(viastr); + header_resp.valueForKey(TS_MIME_FIELD_VIA, TS_MIME_LEN_VIA, viastr, &vialen); + + char etagexpstr[1024]; + size_t etagexplen = sizeof(etagexpstr); + TSStringPercentEncode(data->m_etag, data->m_etaglen, etagexpstr, etagexplen, &etagexplen, nullptr); + + char etaggotstr[1024]; + size_t etaggotlen = sizeof(etaggotstr); + TSStringPercentEncode(etagstr, etaglen, etaggotstr, etaggotlen, &etaggotlen, nullptr); + + TSError("[%s] %" PRId64 ".%" PRId64 " reason=\"%s\"" + " uri=\"%.*s\"" + " uas=\"%.*s\"" + " req_range=\"%.*s\"" + " norm_range=\"%.*s\"" + + " etag_exp=\"%.*s\"" + " lm_exp=\"%.*s\"" + + " blk_range=\"%" PRId64 "-%" PRId64 "\"" + + " status_got=\"%d\"" + " cr_got=\"%.*s\"" + " etag_got=\"%.*s\"" + " lm_got=\"%.*s\"" + " cc=\"%.*s\"" + " via=\"%.*s\"", + PLUGIN_NAME, secs, ms, message, (int)urlplen, urlpstr, uaslen, uasstr, rangelen, rangestr, normlen, normstr, + (int)etagexplen, etagexpstr, data->m_lastmodifiedlen, data->m_lastmodified, blockbeg, blockend - 1, statusgot, crlen, + crstr, (int)etaggotlen, etaggotstr, lmlen, lmstr, cclen, ccstr, vialen, viastr); +} + bool handleNextServerHeader(Data *const data, TSCont const contp) { + // block response header HttpHeader header(data->m_resp_hdrmgr.m_buffer, data->m_resp_hdrmgr.m_lochdr); + // DEBUG_LOG("Next Header:\n%s", header.toString().c_str()); // only process a 206, everything else just aborts if (TS_HTTP_STATUS_PARTIAL_CONTENT != header.status()) { - ERROR_LOG("Non 206 internal block response from parent: %d", header.status()); + logSliceError("Non 206 internal block response", data, header); data->m_bail = true; return false; } // can't parse the content range header, abort -- might be too strict ContentRange const blockcr = contentRangeFrom(header); - if (!blockcr.isValid()) { - ERROR_LOG("Unable to parse internal block Content-Range header"); - data->m_bail = true; - return false; - } - - // make sure the block comes from the same asset as the first block - if (data->m_contentlen != blockcr.m_length) { - ERROR_LOG("Mismatch in slice block Content-Range Len %" PRId64 " and %" PRId64, data->m_contentlen, blockcr.m_length); + if (!blockcr.isValid() || blockcr.m_length != data->m_contentlen) { + logSliceError("Mismatch/Bad block Content-Range", data, header); data->m_bail = true; return false; } @@ -211,23 +309,22 @@ handleNextServerHeader(Data *const data, TSCont const contp) // prefer the etag but use Last-Modified if we must. char etag[8192]; - int etaglen = sizeof(etag) - 1; + int etaglen = sizeof(etag); header.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, etag, &etaglen); if (0 < data->m_etaglen || 0 < etaglen) { same = data->m_etaglen == etaglen && 0 == strncmp(etag, data->m_etag, etaglen); if (!same) { - ERROR_LOG("Mismatch in slice block ETAG '%.*s' and '%.*s'", data->m_etaglen, data->m_etag, etaglen, etag); + logSliceError("Mismatch block Etag", data, header); } } else { char lastmodified[8192]; - int lastmodifiedlen = sizeof(lastmodified) - 1; + int lastmodifiedlen = sizeof(lastmodified); header.valueForKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED, lastmodified, &lastmodifiedlen); if (0 < data->m_lastmodifiedlen || 0 != lastmodifiedlen) { same = data->m_lastmodifiedlen == lastmodifiedlen && 0 == strncmp(lastmodified, data->m_lastmodified, lastmodifiedlen); if (!same) { - ERROR_LOG("Mismatch in slice block Last-Modified '%.*s' and '%.*s'", data->m_lastmodifiedlen, data->m_lastmodified, - lastmodifiedlen, lastmodified); + logSliceError("Mismatch block Last-Modified", data, header); } } } diff --git a/tests/gold_tests/pluginTest/slice/gold_error/crr.stderr.gold b/tests/gold_tests/pluginTest/slice/gold_error/crr.stderr.gold new file mode 100644 index 00000000000..fad05754105 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/crr.stderr.gold @@ -0,0 +1 @@ +the quick`` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold new file mode 100644 index 00000000000..955a42451c0 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/etag.stderr.gold b/tests/gold_tests/pluginTest/slice/gold_error/etag.stderr.gold new file mode 100644 index 00000000000..fad05754105 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/etag.stderr.gold @@ -0,0 +1 @@ +the quick`` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold new file mode 100644 index 00000000000..955a42451c0 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/lm.stderr.gold b/tests/gold_tests/pluginTest/slice/gold_error/lm.stderr.gold new file mode 100644 index 00000000000..fad05754105 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/lm.stderr.gold @@ -0,0 +1 @@ +the quick`` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold new file mode 100644 index 00000000000..2d83f7e9fd3 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +HTTP/1.1 200 OK +Last-Modified: `` +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/non206.stderr.gold b/tests/gold_tests/pluginTest/slice/gold_error/non206.stderr.gold new file mode 100644 index 00000000000..fad05754105 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/non206.stderr.gold @@ -0,0 +1 @@ +the quick`` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold new file mode 100644 index 00000000000..86ee5ddcb2a --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold @@ -0,0 +1,10 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Last-Modified: `` +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/slice_error.test.py b/tests/gold_tests/pluginTest/slice/slice_error.test.py new file mode 100644 index 00000000000..f6fd4d50627 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice_error.test.py @@ -0,0 +1,326 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Slice plugin error.log test +''' + +## Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with slice plugin +# Request content through the slice plugin + +Test.SkipUnless( + Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), + Condition.PluginExists('slice.so'), +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server", lookup_key="{%Range}{PATH}") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager") + +body = "the quick brown fox" # len 19 + +# default root +request_header_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes=0-\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_chk = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, +} + +server.addResponse("sessionlog.json", request_header_chk, response_header_chk) + +blockbytes = 9 + +range0 = "{}-{}".format(0, blockbytes - 1) +range1 = "{}-{}".format(blockbytes, (2 * blockbytes) - 1) + +body0 = body[0:blockbytes] +body1 = body[blockbytes:2 * blockbytes] + +# Mismatch etag + +request_header_etag0 = {"headers": + "GET /etag HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_etag0 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag0"\r\n' + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, +} + +server.addResponse("sessionlog.json", request_header_etag0, response_header_etag0) + +request_header_etag1 = {"headers": + "GET /etag HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_etag1 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag1"\r\n' + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, +} + +server.addResponse("sessionlog.json", request_header_etag1, response_header_etag1) + +# mismatch Last-Modified + +request_header_lm0 = {"headers": + "GET /lastmodified HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_lm0 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Last-Modified: Tue, 08 May 2018 15:49:41 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, +} + +server.addResponse("sessionlog.json", request_header_lm0, response_header_lm0) + +request_header_lm1 = {"headers": + "GET /lastmodified HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_lm1 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Last-Modified: Tue, 08 Apr 2019 18:00:00 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, +} + +server.addResponse("sessionlog.json", request_header_lm1, response_header_lm1) + +# non 206 slice block + +request_header_n206_0 = {"headers": + "GET /non206 HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_n206_0 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag"\r\n' + + "Last-Modified: Tue, 08 May 2018 15:49:41 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, +} + +server.addResponse("sessionlog.json", request_header_n206_0, response_header_n206_0) + +# mismatch content-range + +request_header_crr0 = {"headers": + "GET /crr HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_crr0 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Etag: crr\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, +} + +server.addResponse("sessionlog.json", request_header_crr0, response_header_crr0) + +request_header_crr1 = {"headers": + "GET /crr HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_crr1 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Etag: crr\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body) - 1) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, +} + +server.addResponse("sessionlog.json", request_header_crr1, response_header_crr1) + +ts.Setup.CopyAs('curlsort.sh', Test.RunDirectory) +curl_and_args = 'sh curlsort.sh -H "Host: www.example.com"' + +# set up whole asset fetch into cache +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=slice.so @pparam=bytesover:{}'.format(blockbytes) +) + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'slice', + 'proxy.config.http.cache.http': 0, + 'proxy.config.http.wait_for_cache': 0, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.insert_request_via_str': 0, + 'proxy.config.http.insert_response_via_str': 3, + 'proxy.config.http.server_ports': '{}'.format(ts.Variables.port), +}) + +# Override builtin error check as these cases will fail +# taken from the slice plug code +ts.Disk.diags_log.Content = Testers.ContainsExpression('reason="Mismatch block Etag"', "Mismatch block etag") +ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Mismatch block Last-Modified"', "Mismatch block Last-Modified") +ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Non 206 internal block response"', "Non 206 internal block response") +ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Mismatch/Bad block Content-Range"', "Mismatch/Bad block Content-Range") + +# 0 Test - Etag mismatch test +tr = Test.AddTestRun("Etag test") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold_error/etag.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold_error/etag.stderr.gold" +tr.StillRunningAfter = ts + +# 1 Check - diags.log message +tr = Test.AddTestRun("Etag error check") +tr.Processes.Default.Command = "grep 'Mismatch block Etag' {}".format(ts.Disk.diags_log.Name) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +# 2 Test - Last Modified mismatch test +tr = Test.AddTestRun("Last-Modified test") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/lastmodified'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold_error/lm.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold_error/lm.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Check - diags.log message +tr = Test.AddTestRun("Last-Modified error check") +tr.Processes.Default.Command = "grep 'Mismatch block Last-Modified' {}".format(ts.Disk.diags_log.Name) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +# 4 Test - Non 206 mismatch test +tr = Test.AddTestRun("Non 206 test") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/non206'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold_error/non206.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold_error/non206.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Check - diags.log message +tr = Test.AddTestRun("Non 206 error check") +tr.Processes.Default.Command = "grep 'Non 206 internal block response' {}".format(ts.Disk.diags_log.Name) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +# 4 Test - Block content-range +tr = Test.AddTestRun("Content-Range test") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/crr'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold_error/crr.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold_error/crr.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Check - diags.log message +tr = Test.AddTestRun("Content-Range error check") +tr.Processes.Default.Command = "grep 'Mismatch/Bad block Content-Range' {}".format(ts.Disk.diags_log.Name) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts From 1ff9e7515aeb87887d5c138e663af8dc1df7a950 Mon Sep 17 00:00:00 2001 From: Nozomi Inanami <42267827+nozomi1773@users.noreply.github.com> Date: Sun, 21 Apr 2019 21:09:26 +0900 Subject: [PATCH 438/526] fix proxy.config.cache.ram_cache.algorithm desc --- doc/admin-guide/files/records.config.en.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 6eb2fc06743..7a729b98968 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -2203,10 +2203,10 @@ RAM Cache .. ts:cv:: CONFIG proxy.config.cache.ram_cache.algorithm INT 1 - Two distinct RAM caches are supported, the default (0) being the **CLFUS** - (*Clocked Least Frequently Used by Size*). As an alternative, a simpler - **LRU** (*Least Recently Used*) cache is also available, by changing this - configuration to 1. + Two distinct RAM caches are supported, the default (1) being the simpler + **LRU** (*Least Recently Used*) cache. As an alternative, the **CLFUS** + (*Clocked Least Frequently Used by Size*) is also available, by changing this + configuration to 0. .. ts:cv:: CONFIG proxy.config.cache.ram_cache.use_seen_filter INT 1 From ed9b5659bec77cdce8320693db2ea65b33307a74 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Fri, 19 Apr 2019 17:08:20 +0900 Subject: [PATCH 439/526] Fix a build error in xdebug on macos --- plugins/xdebug/xdebug_transforms.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/xdebug/xdebug_transforms.cc b/plugins/xdebug/xdebug_transforms.cc index 4ceeea1a6c0..1bec6e80c3e 100644 --- a/plugins/xdebug/xdebug_transforms.cc +++ b/plugins/xdebug/xdebug_transforms.cc @@ -132,14 +132,14 @@ body_transform(TSCont contp, TSEvent event, void *edata) } int64_t towrite = TSVIONTodoGet(src_vio); - TSDebug("xdebug_transform", "body_transform(): %li bytes of body is expected", towrite); + TSDebug("xdebug_transform", "body_transform(): %" PRId64 " bytes of body is expected", towrite); int64_t avail = TSIOBufferReaderAvail(TSVIOReaderGet(src_vio)); towrite = towrite > avail ? avail : towrite; if (towrite > 0) { TSIOBufferCopy(TSVIOBufferGet(data->output_vio), TSVIOReaderGet(src_vio), towrite, 0); TSIOBufferReaderConsume(TSVIOReaderGet(src_vio), towrite); TSVIONDoneSet(src_vio, TSVIONDoneGet(src_vio) + towrite); - TSDebug("xdebug_transform", "body_transform(): writing %li bytes of body", towrite); + TSDebug("xdebug_transform", "body_transform(): writing %" PRId64 " bytes of body", towrite); } if (TSVIONTodoGet(src_vio) > 0) { From 1e9e4ed3be52dc8ef2433baba6816df4eb032055 Mon Sep 17 00:00:00 2001 From: Vijay Mamidi Date: Tue, 23 Apr 2019 10:12:17 +0800 Subject: [PATCH 440/526] Throttling results in tight loop if there are no new connections --- iocore/net/UnixNetAccept.cc | 42 +++++++------------------------------ 1 file changed, 8 insertions(+), 34 deletions(-) diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index 2a2c2f7f5fc..a936c5ef8c0 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -43,29 +43,6 @@ safe_delay(int msec) socketManager.poll(nullptr, 0, msec); } -static int -drain_throttled_accepts(NetAccept *na) -{ - struct pollfd afd; - Connection con[THROTTLE_AT_ONCE]; - - afd.fd = na->server.fd; - afd.events = POLLIN; - - // Try to close at most THROTTLE_AT_ONCE accept requests - // Stop if there is nothing waiting - int n = 0; - for (; n < THROTTLE_AT_ONCE && socketManager.poll(&afd, 1, 0) > 0; n++) { - int res = 0; - if ((res = na->server.accept(&con[n])) < 0) { - return res; - } - con[n].close(); - } - // Return the number of accept cases we closed - return n; -} - // // General case network connection accept code // @@ -307,18 +284,7 @@ NetAccept::do_blocking_accept(EThread *t) // do-while for accepting all the connections // added by YTS Team, yamsat do { - // Throttle accepts - while (!opt.backdoor && check_net_throttle(ACCEPT)) { - check_throttle_warning(ACCEPT); - int num_throttled = drain_throttled_accepts(this); - if (num_throttled < 0) { - goto Lerror; - } - NET_SUM_DYN_STAT(net_connections_throttled_in_stat, num_throttled); - } - if ((res = server.accept(&con)) < 0) { - Lerror: int seriousness = accept_error_seriousness(res); if (seriousness >= 0) { // not so bad if (!seriousness) { // bad enough to warn about @@ -334,6 +300,14 @@ NetAccept::do_blocking_accept(EThread *t) } return -1; } + // check for throttle + if (!opt.backdoor && check_net_throttle(ACCEPT)) { + check_throttle_warning(ACCEPT); + // close the connection as we are in throttle state + con.close(); + NET_SUM_DYN_STAT(net_connections_throttled_in_stat, 1); + continue; + } if (TSSystemState::is_event_system_shut_down()) { return -1; From c438f946077dc25e60f42c09bec098214f75cf18 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 23 Apr 2019 16:37:00 +0800 Subject: [PATCH 441/526] cppcheck: fix issue found in BufferWriterFormat.cc Variable '_name' is assigned in constructor body. Consider performing initialization in initialization list. --- src/tscore/BufferWriterFormat.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tscore/BufferWriterFormat.cc b/src/tscore/BufferWriterFormat.cc index 70a5e016281..05d821a07d3 100644 --- a/src/tscore/BufferWriterFormat.cc +++ b/src/tscore/BufferWriterFormat.cc @@ -99,12 +99,11 @@ BWFSpec::Property::Property() } /// Parse a format specification. -BWFSpec::BWFSpec(TextView fmt) +BWFSpec::BWFSpec(TextView fmt) : _name(fmt.take_prefix_at(':')) { TextView num; // temporary for number parsing. intmax_t n; - _name = fmt.take_prefix_at(':'); // if it's parsable as a number, treat it as an index. n = tv_to_positive_decimal(_name, &num); if (num.size()) { From 3c3d9941ad65f2861ada076a76b40e9e6835d7bd Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 23 Apr 2019 16:04:38 +0800 Subject: [PATCH 442/526] cppcheck: Use initialization list instead of assigning in constructor body > [Http2ClientSession.h:77]: (performance) Variable 'hdr' is assigned in constructor body. Consider performing initialization in initialization list. > [Http2ClientSession.h:83]: (performance) Variable 'hdr' is assigned in constructor body. Consider performing initialization in initialization list. --- proxy/http2/Http2ClientSession.h | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 23b0f53e820..b6d1faed0f3 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -72,17 +72,8 @@ struct Http2UpgradeContext { class Http2Frame { public: - Http2Frame(const Http2FrameHeader &h, IOBufferReader *r) - { - this->hdr = h; - this->ioreader = r; - } - - Http2Frame(Http2FrameType type, Http2StreamId streamid, uint8_t flags) - { - this->hdr = {0, (uint8_t)type, flags, streamid}; - this->ioreader = nullptr; - } + Http2Frame(const Http2FrameHeader &h, IOBufferReader *r) : hdr(h), ioreader(r) {} + Http2Frame(Http2FrameType type, Http2StreamId streamid, uint8_t flags) : hdr({0, (uint8_t)type, flags, streamid}) {} IOBufferReader * reader() const @@ -155,7 +146,7 @@ class Http2Frame private: Http2FrameHeader hdr; // frame header Ptr ioblock; // frame payload - IOBufferReader *ioreader; + IOBufferReader *ioreader = nullptr; }; class Http2ClientSession : public ProxyClientSession From d124a4978438a7559588e1a82d04df7b8766a926 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Tue, 23 Apr 2019 15:15:02 +0800 Subject: [PATCH 443/526] cppcheck: Fixed various issues with SSL files Cleaned up checking pointer for null twice Removed copy constructors on class Fixed scoped variables with the same name --- iocore/net/SSLClientUtils.cc | 7 ++++--- iocore/net/SSLSessionCache.h | 3 +++ iocore/net/SSLSessionTicket.cc | 7 +++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/iocore/net/SSLClientUtils.cc b/iocore/net/SSLClientUtils.cc index 2a0023f1f3a..41aa534c410 100644 --- a/iocore/net/SSLClientUtils.cc +++ b/iocore/net/SSLClientUtils.cc @@ -50,11 +50,12 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) SSLNetVConnection *netvc = SSLNetVCAccess(ssl); // No enforcing, go away - if (netvc && netvc->options.verifyServerPolicy == YamlSNIConfig::Policy::DISABLED) { - return true; // Tell them that all is well - } else if (!netvc) { // No netvc, very bad. Go away. Things are not good. + if (netvc == nullptr) { + // No netvc, very bad. Go away. Things are not good. Warning("Netvc gone by in verify_callback"); return false; + } else if (netvc->options.verifyServerPolicy == YamlSNIConfig::Policy::DISABLED) { + return true; // Tell them that all is well } depth = X509_STORE_CTX_get_error_depth(ctx); diff --git a/iocore/net/SSLSessionCache.h b/iocore/net/SSLSessionCache.h index 5a5d50fc058..a32809c3c6a 100644 --- a/iocore/net/SSLSessionCache.h +++ b/iocore/net/SSLSessionCache.h @@ -153,6 +153,9 @@ class SSLSessionCache SSLSessionCache(); ~SSLSessionCache(); + SSLSessionCache(const SSLSessionCache &) = delete; + SSLSessionCache &operator=(const SSLSessionCache &) = delete; + private: SSLSessionBucket *session_bucket = nullptr; size_t nbuckets; diff --git a/iocore/net/SSLSessionTicket.cc b/iocore/net/SSLSessionTicket.cc index 151c64a95a1..07eea13914d 100644 --- a/iocore/net/SSLSessionTicket.cc +++ b/iocore/net/SSLSessionTicket.cc @@ -57,13 +57,13 @@ ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, { SSLCertificateConfig::scoped_config lookup; SSLTicketKeyConfig::scoped_config params; - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + SSLNetVConnection &netvc = *SSLNetVCAccess(ssl); // Get the IP address to look up the keyblock IpEndpoint ip; int namelen = sizeof(ip); SSLCertContext *cc = nullptr; - if (0 == safe_getsockname(netvc->get_socket(), &ip.sa, &namelen)) { + if (0 == safe_getsockname(netvc.get_socket(), &ip.sa, &namelen)) { cc = lookup->find(ip); } ssl_ticket_key_block *keyblock = nullptr; @@ -99,8 +99,7 @@ ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_old_key_stat); } - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - netvc->setSSLSessionCacheHit(true); + netvc.setSSLSessionCacheHit(true); // When we decrypt with an "older" key, encrypt the ticket again with the most recent key. return (i == 0) ? 1 : 2; } From ed6058cf1cc7b90c0c3a4ee4ff02ca59f660c0d0 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Tue, 23 Apr 2019 16:31:00 +0800 Subject: [PATCH 444/526] Change to C++ style pointer casting --- iocore/net/P_UDPConnection.h | 24 ++++++++++++------------ iocore/net/P_UDPNet.h | 4 ++-- iocore/net/P_UDPPacket.h | 16 ++++++++-------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/iocore/net/P_UDPConnection.h b/iocore/net/P_UDPConnection.h index 443df0ed07c..ef69b31f68b 100644 --- a/iocore/net/P_UDPConnection.h +++ b/iocore/net/P_UDPConnection.h @@ -79,13 +79,13 @@ UDPConnectionInternal::~UDPConnectionInternal() TS_INLINE SOCKET UDPConnection::getFd() { - return ((UDPConnectionInternal *)this)->fd; + return static_cast(this)->fd; } TS_INLINE void UDPConnection::setBinding(struct sockaddr const *s) { - UDPConnectionInternal *p = (UDPConnectionInternal *)this; + UDPConnectionInternal *p = static_cast(this); ats_ip_copy(&p->binding, s); p->binding_valid = 1; } @@ -93,7 +93,7 @@ UDPConnection::setBinding(struct sockaddr const *s) TS_INLINE void UDPConnection::setBinding(IpAddr const &ip, in_port_t port) { - UDPConnectionInternal *p = (UDPConnectionInternal *)this; + UDPConnectionInternal *p = static_cast(this); IpEndpoint addr; addr.assign(ip, htons(port)); ats_ip_copy(&p->binding, addr); @@ -103,7 +103,7 @@ UDPConnection::setBinding(IpAddr const &ip, in_port_t port) TS_INLINE int UDPConnection::getBinding(struct sockaddr *s) { - UDPConnectionInternal *p = (UDPConnectionInternal *)this; + UDPConnectionInternal *p = static_cast(this); ats_ip_copy(s, &p->binding); return p->binding_valid; } @@ -111,13 +111,13 @@ UDPConnection::getBinding(struct sockaddr *s) TS_INLINE void UDPConnection::destroy() { - ((UDPConnectionInternal *)this)->tobedestroyed = 1; + static_cast(this)->tobedestroyed = 1; } TS_INLINE int UDPConnection::shouldDestroy() { - return ((UDPConnectionInternal *)this)->tobedestroyed; + return static_cast(this)->tobedestroyed; } TS_INLINE void @@ -129,13 +129,13 @@ UDPConnection::AddRef() TS_INLINE int UDPConnection::GetRefCount() { - return ((UDPConnectionInternal *)this)->refcount; + return static_cast(this)->refcount; } TS_INLINE int UDPConnection::GetSendGenerationNumber() { - return ((UDPConnectionInternal *)this)->sendGenerationNum; + return static_cast(this)->sendGenerationNum; } TS_INLINE int @@ -147,7 +147,7 @@ UDPConnection::getPortNum() TS_INLINE int64_t UDPConnection::cancel() { - UDPConnectionInternal *p = (UDPConnectionInternal *)this; + UDPConnectionInternal *p = static_cast(this); p->sendGenerationNum++; p->lastPktStartTime = p->lastSentPktStartTime; @@ -157,7 +157,7 @@ UDPConnection::cancel() TS_INLINE void UDPConnection::SetLastSentPktTSSeqNum(int64_t sentSeqNum) { - ((UDPConnectionInternal *)this)->lastSentPktTSSeqNum = sentSeqNum; + static_cast(this)->lastSentPktTSSeqNum = sentSeqNum; } TS_INLINE void @@ -165,6 +165,6 @@ UDPConnection::setContinuation(Continuation *c) { // it is not safe to switch among continuations that don't share locks ink_assert(mutex.get() == nullptr || c->mutex == mutex); - mutex = c->mutex; - ((UDPConnectionInternal *)this)->continuation = c; + mutex = c->mutex; + static_cast(this)->continuation = c; } diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index 5340178e81c..99ca6150a01 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -336,11 +336,11 @@ struct PollCont; static inline PollCont * get_UDPPollCont(EThread *t) { - return (PollCont *)ETHREAD_GET_PTR(t, udpNetInternal.pollCont_offset); + return static_cast(ETHREAD_GET_PTR(t, udpNetInternal.pollCont_offset)); } static inline UDPNetHandler * get_UDPNetHandler(EThread *t) { - return (UDPNetHandler *)ETHREAD_GET_PTR(t, udpNetInternal.udpNetHandler_offset); + return static_cast(ETHREAD_GET_PTR(t, udpNetInternal.udpNetHandler_offset)); } diff --git a/iocore/net/P_UDPPacket.h b/iocore/net/P_UDPPacket.h index 80426caa2fc..21fbfe7269d 100644 --- a/iocore/net/P_UDPPacket.h +++ b/iocore/net/P_UDPPacket.h @@ -86,7 +86,7 @@ UDPPacketInternal::free() TS_INLINE void UDPPacket::append_block(IOBufferBlock *block) { - UDPPacketInternal *p = (UDPPacketInternal *)this; + UDPPacketInternal *p = static_cast(this); if (block) { if (p->chain) { // append to end @@ -104,7 +104,7 @@ UDPPacket::append_block(IOBufferBlock *block) TS_INLINE int64_t UDPPacket::getPktLength() const { - UDPPacketInternal *p = (UDPPacketInternal *)this; + UDPPacketInternal *p = const_cast(static_cast(this)); IOBufferBlock *b; p->pktLength = 0; @@ -119,13 +119,13 @@ UDPPacket::getPktLength() const TS_INLINE void UDPPacket::free() { - ((UDPPacketInternal *)this)->free(); + static_cast(this)->free(); } TS_INLINE void UDPPacket::setContinuation(Continuation *c) { - ((UDPPacketInternal *)this)->cont = c; + static_cast(this)->cont = c; } TS_INLINE void @@ -138,7 +138,7 @@ UDPPacket::setConnection(UDPConnection *c) assert will prevent that. The "if" clause enables correct handling of the connection ref. counts in such a scenario. */ - UDPConnectionInternal *&conn = ((UDPPacketInternal *)this)->conn; + UDPConnectionInternal *&conn = static_cast(this)->conn; if (conn) { if (conn == c) @@ -146,7 +146,7 @@ UDPPacket::setConnection(UDPConnection *c) conn->Release(); conn = nullptr; } - conn = (UDPConnectionInternal *)c; + conn = static_cast(c); conn->AddRef(); } @@ -154,13 +154,13 @@ TS_INLINE IOBufferBlock * UDPPacket::getIOBlockChain() { ink_assert(dynamic_cast(this) != nullptr); - return ((UDPPacketInternal *)this)->chain.get(); + return static_cast(this)->chain.get(); } TS_INLINE UDPConnection * UDPPacket::getConnection() { - return ((UDPPacketInternal *)this)->conn; + return static_cast(this)->conn; } TS_INLINE UDPPacket * From 1b84e2a45a25f1af903304ee2cc07dd13e295684 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 24 Apr 2019 07:14:07 +0800 Subject: [PATCH 445/526] cppcheck: fixes issues in ink_uuid.h (BROKEN) Function parameter 'other' should be passed by const reference. --- include/tscore/ink_uuid.h | 2 +- src/tscore/ink_uuid.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/tscore/ink_uuid.h b/include/tscore/ink_uuid.h index 32127dcfc3a..7106d24c60e 100644 --- a/include/tscore/ink_uuid.h +++ b/include/tscore/ink_uuid.h @@ -34,7 +34,7 @@ class ATSUuid public: // Constructors ATSUuid() {} - ATSUuid &operator=(const ATSUuid other); + ATSUuid &operator=(const ATSUuid &other); // Initialize the UUID from a string bool parseString(const char *str); diff --git a/src/tscore/ink_uuid.cc b/src/tscore/ink_uuid.cc index 92266cdf23c..e2a56f0c033 100644 --- a/src/tscore/ink_uuid.cc +++ b/src/tscore/ink_uuid.cc @@ -52,7 +52,7 @@ ATSUuid::initialize(TSUuidVersion v) // Copy assignment ATSUuid & -ATSUuid::operator=(const ATSUuid other) +ATSUuid::operator=(const ATSUuid &other) { memcpy(_uuid.data, other._uuid.data, sizeof(_uuid.data)); memcpy(_string, other._string, sizeof(_string)); From f5f4bf0ed32272b972e46c67e7c5f8ff510a71c8 Mon Sep 17 00:00:00 2001 From: Nozomi Inanami <42267827+nozomi1773@users.noreply.github.com> Date: Tue, 23 Apr 2019 16:49:37 +0900 Subject: [PATCH 446/526] fix ssl.server.ticket_key.filename default --- doc/admin-guide/files/records.config.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 7a729b98968..953473163e0 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3317,7 +3317,7 @@ SSL Termination The filename of the certificate authority that client certificates will be verified against. -.. ts:cv:: CONFIG proxy.config.ssl.server.ticket_key.filename STRING ssl_ticket.key +.. ts:cv:: CONFIG proxy.config.ssl.server.ticket_key.filename STRING NULL The filename of the default and global ticket key for SSL sessions. The location is relative to the :ts:cv:`proxy.config.ssl.server.cert.path` directory. One way to generate this would be to run From 41ce63554d6a42c77a751dcc5d052e105f684d22 Mon Sep 17 00:00:00 2001 From: Nozomi Inanami <42267827+nozomi1773@users.noreply.github.com> Date: Tue, 23 Apr 2019 16:50:43 +0900 Subject: [PATCH 447/526] fix http2.max_header_list_size default --- doc/admin-guide/files/records.config.en.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 953473163e0..d9120b21710 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3660,7 +3660,7 @@ HTTP/2 Configuration The maximum size of the header compression table used to decode header blocks. -.. ts:cv:: CONFIG proxy.config.http2.max_header_list_size INT 4294967295 +.. ts:cv:: CONFIG proxy.config.http2.max_header_list_size INT 131072 :reloadable: This advisory setting informs a peer of the maximum size of header list From 56bb32058cf6c27f8c2a5bfd8fe61ae748e49ac2 Mon Sep 17 00:00:00 2001 From: Jan van Doorn Date: Tue, 23 Apr 2019 16:11:57 +0800 Subject: [PATCH 448/526] cppcheck: fixes issues found for plugins/compress --- plugins/compress/compress.cc | 48 +++++++++++++++--------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index a14277f4c6b..0e748cc50b1 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -201,11 +201,10 @@ vary_header(TSMBuffer bufp, TSMLoc hdr_loc) ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_VARY, TS_MIME_LEN_VARY); if (ce_loc) { int idx, count, len; - const char *value; count = TSMimeHdrFieldValuesCount(bufp, hdr_loc, ce_loc); for (idx = 0; idx < count; idx++) { - value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len); + const char *value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len); if (len && strncasecmp("Accept-Encoding", value, len) == 0) { // Bail, Vary: Accept-Encoding already sent from origin TSHandleMLocRelease(bufp, hdr_loc, ce_loc); @@ -244,13 +243,13 @@ etag_header(TSMBuffer bufp, TSMLoc hdr_loc) ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG); if (ce_loc) { - int changetag = 1; int strl; const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl); // do not alter weak etags. // FIXME: consider just making the etag weak for compressed content if (strl >= 2) { + int changetag = 1; if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') { changetag = 0; } @@ -301,15 +300,14 @@ static void gzip_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) { TSIOBufferBlock downstream_blkp; - char *downstream_buffer; int64_t downstream_length; int err; data->zstrm.next_in = (unsigned char *)upstream_buffer; data->zstrm.avail_in = upstream_length; while (data->zstrm.avail_in > 0) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); data->zstrm.next_out = (unsigned char *)downstream_buffer; data->zstrm.avail_out = downstream_length; @@ -342,7 +340,6 @@ static bool brotli_compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_length, BrotliEncoderOperation op) { TSIOBufferBlock downstream_blkp; - char *downstream_buffer; int64_t downstream_length; data->bstrm.next_in = (uint8_t *)upstream_buffer; @@ -350,8 +347,8 @@ brotli_compress_operation(Data *data, const char *upstream_buffer, int64_t upstr bool ok = true; while (ok) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); data->bstrm.next_out = (unsigned char *)downstream_buffer; data->bstrm.avail_out = downstream_length; @@ -363,7 +360,7 @@ brotli_compress_operation(Data *data, const char *upstream_buffer, int64_t upstr if (!ok) { error("BrotliEncoderCompressStream(%d) call failed", op); - return ok; + return false; } TSIOBufferProduce(data->downstream_buffer, downstream_length - data->bstrm.avail_out); @@ -405,7 +402,6 @@ static void compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) { TSIOBufferBlock downstream_blkp; - const char *upstream_buffer; int64_t upstream_length; while (amount > 0) { downstream_blkp = TSIOBufferReaderStart(upstream_reader); @@ -414,7 +410,7 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) return; } - upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length); + const char *upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length); if (!upstream_buffer) { error("couldn't get from TSIOBufferBlockReadStart"); return; @@ -446,20 +442,18 @@ gzip_transform_finish(Data *data) { if (data->state == transform_state_output) { TSIOBufferBlock downstream_blkp; - char *downstream_buffer; int64_t downstream_length; - int err; data->state = transform_state_finished; for (;;) { downstream_blkp = TSIOBufferStart(data->downstream_buffer); - downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - data->zstrm.next_out = (unsigned char *)downstream_buffer; - data->zstrm.avail_out = downstream_length; + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + data->zstrm.next_out = (unsigned char *)downstream_buffer; + data->zstrm.avail_out = downstream_length; - err = deflate(&data->zstrm, Z_FINISH); + int err = deflate(&data->zstrm, Z_FINISH); if (downstream_length > (int64_t)data->zstrm.avail_out) { TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out); @@ -534,7 +528,6 @@ compress_transform_do(TSCont contp) TSVIO upstream_vio; Data *data; int64_t upstream_todo; - int64_t upstream_avail; int64_t downstream_bytes_written; data = (Data *)TSContDataGet(contp); @@ -559,7 +552,7 @@ compress_transform_do(TSCont contp) upstream_todo = TSVIONTodoGet(upstream_vio); if (upstream_todo > 0) { - upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio)); + int64_t upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio)); if (upstream_todo > upstream_avail) { upstream_todo = upstream_avail; @@ -637,8 +630,7 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration TSMLoc cfield; const char *value; - int nvalues; - int i, compression_acceptable, len; + int len; TSHttpStatus resp_status; if (server) { @@ -680,9 +672,9 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration *algorithms = host_configuration->compression_algorithms(); cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); if (cfield != TS_NULL_MLOC) { - compression_acceptable = 0; - nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield); - for (i = 0; i < nvalues; i++) { + int compression_acceptable = 0; + int nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield); + for (int i = 0; i < nvalues; i++) { value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len); if (!value) { continue; @@ -734,14 +726,14 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH); if (field_loc != TS_NULL_MLOC) { - unsigned int value = TSMimeHdrFieldValueUintGet(bufp, hdr_loc, field_loc, -1); + unsigned int hdr_value = TSMimeHdrFieldValueUintGet(bufp, hdr_loc, field_loc, -1); TSHandleMLocRelease(bufp, hdr_loc, field_loc); - if (value == 0) { + if (hdr_value == 0) { info("response is 0-length, not compressible"); return 0; } - if (value < host_configuration->minimum_content_length()) { + if (hdr_value < host_configuration->minimum_content_length()) { info("response is is smaller than minimum content length, not compressing"); return 0; } From c6d08f430588cda0dd3f3b34e3ac902532d31adc Mon Sep 17 00:00:00 2001 From: bichen Date: Tue, 23 Apr 2019 10:12:44 -0700 Subject: [PATCH 449/526] cppcheck: Change to C++ style pointer casting in Thread.cc --- iocore/eventsystem/Thread.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iocore/eventsystem/Thread.cc b/iocore/eventsystem/Thread.cc index f51232d1616..e239bf3a5b1 100644 --- a/iocore/eventsystem/Thread.cc +++ b/iocore/eventsystem/Thread.cc @@ -50,13 +50,13 @@ static bool initialized ATS_UNUSED = ([]() -> bool { Thread::Thread() { mutex = new_ProxyMutex(); - MUTEX_TAKE_LOCK(mutex, (EThread *)this); + MUTEX_TAKE_LOCK(mutex, static_cast(this)); mutex->nthread_holding += THREAD_MUTEX_THREAD_HOLDING; } Thread::~Thread() { - ink_release_assert(mutex->thread_holding == (EThread *)this); + ink_release_assert(mutex->thread_holding == static_cast(this)); if (ink_thread_getspecific(Thread::thread_data_key) == this) { // Clear pointer to this object stored in thread-specific data by set_specific. @@ -65,7 +65,7 @@ Thread::~Thread() } mutex->nthread_holding -= THREAD_MUTEX_THREAD_HOLDING; - MUTEX_UNTAKE_LOCK(mutex, (EThread *)this); + MUTEX_UNTAKE_LOCK(mutex, static_cast(this)); } /////////////////////////////////////////////// From 4260241bdc6f8ebd917920b0972b9e528b096243 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 24 Apr 2019 09:44:54 +0800 Subject: [PATCH 450/526] cppcheck: fixes issues found in proxy/logging (style) The scope of the variable 'X' can be reduced. (style) Variable 'size' is reassigned a value before the old one has been used. --- proxy/logging/LogFieldAliasMap.cc | 3 +-- proxy/logging/LogSock.cc | 20 +++++++++----------- proxy/logging/LogUtils.cc | 11 ++++++----- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/proxy/logging/LogFieldAliasMap.cc b/proxy/logging/LogFieldAliasMap.cc index 4e186d061b7..caf68a94599 100644 --- a/proxy/logging/LogFieldAliasMap.cc +++ b/proxy/logging/LogFieldAliasMap.cc @@ -44,7 +44,6 @@ LogFieldAliasTable::init(size_t numPairs, ...) size_t n; va_list ap; va_start(ap, numPairs); - char *name; /* A note on the varargs - Although IntType is used internally the compiler doesn't know that @@ -73,7 +72,7 @@ LogFieldAliasTable::init(size_t numPairs, ...) for (n = 0; n < numPairs; n++) { IntType val = va_arg(ap, int); size_t i = val - m_min; - name = va_arg(ap, char *); + char *name = va_arg(ap, char *); m_table[i].name = ats_strdup(name); m_table[i].length = strlen(name); diff --git a/proxy/logging/LogSock.cc b/proxy/logging/LogSock.cc index 7746b9668dc..bf9a9cdf9b3 100644 --- a/proxy/logging/LogSock.cc +++ b/proxy/logging/LogSock.cc @@ -81,7 +81,6 @@ int LogSock::listen(int accept_port, int family) { IpEndpoint bind_addr; - int size = sizeof(bind_addr); char this_host[MAXDNAME]; int ret; ats_scoped_fd accept_sd; @@ -95,7 +94,8 @@ LogSock::listen(int accept_port, int family) return -1; } bind_addr.port() = htons(accept_port); - size = ats_ip_size(&bind_addr.sa); + + int size = ats_ip_size(&bind_addr.sa); // // create the socket for accepting new connections @@ -287,7 +287,7 @@ LogSock::connect(sockaddr const *ip) bool LogSock::pending_data(int *cid, int timeout_msec, bool include_connects) { - int start_index, ret, n_poll_fds, i; + int ret, n_poll_fds, i; static struct pollfd fds[LS_CONST_MAX_CONNS]; int fd_to_cid[LS_CONST_MAX_CONNS]; @@ -312,7 +312,7 @@ LogSock::pending_data(int *cid, int timeout_msec, bool include_connects) n_poll_fds = 1; } else { // look for data on any INCOMING socket - + int start_index; if (include_connects) { start_index = 0; } else { @@ -561,8 +561,6 @@ LogSock::read_alloc(int cid, int *size) bool LogSock::is_connected(int cid, bool ping) const { - int i, j, flags; - ink_assert(cid >= 0 && cid < m_max_connections); if (ct[cid].state == LogSock::LS_STATE_UNUSED) { @@ -570,9 +568,10 @@ LogSock::is_connected(int cid, bool ping) const } if (ping) { - flags = fcntl(ct[cid].sd, F_GETFL); + int flags = fcntl(ct[cid].sd, F_GETFL); ::fcntl(ct[cid].sd, F_SETFL, O_NONBLOCK); - j = ::recv(ct[cid].sd, (char *)&i, sizeof(int), MSG_PEEK); + int i; + int j = ::recv(ct[cid].sd, (char *)&i, sizeof(int), MSG_PEEK); ::fcntl(ct[cid].sd, F_SETFL, flags); if (j != 0) { return true; @@ -723,11 +722,10 @@ LogSock::read_body(int sd, void *buf, int bytes) } unsigned bytes_left = bytes; - unsigned bytes_read; - char *to = (char *)buf; + char *to = (char *)buf; while (bytes_left) { - bytes_read = ::recv(sd, to, bytes_left, 0); + unsigned bytes_read = ::recv(sd, to, bytes_left, 0); to += bytes_read; bytes_left -= bytes_read; } diff --git a/proxy/logging/LogUtils.cc b/proxy/logging/LogUtils.cc index 49587c48992..61f2891222c 100644 --- a/proxy/logging/LogUtils.cc +++ b/proxy/logging/LogUtils.cc @@ -99,12 +99,11 @@ char * LogUtils::timestamp_to_netscape_str(long timestamp) { static char timebuf[64]; // NOTE: not MT safe - static char gmtstr[16]; static long last_timestamp = 0; - static char bad_time[] = "Bad timestamp"; // safety check if (timestamp < 0) { + static char bad_time[] = "Bad timestamp"; return bad_time; } // @@ -134,6 +133,8 @@ LogUtils::timestamp_to_netscape_str(long timestamp) offset = zone / -60; sign = '+'; } + + static char gmtstr[16]; int glen = snprintf(gmtstr, 16, "%c%.2d%.2d", sign, offset / 60, offset % 60); strftime(timebuf, 64 - glen, "%d/%b/%Y:%H:%M:%S ", tms); @@ -155,10 +156,10 @@ LogUtils::timestamp_to_date_str(long timestamp) { static char timebuf[64]; // NOTE: not MT safe static long last_timestamp = 0; - static char bad_time[] = "Bad timestamp"; // safety check if (timestamp < 0) { + static char bad_time[] = "Bad timestamp"; return bad_time; } // @@ -187,10 +188,10 @@ LogUtils::timestamp_to_time_str(long timestamp) { static char timebuf[64]; // NOTE: not MT safe static long last_timestamp = 0; - static char bad_time[] = "Bad timestamp"; // safety check if (timestamp < 0) { + static char bad_time[] = "Bad timestamp"; return bad_time; } // @@ -221,13 +222,13 @@ void LogUtils::manager_alarm(LogUtils::AlarmType alarm_type, const char *msg, ...) { char msg_buf[LOG_MAX_FORMATTED_LINE]; - va_list ap; ink_assert(alarm_type >= 0 && alarm_type < LogUtils::LOG_ALARM_N_TYPES); if (msg == nullptr) { snprintf(msg_buf, sizeof(msg_buf), "No Message"); } else { + va_list ap; va_start(ap, msg); vsnprintf(msg_buf, LOG_MAX_FORMATTED_LINE, msg, ap); va_end(ap); From 0c8da0d6747edbfa72869d1671ec2ecd43c24ad0 Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Tue, 23 Apr 2019 00:23:25 -0700 Subject: [PATCH 451/526] cppcheck C-style pointer casting --- iocore/hostdb/HostDB.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 8a097d8fbd0..f6ec709f2bc 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -240,7 +240,7 @@ bool HostDBCache::is_pending_dns_for_hash(const CryptoHash &hash) { Queue &q = pending_dns_for_hash(hash); - for (HostDBContinuation *c = q.head; c; c = (HostDBContinuation *)c->link.next) { + for (HostDBContinuation *c = q.head; c; c = static_cast(c->link.next)) { if (hash == c->hash.hash) { return true; } @@ -1320,7 +1320,7 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) if (is_rr) { r->app.rr.offset = offset; // This will only be set if is_rr - HostDBRoundRobin *rr_data = (HostDBRoundRobin *)(r->rr()); + HostDBRoundRobin *rr_data = static_cast(r->rr()); ; if (is_srv()) { int skip = 0; @@ -1583,7 +1583,7 @@ HostDBContinuation::set_check_pending_dns() Queue &q = hostDB.pending_dns_for_hash(hash.hash); this->setThreadAffinity(this_ethread()); HostDBContinuation *c = q.head; - for (; c; c = (HostDBContinuation *)c->link.next) { + for (; c; c = static_cast(c->link.next)) { if (hash.hash == c->hash.hash) { Debug("hostdb", "enqueuing additional request"); q.enqueue(this); @@ -1602,7 +1602,7 @@ HostDBContinuation::remove_trigger_pending_dns() HostDBContinuation *c = q.head; Queue qq; while (c) { - HostDBContinuation *n = (HostDBContinuation *)c->link.next; + HostDBContinuation *n = static_cast(c->link.next); if (hash.hash == c->hash.hash) { Debug("hostdb", "dequeuing additional request"); q.remove(c); @@ -1958,7 +1958,7 @@ struct ShowHostDB : public ShowCont { int showLookupDone(int event, Event *e) { - HostDBInfo *r = (HostDBInfo *)e; + HostDBInfo *r = reinterpret_cast(e); CHECK_SHOW(begin("HostDB Lookup")); if (name) { From 3c1b9877cf1cfe1845058144c8b13843606399f7 Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Tue, 23 Apr 2019 01:00:01 -0700 Subject: [PATCH 452/526] cppcheck: Function parameters should be passed by reference --- iocore/hostdb/HostDB.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index f6ec709f2bc..fc876bd8702 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -286,7 +286,7 @@ HostDBBackgroundTask::wait_event(int, void *) struct HostDBSync : public HostDBBackgroundTask { std::string storage_path; std::string full_path; - HostDBSync(int frequency, std::string storage_path, std::string full_path) + HostDBSync(int frequency, const std::string &storage_path, const std::string &full_path) : HostDBBackgroundTask(frequency), storage_path(std::move(storage_path)), full_path(std::move(full_path)){}; int sync_event(int, void *) override From 14df6b0379303eb299d529380d070d9105062eca Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Tue, 23 Apr 2019 01:05:55 -0700 Subject: [PATCH 453/526] cppcheck: syntax error and memory leak fix --- iocore/hostdb/test_I_HostDB.cc | 1 + iocore/hostdb/test_P_HostDB.cc | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/iocore/hostdb/test_I_HostDB.cc b/iocore/hostdb/test_I_HostDB.cc index fe507c73b1a..ad283625fde 100644 --- a/iocore/hostdb/test_I_HostDB.cc +++ b/iocore/hostdb/test_I_HostDB.cc @@ -30,6 +30,7 @@ class HostDBTest : Continuation { }; +int main() { init_diags("net_test", nullptr); diff --git a/iocore/hostdb/test_P_HostDB.cc b/iocore/hostdb/test_P_HostDB.cc index 5656543e8f8..2768b43d96d 100644 --- a/iocore/hostdb/test_P_HostDB.cc +++ b/iocore/hostdb/test_P_HostDB.cc @@ -52,6 +52,7 @@ struct NetTesterSM : public Continuation { str = new char[r + 10]; reader->read(str, r); printf("%s", str); + delete[] str; fflush(stdout); break; case VC_EVENT_READ_COMPLETE: @@ -61,6 +62,7 @@ struct NetTesterSM : public Continuation { str = new char[r + 10]; reader->read(str, r); printf("%s", str); + delete[] str; fflush(stdout); case VC_EVENT_ERROR: vc->do_io_close(); @@ -74,4 +76,7 @@ struct NetTesterSM : public Continuation { } }; -main() {} +int +main() +{ +} From 7fa3b232d5645c24e030bd67a4055cbc0512f781 Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Tue, 23 Apr 2019 01:07:53 -0700 Subject: [PATCH 454/526] cppcheck: (style) The scope of the variable can be reduced. --- iocore/hostdb/P_HostDBProcessor.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h index 00b8731a36b..9739bf6a6eb 100644 --- a/iocore/hostdb/P_HostDBProcessor.h +++ b/iocore/hostdb/P_HostDBProcessor.h @@ -306,10 +306,9 @@ HostDBRoundRobin::select_best_http(sockaddr const *client_ip, ink_time_t now, in Debug("hostdb", "Using default round robin"); unsigned int best_hash_any = 0; unsigned int best_hash_up = 0; - sockaddr const *ip; for (int i = 0; i < good; i++) { - ip = info(i).ip(); - unsigned int h = HOSTDB_CLIENT_IP_HASH(client_ip, ip); + sockaddr const *ip = info(i).ip(); + unsigned int h = HOSTDB_CLIENT_IP_HASH(client_ip, ip); if (best_hash_any <= h) { best_any = i; best_hash_any = h; From c29d8d8900793de1a1a91d0838557ce600e7668f Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Tue, 23 Apr 2019 01:15:46 -0700 Subject: [PATCH 455/526] cppcheck: (portability) %zd in format string (no. 2) requires 'ssize_t' but the argument type is 'size_t {aka unsigned long}' --- iocore/hostdb/test_RefCountCache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iocore/hostdb/test_RefCountCache.cc b/iocore/hostdb/test_RefCountCache.cc index 04df6c148ca..cfa8854570c 100644 --- a/iocore/hostdb/test_RefCountCache.cc +++ b/iocore/hostdb/test_RefCountCache.cc @@ -63,7 +63,7 @@ class ExampleStruct : public RefCountObj { this->idx = -1; items_freed.insert(this); - printf("freeing: %p items_freed.size(): %zd\n", this, items_freed.size()); + printf("freeing: %p items_freed.size(): %zu\n", this, items_freed.size()); } static ExampleStruct * From 65adc6e49891e0b2afafbf8b50e8ca4af3e63da9 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 24 Apr 2019 09:10:23 +0800 Subject: [PATCH 456/526] cppcheck: Fixes issues found in async_http_fetch_streaming (style) Class 'Intercept' has a constructor with 1 argument that is not explicit. (performance) Variable 'main_url_' is assigned in constructor body. Consider performing initialization in initialization list. --- .../AsyncHttpFetchStreaming.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/example/cppapi/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc b/example/cppapi/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc index 1389e9c277c..f953ab08259 100644 --- a/example/cppapi/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc +++ b/example/cppapi/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc @@ -42,11 +42,14 @@ GlobalPlugin *plugin; class Intercept : public InterceptPlugin, public AsyncReceiver { public: - Intercept(Transaction &transaction) - : InterceptPlugin(transaction, InterceptPlugin::SERVER_INTERCEPT), transaction_(transaction), num_fetches_(0) + explicit Intercept(Transaction &transaction) + : InterceptPlugin(transaction, InterceptPlugin::SERVER_INTERCEPT), + transaction_(transaction), + main_url_(transaction.getClientRequest().getUrl().getUrlString()), + num_fetches_(0) { - main_url_ = transaction.getClientRequest().getUrl().getUrlString(); } + void consume(const string &data, InterceptPlugin::RequestDataType type) override; void handleInputComplete() override; void handleAsyncComplete(AsyncHttpFetch &async_http_fetch) override; From 8de79c59d31c6deafe5f02ccdc7e1ad2163c5da5 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Tue, 23 Apr 2019 14:44:56 +0800 Subject: [PATCH 457/526] cppcheck: Minimize variable scopes and use different names to not shadow others --- proxy/http2/HTTP2.cc | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index 082c7b97ebb..8268cd5dd2e 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -89,6 +89,7 @@ write_and_advance(byte_pointer &dst, uint32_t src) { byte_addressable_value pval; + // cppcheck-suppress unreadVariable ; it's an union and be read as pval.bytes pval.value = htonl(src); memcpy(dst.u8, pval.bytes, sizeof(pval.bytes)); dst.u8 += sizeof(pval.bytes); @@ -99,6 +100,7 @@ write_and_advance(byte_pointer &dst, uint16_t src) { byte_addressable_value pval; + // cppcheck-suppress unreadVariable ; it's an union and be read as pval.bytes pval.value = htons(src); memcpy(dst.u8, pval.bytes, sizeof(pval.bytes)); dst.u8 += sizeof(pval.bytes); @@ -218,6 +220,7 @@ http2_write_frame_header(const Http2FrameHeader &hdr, IOVec iov) } byte_addressable_value length; + // cppcheck-suppress unreadVariable ; it's an union and be read as pval.bytes length.value = htonl(hdr.length); // MSB length.bytes[0] is unused. write_and_advance(ptr, length.bytes[1]); @@ -417,8 +420,8 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) ink_assert(http_hdr_type_get(headers->m_http) != HTTP_TYPE_UNKNOWN); if (http_hdr_type_get(headers->m_http) == HTTP_TYPE_REQUEST) { - const char *scheme, *authority, *path, *method; - int scheme_len, authority_len, path_len, method_len; + const char *scheme, *authority, *path; + int scheme_len, authority_len, path_len; // Get values of :scheme, :authority and :path to assemble requested URL if ((field = headers->field_find(HTTP2_VALUE_SCHEME, HTTP2_LEN_SCHEME)) != nullptr && field->value_is_valid()) { @@ -454,7 +457,8 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) // Get value of :method if ((field = headers->field_find(HTTP2_VALUE_METHOD, HTTP2_LEN_METHOD)) != nullptr && field->value_is_valid()) { - method = field->value_get(&method_len); + int method_len; + const char *method = field->value_get(&method_len); int method_wks_idx = hdrtoken_tokenize(method, method_len); http_hdr_method_set(headers->m_heap, headers->m_http, method, method_wks_idx, method_len, false); @@ -478,11 +482,9 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) headers->field_delete(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY); headers->field_delete(HTTP2_VALUE_PATH, HTTP2_LEN_PATH); } else { - int status_len; - const char *status; - if ((field = headers->field_find(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS)) != nullptr) { - status = field->value_get(&status_len); + int status_len; + const char *status = field->value_get(&status_len); headers->status_set(http_parse_status(status, status + status_len)); } else { return PARSE_RESULT_ERROR; @@ -494,8 +496,8 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) // Check validity of all names and values MIMEFieldIter iter; - for (const MIMEField *field = headers->iter_get_first(&iter); field != nullptr; field = headers->iter_get_next(&iter)) { - if (!field->name_is_valid() || !field->value_is_valid()) { + for (auto *mf = headers->iter_get_first(&iter); mf != nullptr; mf = headers->iter_get_next(&iter)) { + if (!mf->name_is_valid() || !mf->value_is_valid()) { return PARSE_RESULT_ERROR; } } From 4338d9efe21c9b6585b1d41cc3a16d4d0589b8b5 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 24 Apr 2019 07:39:12 +0800 Subject: [PATCH 458/526] cppcheck: Reduces variable scope for files in mgmt/... The scope of the variable 'X' can be reduced --- mgmt/LocalManager.cc | 13 ++++--------- mgmt/ProxyConfig.cc | 5 ++--- mgmt/Rollback.cc | 28 +++++++++++----------------- mgmt/WebMgmtUtils.cc | 15 +++++---------- mgmt/api/APITestCliRemote.cc | 9 +++------ mgmt/api/CoreAPI.cc | 7 ++----- mgmt/api/EventCallback.cc | 4 +--- mgmt/api/EventControlMain.cc | 8 ++------ mgmt/api/INKMgmtAPI.cc | 23 +++++++---------------- mgmt/api/NetworkUtilsRemote.cc | 3 +-- mgmt/api/TSControlMain.cc | 3 +-- mgmt/utils/MgmtUtils.cc | 16 +++++++--------- 12 files changed, 46 insertions(+), 88 deletions(-) diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index 8caf6f03303..fae81d336e5 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -312,7 +312,6 @@ LocalManager::initMgmtProcessServer() void LocalManager::pollMgmtProcessServer() { - int num; struct timeval timeout; fd_set fdlist; @@ -355,7 +354,7 @@ LocalManager::pollMgmtProcessServer() } #endif - num = mgmt_select(FD_SETSIZE, &fdlist, nullptr, nullptr, &timeout); + int num = mgmt_select(FD_SETSIZE, &fdlist, nullptr, nullptr, &timeout); switch (num) { case 0: @@ -402,16 +401,14 @@ LocalManager::pollMgmtProcessServer() if (ts::NO_FD != watched_process_fd && FD_ISSET(watched_process_fd, &fdlist)) { int res; MgmtMessageHdr mh_hdr; - MgmtMessageHdr *mh_full; - char *data_raw; keep_polling = true; // read the message if ((res = mgmt_read_pipe(watched_process_fd, (char *)&mh_hdr, sizeof(MgmtMessageHdr))) > 0) { - mh_full = (MgmtMessageHdr *)alloca(sizeof(MgmtMessageHdr) + mh_hdr.data_len); + MgmtMessageHdr *mh_full = (MgmtMessageHdr *)alloca(sizeof(MgmtMessageHdr) + mh_hdr.data_len); memcpy(mh_full, &mh_hdr, sizeof(MgmtMessageHdr)); - data_raw = (char *)mh_full + sizeof(MgmtMessageHdr); + char *data_raw = (char *)mh_full + sizeof(MgmtMessageHdr); if ((res = mgmt_read_pipe(watched_process_fd, data_raw, mh_hdr.data_len)) > 0) { handleMgmtMsgFromProcesses(mh_full); } else if (res < 0) { @@ -781,10 +778,8 @@ LocalManager::signalEvent(int msg_id, const char *data_raw, int data_len) void LocalManager::processEventQueue() { - bool handled_by_mgmt; - while (!this->queue_empty()) { - handled_by_mgmt = false; + bool handled_by_mgmt = false; MgmtMessageHdr *mh = this->dequeue(); auto payload = mh->payload(); diff --git a/mgmt/ProxyConfig.cc b/mgmt/ProxyConfig.cc index 2753ca41380..9c189ca3878 100644 --- a/mgmt/ProxyConfig.cc +++ b/mgmt/ProxyConfig.cc @@ -69,13 +69,12 @@ config_string_alloc_cb(void *data, void *value) char *_ss = (char *)value; char *_new_value = nullptr; -//#define DEBUG_CONFIG_STRING_UPDATE #if defined(DEBUG_CONFIG_STRING_UPDATE) printf("config callback [new, old] = [%s : %s]\n", (_ss) ? (_ss) : (""), (*(char **)data) ? (*(char **)data) : ("")); #endif - int len = -1; + if (_ss) { - len = strlen(_ss); + int len = strlen(_ss); _new_value = (char *)ats_malloc(len + 1); memcpy(_new_value, _ss, len + 1); } diff --git a/mgmt/Rollback.cc b/mgmt/Rollback.cc index 0fcc8e702c7..05e4c427f7e 100644 --- a/mgmt/Rollback.cc +++ b/mgmt/Rollback.cc @@ -72,11 +72,6 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc int testFD; // For open test int testErrno; // For open test - // In case the file is missing - char *highestSeenStr; - char *activeVerStr; - bool needZeroLength; - ink_assert(fileName_ != nullptr); // parent must not also have a parent @@ -133,11 +128,12 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc // If it does not, create a zero length file to prevent total havoc // if (errno == ENOENT) { + bool needZeroLength; mgmt_log("[RollBack::Rollback] Missing Configuration File: %s\n", fileName); if (highestSeen > 0) { - highestSeenStr = createPathStr(highestSeen); - activeVerStr = createPathStr(ACTIVE_VERSION); + char *highestSeenStr = createPathStr(highestSeen); + char *activeVerStr = createPathStr(ACTIVE_VERSION); if (rename(highestSeenStr, activeVerStr) < 0) { mgmt_log("[RollBack::Rollback] Automatic Rollback to prior version failed for %s : %s\n", fileName, strerror(errno)); @@ -687,7 +683,7 @@ Rollback::findVersions_ml(ExpandingArray *listNames) version_t Rollback::extractVersionInfo(ExpandingArray *listNames, const char *testFileName) { - const char *currentVersionStr, *str; + const char *str; version_t version = INVALID_VERSION; // Check to see if the current entry is a rollback file @@ -700,7 +696,7 @@ Rollback::extractVersionInfo(ExpandingArray *listNames, const char *testFileName // Check for the underscore if (*(testFileName + fileNameLen) == '_') { // Check for the integer version number - currentVersionStr = str = testFileName + fileNameLen + 1; + const char *currentVersionStr = str = testFileName + fileNameLen + 1; for (; isdigit(*str) && *str != '\0'; str++) { ; @@ -714,12 +710,11 @@ Rollback::extractVersionInfo(ExpandingArray *listNames, const char *testFileName // Add info about version number and modTime if (listNames != nullptr) { struct stat fileInfo; - versionInfo *verInfo; if (statFile(version, &fileInfo) >= 0) { - verInfo = (versionInfo *)ats_malloc(sizeof(versionInfo)); - verInfo->version = version; - verInfo->modTime = fileInfo.st_mtime; + versionInfo *verInfo = (versionInfo *)ats_malloc(sizeof(versionInfo)); + verInfo->version = version; + verInfo->modTime = fileInfo.st_mtime; listNames->addEntry((void *)verInfo); } } @@ -744,7 +739,6 @@ Rollback::findVersions_ml(Queue &q) ExpandingArray versions(25, true); int num; versionInfo *foundVer; - versionInfo *addInfo; version_t highest; // Get the version info and sort it @@ -757,9 +751,9 @@ Rollback::findVersions_ml(Queue &q) foundVer = (versionInfo *)versions[i]; // We need to create our own copy so that // constructor gets run - addInfo = new versionInfo; - addInfo->version = foundVer->version; - addInfo->modTime = foundVer->modTime; + versionInfo *addInfo = new versionInfo; + addInfo->version = foundVer->version; + addInfo->modTime = foundVer->modTime; q.enqueue(addInfo); } diff --git a/mgmt/WebMgmtUtils.cc b/mgmt/WebMgmtUtils.cc index 474114cec17..fa7fd9e9576 100644 --- a/mgmt/WebMgmtUtils.cc +++ b/mgmt/WebMgmtUtils.cc @@ -519,7 +519,6 @@ bytesFromInt(RecInt bytes, char *bufVal) const int64_t gb = 1073741824; const long int mb = 1048576; const long int kb = 1024; - int bytesP; double unitBytes; if (bytes >= gb) { @@ -531,7 +530,7 @@ bytesFromInt(RecInt bytes, char *bufVal) // has plenty of precision for a regular int // and saves from 64 bit arithmetic which may // be expensive on some processors - bytesP = (int)bytes; + int bytesP = (int)bytes; if (bytesP >= mb) { unitBytes = bytes / (double)mb; snprintf(bufVal, 15, "%.1f MB", unitBytes); @@ -1041,9 +1040,6 @@ recordRegexCheck(const char *pattern, const char *value) bool recordRangeCheck(const char *pattern, const char *value) { - int l_limit; - int u_limit; - int val; char *p = (char *)pattern; Tokenizer dashTok("-"); @@ -1052,9 +1048,9 @@ recordRangeCheck(const char *pattern, const char *value) p++; } // skip to '[' if (dashTok.Initialize(++p, COPY_TOKS) == 2) { - l_limit = atoi(dashTok[0]); - u_limit = atoi(dashTok[1]); - val = atoi(value); + int l_limit = atoi(dashTok[0]); + int u_limit = atoi(dashTok[1]); + int val = atoi(value); if (val >= l_limit && val <= u_limit) { return true; } @@ -1075,12 +1071,11 @@ recordIPCheck(const char *pattern, const char *value) Tokenizer dotTok1("."); Tokenizer dotTok2("."); - int i; check = true; if (recordRegexCheck(range_pattern, pattern) && recordRegexCheck(ip_pattern, value)) { if (dotTok1.Initialize((char *)pattern, COPY_TOKS) == 4 && dotTok2.Initialize((char *)value, COPY_TOKS) == 4) { - for (i = 0; i < 4 && check; i++) { + for (int i = 0; i < 4 && check; i++) { if (!recordRangeCheck(dotTok1[i], dotTok2[i])) { check = false; } diff --git a/mgmt/api/APITestCliRemote.cc b/mgmt/api/APITestCliRemote.cc index 80f8cca14e9..aa1e7a224c5 100644 --- a/mgmt/api/APITestCliRemote.cc +++ b/mgmt/api/APITestCliRemote.cc @@ -126,7 +126,6 @@ void print_string_list(TSStringList list) { int i, count, buf_pos = 0; - char *str; char buf[1000]; if (!list) { @@ -134,7 +133,7 @@ print_string_list(TSStringList list) } count = TSStringListLen(list); for (i = 0; i < count; i++) { - str = TSStringListDequeue(list); + char *str = TSStringListDequeue(list); snprintf(buf + buf_pos, sizeof(buf) - buf_pos, "%s,", str); buf_pos = strlen(buf); TSStringListEnqueue(list, str); @@ -149,12 +148,11 @@ void print_int_list(TSIntList list) { int i, count, buf_pos = 0; - int *elem; char buf[1000]; count = TSIntListLen(list); for (i = 0; i < count; i++) { - elem = TSIntListDequeue(list); + int *elem = TSIntListDequeue(list); snprintf(buf + buf_pos, sizeof(buf) - buf_pos, "%d:", *elem); buf_pos = strlen(buf); TSIntListEnqueue(list, elem); @@ -504,7 +502,6 @@ test_rec_get(char *rec_name) void test_record_get_mlt() { - TSRecordEle *rec_ele; TSStringList name_list; TSList rec_list; int i, num; @@ -547,7 +544,7 @@ test_record_get_mlt() } for (i = 0; i < num; i++) { - rec_ele = (TSRecordEle *)TSListDequeue(rec_list); + TSRecordEle *rec_ele = (TSRecordEle *)TSListDequeue(rec_list); if (!rec_ele) { printf("ERROR\n"); break; diff --git a/mgmt/api/CoreAPI.cc b/mgmt/api/CoreAPI.cc index d75d7669ff1..56da4d4f044 100644 --- a/mgmt/api/CoreAPI.cc +++ b/mgmt/api/CoreAPI.cc @@ -789,9 +789,6 @@ EventResolve(const char *event_name) TSMgmtError ActiveEventGetMlt(LLQ *active_events) { - int event_id; - char *event_name; - if (!active_events) { return TS_ERR_PARAMS; } @@ -804,8 +801,8 @@ ActiveEventGetMlt(LLQ *active_events) // iterate through hash-table and insert event_name's into active_events list for (auto &&it : event_ht) { // convert key to int; insert into llQ - event_id = ink_atoi(it.first.c_str()); - event_name = get_event_name(event_id); + int event_id = ink_atoi(it.first.c_str()); + char *event_name = get_event_name(event_id); if (event_name) { if (!enqueue(active_events, event_name)) { // returns true if successful return TS_ERR_FAIL; diff --git a/mgmt/api/EventCallback.cc b/mgmt/api/EventCallback.cc index 2fc463a4cfd..59bdf930186 100644 --- a/mgmt/api/EventCallback.cc +++ b/mgmt/api/EventCallback.cc @@ -183,7 +183,6 @@ TSMgmtError cb_table_register(CallbackTable *cb_table, const char *event_name, TSEventSignalFunc func, void *data, bool *first_cb) { bool first_time = false; - int id; EventCallbackT *event_cb; // create new EventCallbackT EACH TIME enqueue // the data and event_name can be NULL @@ -211,8 +210,7 @@ cb_table_register(CallbackTable *cb_table, const char *event_name, TSEventSignal enqueue(i, event_cb); } } else { // register callback for specific alarm - // printf("[EventSignalCbRegister] Register callback for %s\n", event_name); - id = get_event_id(event_name); + int id = get_event_id(event_name); if (id != -1) { if (!cb_table->event_callback_l[id]) { cb_table->event_callback_l[id] = create_queue(); diff --git a/mgmt/api/EventControlMain.cc b/mgmt/api/EventControlMain.cc index 4fc2668c87a..b4859ebb399 100644 --- a/mgmt/api/EventControlMain.cc +++ b/mgmt/api/EventControlMain.cc @@ -177,11 +177,8 @@ delete_event_queue(LLQ *q) return; } - // now for every element, dequeue and free - TSMgmtEvent *ele; - while (!queue_is_empty(q)) { - ele = (TSMgmtEvent *)dequeue(q); + TSMgmtEvent *ele = (TSMgmtEvent *)dequeue(q); ats_free(ele); } @@ -257,7 +254,6 @@ event_callback_main(void *arg) fd_set selectFDs; // for select call EventClientT *client_entry; // an entry of fd to alarms mapping - int fds_ready; // return value for select go here struct timeval timeout; while (true) { @@ -280,7 +276,7 @@ event_callback_main(void *arg) } // select call - timeout is set so we can check events at regular intervals - fds_ready = mgmt_select(FD_SETSIZE, &selectFDs, (fd_set *)nullptr, (fd_set *)nullptr, &timeout); + int fds_ready = mgmt_select(FD_SETSIZE, &selectFDs, (fd_set *)nullptr, (fd_set *)nullptr, &timeout); // check return if (fds_ready > 0) { diff --git a/mgmt/api/INKMgmtAPI.cc b/mgmt/api/INKMgmtAPI.cc index 15d2caf8d78..e332764423c 100644 --- a/mgmt/api/INKMgmtAPI.cc +++ b/mgmt/api/INKMgmtAPI.cc @@ -147,7 +147,6 @@ tsapi bool TSListIsValid(TSList l) { int i, len; - void *ele; if (!l) { return false; @@ -155,7 +154,7 @@ TSListIsValid(TSList l) len = queue_len((LLQ *)l); for (i = 0; i < len; i++) { - ele = (void *)dequeue((LLQ *)l); + void *ele = (void *)dequeue((LLQ *)l); if (!ele) { return false; } @@ -175,15 +174,13 @@ TSStringListCreate() tsapi void TSStringListDestroy(TSStringList strl) { - char *str; - if (!strl) { return; } /* dequeue each element and free it */ while (!queue_is_empty((LLQ *)strl)) { - str = (char *)dequeue((LLQ *)strl); + char *str = (char *)dequeue((LLQ *)strl); ats_free(str); } @@ -246,7 +243,6 @@ tsapi bool TSStringListIsValid(TSStringList strl) { int i, len; - char *str; if (!strl) { return false; @@ -254,7 +250,7 @@ TSStringListIsValid(TSStringList strl) len = queue_len((LLQ *)strl); for (i = 0; i < len; i++) { - str = (char *)dequeue((LLQ *)strl); + char *str = (char *)dequeue((LLQ *)strl); if (!str) { return false; } @@ -274,15 +270,13 @@ TSIntListCreate() tsapi void TSIntListDestroy(TSIntList intl) { - int *iPtr; - if (!intl) { return; } /* dequeue each element and free it */ while (!queue_is_empty((LLQ *)intl)) { - iPtr = (int *)dequeue((LLQ *)intl); + int *iPtr = (int *)dequeue((LLQ *)intl); ats_free(iPtr); } @@ -534,8 +528,6 @@ TSRecordGetString(const char *rec_name, TSString *string_val) tsapi TSMgmtError TSRecordGetMlt(TSStringList rec_names, TSList rec_vals) { - TSRecordEle *ele; - char *rec_name; int num_recs, i, j; TSMgmtError ret; @@ -545,12 +537,12 @@ TSRecordGetMlt(TSStringList rec_names, TSList rec_vals) num_recs = queue_len((LLQ *)rec_names); for (i = 0; i < num_recs; i++) { - rec_name = (char *)dequeue((LLQ *)rec_names); // remove name from list + char *rec_name = (char *)dequeue((LLQ *)rec_names); // remove name from list if (!rec_name) { return TS_ERR_PARAMS; // NULL is invalid record name } - ele = TSRecordEleCreate(); + TSRecordEle *ele = TSRecordEleCreate(); ret = MgmtRecordGet(rec_name, ele); enqueue((LLQ *)rec_names, rec_name); // return name to list @@ -632,7 +624,6 @@ tsapi TSMgmtError TSRecordSetMlt(TSList rec_list, TSActionNeedT *action_need) { int num_recs, ret, i; - TSRecordEle *ele; TSMgmtError status = TS_ERR_OKAY; TSActionNeedT top_action_req = TS_ACTION_UNDEFINED; @@ -643,7 +634,7 @@ TSRecordSetMlt(TSList rec_list, TSActionNeedT *action_need) num_recs = queue_len((LLQ *)rec_list); for (i = 0; i < num_recs; i++) { - ele = (TSRecordEle *)dequeue((LLQ *)rec_list); + TSRecordEle *ele = (TSRecordEle *)dequeue((LLQ *)rec_list); if (ele) { switch (ele->rec_type) { case TS_REC_INT: diff --git a/mgmt/api/NetworkUtilsRemote.cc b/mgmt/api/NetworkUtilsRemote.cc index 4c75f802b69..35e731bb089 100644 --- a/mgmt/api/NetworkUtilsRemote.cc +++ b/mgmt/api/NetworkUtilsRemote.cc @@ -507,7 +507,6 @@ send_register_all_callbacks(int fd, CallbackTable *cb_table) TSMgmtError send_unregister_all_callbacks(int fd, CallbackTable *cb_table) { - int event_id; LLQ *events_with_cb; // list of events with at least one callback int reg_callback[NUM_EVENTS]; TSMgmtError err, send_err = TS_ERR_FAIL; @@ -525,7 +524,7 @@ send_unregister_all_callbacks(int fd, CallbackTable *cb_table) int num_events = queue_len(events_with_cb); // iterate through the LLQ and mark events that have a callback for (int i = 0; i < num_events; i++) { - event_id = *(int *)dequeue(events_with_cb); + int event_id = *(int *)dequeue(events_with_cb); reg_callback[event_id] = 1; // mark the event as having a callback } delete_queue(events_with_cb); diff --git a/mgmt/api/TSControlMain.cc b/mgmt/api/TSControlMain.cc index 2f59f53227b..57a676115ee 100644 --- a/mgmt/api/TSControlMain.cc +++ b/mgmt/api/TSControlMain.cc @@ -133,7 +133,6 @@ ts_ctrl_main(void *arg) fd_set selectFDs; // for select call ClientT *client_entry; // an entry of fd to alarms mapping - int fds_ready; // stores return value for select struct timeval timeout; // loops until TM dies; waits for and processes requests from clients @@ -158,7 +157,7 @@ ts_ctrl_main(void *arg) } // select call - timeout is set so we can check events at regular intervals - fds_ready = mgmt_select(FD_SETSIZE, &selectFDs, (fd_set *)nullptr, (fd_set *)nullptr, &timeout); + int fds_ready = mgmt_select(FD_SETSIZE, &selectFDs, (fd_set *)nullptr, (fd_set *)nullptr, &timeout); // check if have any connections or requests if (fds_ready > 0) { diff --git a/mgmt/utils/MgmtUtils.cc b/mgmt/utils/MgmtUtils.cc index 57ad7ed737d..71e5fd6601e 100644 --- a/mgmt/utils/MgmtUtils.cc +++ b/mgmt/utils/MgmtUtils.cc @@ -53,11 +53,11 @@ mgmt_use_syslog() int mgmt_readline(int soc, char *buf, int maxlen) { - int n = 0, rc; + int n = 0; char c; for (; n < maxlen; n++) { - rc = read_socket(soc, &c, 1); + int rc = read_socket(soc, &c, 1); if (rc == 1) { *buf++ = c; if (c == '\n') { @@ -105,12 +105,12 @@ mgmt_readline(int soc, char *buf, int maxlen) int mgmt_writeline(int soc, const char *data, int nbytes) { - int nleft, nwritten, n; + int nleft, n; const char *tmp = data; nleft = nbytes; while (nleft > 0) { - nwritten = write_socket(soc, tmp, nleft); + int nwritten = write_socket(soc, tmp, nleft); if (nwritten <= 0) { /* Error or nothing written */ return nwritten; } @@ -141,12 +141,11 @@ mgmt_writeline(int soc, const char *data, int nbytes) int mgmt_read_pipe(int fd, char *buf, int bytes_to_read) { - int err = 0; char *p = buf; int bytes_read = 0; while (bytes_to_read > 0) { - err = read_socket(fd, p, bytes_to_read); + int err = read_socket(fd, p, bytes_to_read); if (err == 0) { return err; } else if (err < 0) { @@ -183,12 +182,11 @@ mgmt_read_pipe(int fd, char *buf, int bytes_to_read) int mgmt_write_pipe(int fd, char *buf, int bytes_to_write) { - int err = 0; char *p = buf; int bytes_written = 0; while (bytes_to_write > 0) { - err = write_socket(fd, p, bytes_to_write); + int err = write_socket(fd, p, bytes_to_write); if (err == 0) { return err; } else if (err < 0) { @@ -272,7 +270,6 @@ void mgmt_fatal(const int lerrno, const char *message_format, ...) { va_list ap; - char extended_format[4096], message[4096]; va_start(ap, message_format); @@ -283,6 +280,7 @@ mgmt_fatal(const int lerrno, const char *message_format, ...) FatalV(message_format, ap); } else { + char extended_format[4096], message[4096]; snprintf(extended_format, sizeof(extended_format), "FATAL ==> %s", message_format); vsprintf(message, extended_format, ap); From 769923e21556375d85afdda422533c07d82d7518 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 24 Apr 2019 09:38:38 +0800 Subject: [PATCH 459/526] cppcheck: Reduce the scope of the variable 'netvc' > [Http2SessionAccept.cc:70]: (style) The scope of the variable 'netvc' can be reduced. --- proxy/http2/Http2SessionAccept.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/proxy/http2/Http2SessionAccept.cc b/proxy/http2/Http2SessionAccept.cc index f3180798f00..0fd55f848dd 100644 --- a/proxy/http2/Http2SessionAccept.cc +++ b/proxy/http2/Http2SessionAccept.cc @@ -67,12 +67,11 @@ Http2SessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferRead int Http2SessionAccept::mainEvent(int event, void *data) { - NetVConnection *netvc; ink_release_assert(event == NET_EVENT_ACCEPT || event == EVENT_ERROR); ink_release_assert((event == NET_EVENT_ACCEPT) ? (data != nullptr) : (1)); if (event == NET_EVENT_ACCEPT) { - netvc = static_cast(data); + NetVConnection *netvc = static_cast(data); if (!this->accept(netvc, nullptr, nullptr)) { netvc->do_io_close(); } From 0d821cc53803ba3fbf211e61475cee9e0a47f49a Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 23 Apr 2019 14:39:46 +0800 Subject: [PATCH 460/526] Removes non-existent include dir reference --- src/tscore/Makefile.am | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index 1edb2ceb1bb..714e0bdb908 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -30,7 +30,6 @@ lib_LTLIBRARIES = libtscore.la AM_CPPFLAGS += \ $(iocore_include_dirs) \ -I$(abs_top_srcdir)/include \ - -I$(abs_top_srcdir)/include/records \ -I$(abs_top_srcdir)/lib \ $(TS_INCLUDES) \ @YAMLCPP_INCLUDES@ From f294f6271414b3cca109049cc29b47cfde5c5d35 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 23 Apr 2019 16:10:20 +0800 Subject: [PATCH 461/526] cppcheck: fixes issues found for tests in proxy/http --- proxy/http/test_socket_close.cc | 15 +++++---------- proxy/http/unit_tests/test_ForwardedConfig.cc | 6 +++--- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/proxy/http/test_socket_close.cc b/proxy/http/test_socket_close.cc index 4be9c58a121..46001403c06 100644 --- a/proxy/http/test_socket_close.cc +++ b/proxy/http/test_socket_close.cc @@ -93,7 +93,7 @@ struct State { int64_t nbytes_write; // number of bytes to write intte_t nbytes_read; // number of bytes to read - State() : state(STATE_IDLE), tasks_count(0) {} + State() : state(STATE_IDLE), tasks_count(0), nbytes_write(0), {} }; struct Conn { @@ -241,7 +241,6 @@ state_act(Conn *c) void state_act_task(Conn *c) { - int error; char write_ch = 'T', read_ch; int r; @@ -332,7 +331,6 @@ int do_connect(Conn *from, Conn *to) { assert(to->listen_s > 0); - int error; // create a non-blocking socket if ((from->s = create_nonblocking_socket()) < 0) { @@ -341,7 +339,7 @@ do_connect(Conn *from, Conn *to) } // connect if (connect(from->s, (struct sockaddr *)&to->addr, sizeof(to->addr)) < 0) { - error = -errno; + int error = -errno; if (error != -EINPROGRESS) { ::close(from->s); from->state.state = STATE_ERROR; @@ -463,11 +461,8 @@ create_nonblocking_socket() int set_nonblocking_socket(int s) { - int error = 0; - int on = 1; - if (fcntl(s, F_SETFL, O_NDELAY) < 0) { - error = -errno; + int error = -errno; ::close(s); cout << "fcntl F_SETFD O_NDELAY failed (" << error << ")" << endl; return (error); @@ -484,7 +479,7 @@ set_nonblocking_socket(int s) int do_shutdown(int s, Task_t task) { - int howto, error; + int howto; switch (task) { case TASK_SHUTDOWN_OUTPUT: @@ -503,7 +498,7 @@ do_shutdown(int s, Task_t task) break; } if (shutdown(s, howto) < 0) { - error = -errno; + int error = -errno; cout << "shutdown failed (" << error << ")" << endl; return (error); } diff --git a/proxy/http/unit_tests/test_ForwardedConfig.cc b/proxy/http/unit_tests/test_ForwardedConfig.cc index 72b21d6d9ef..02f2ff7b620 100644 --- a/proxy/http/unit_tests/test_ForwardedConfig.cc +++ b/proxy/http/unit_tests/test_ForwardedConfig.cc @@ -36,7 +36,7 @@ using namespace HttpForwarded; class OptionBitSetListInit : public OptionBitSet { public: - OptionBitSetListInit(std::initializer_list il) + explicit OptionBitSetListInit(std::initializer_list il) { for (std::size_t i : il) { this->set(i); @@ -68,7 +68,7 @@ class XS std::string s; public: - XS(const char *in) : s{nextWs()} + explicit XS(const char *in) : s{nextWs()} { bool upper{true}; for (; *in; ++in) { @@ -92,7 +92,7 @@ class XS }; void -test(const char *spec, const char *reqErr, OptionBitSet bS) +test(const char *spec, const char *reqErr, const OptionBitSet &bS) { ts::LocalBufferWriter<1024> error; From 807c49a82ed08b78c24d9d1a0d56897756ebace7 Mon Sep 17 00:00:00 2001 From: Jan van Doorn Date: Wed, 24 Apr 2019 09:58:41 +0800 Subject: [PATCH 462/526] cppcheck: fixes issues found for plugins/background_fetch --- plugins/background_fetch/configs.cc | 2 +- plugins/background_fetch/headers.cc | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/background_fetch/configs.cc b/plugins/background_fetch/configs.cc index 96fd2518e22..479317b7a9a 100644 --- a/plugins/background_fetch/configs.cc +++ b/plugins/background_fetch/configs.cc @@ -127,9 +127,9 @@ BgFetchConfig::readConfig(const char *config_file) char *cfg_type = strtok_r(buffer, " ", &savePtr); char *cfg_name = nullptr; char *cfg_value = nullptr; - bool exclude = false; if (cfg_type) { + bool exclude = false; if (!strcmp(cfg_type, "exclude")) { exclude = true; } else if (strcmp(cfg_type, "include")) { diff --git a/plugins/background_fetch/headers.cc b/plugins/background_fetch/headers.cc index 4043993f3e6..4317d9da5d8 100644 --- a/plugins/background_fetch/headers.cc +++ b/plugins/background_fetch/headers.cc @@ -100,7 +100,6 @@ dump_headers(TSMBuffer bufp, TSMLoc hdr_loc) TSIOBuffer output_buffer; TSIOBufferReader reader; TSIOBufferBlock block; - const char *block_start; int64_t block_avail; output_buffer = TSIOBufferCreate(); @@ -112,7 +111,7 @@ dump_headers(TSMBuffer bufp, TSMLoc hdr_loc) /* We need to loop over all the buffer blocks, there can be more than 1 */ block = TSIOBufferReaderStart(reader); do { - block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); + const char *block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); if (block_avail > 0) { TSDebug(PLUGIN_NAME, "Headers are:\n%.*s", static_cast(block_avail), block_start); } From 962ccb3c119df0ee2f11bf510cb89d046276633a Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 24 Apr 2019 10:52:39 +0800 Subject: [PATCH 463/526] cppcheck: fixes issues found in example/thread_pool (style) The scope of the variable 'X' can be reduced. (style) Variable 'psi' is assigned a value that is never used. --- example/thread_pool/psi.c | 38 ++++++++----------- example/thread_pool/test/SDKTest/psi_server.c | 6 +-- example/thread_pool/thread.c | 11 ++---- 3 files changed, 20 insertions(+), 35 deletions(-) diff --git a/example/thread_pool/psi.c b/example/thread_pool/psi.c index 29db012b1e6..c21eefc3cd7 100644 --- a/example/thread_pool/psi.c +++ b/example/thread_pool/psi.c @@ -449,7 +449,6 @@ psi_include(TSCont contp, void *edata ATS_UNUSED) #define BUFFER_SIZE 1024 ContData *data; TSFile filep; - char buf[BUFFER_SIZE]; char inc_file[PSI_PATH_MAX_SIZE + PSI_FILENAME_MAX_SIZE]; /* We manipulate plugin continuation data from a separate thread. @@ -472,19 +471,19 @@ psi_include(TSCont contp, void *edata ATS_UNUSED) if ((filep = TSfopen(inc_file, "r")) != NULL) { TSDebug(PLUGIN_NAME, "Reading include file %s", inc_file); + char buf[BUFFER_SIZE]; while (TSfgets(filep, buf, BUFFER_SIZE) != NULL) { - TSIOBufferBlock block; - int64_t len, avail, ndone, ntodo, towrite; - char *ptr_block; + int64_t len, ndone, ntodo; len = strlen(buf); ndone = 0; ntodo = len; while (ntodo > 0) { /* TSIOBufferStart allocates more blocks if required */ - block = TSIOBufferStart(data->psi_buffer); - ptr_block = TSIOBufferBlockWriteStart(block, &avail); - towrite = MIN(ntodo, avail); + TSIOBufferBlock block = TSIOBufferStart(data->psi_buffer); + int64_t avail; + char *ptr_block = TSIOBufferBlockWriteStart(block, &avail); + int64_t towrite = MIN(ntodo, avail); memcpy(ptr_block, buf + ndone, towrite); TSIOBufferProduce(data->psi_buffer, towrite); @@ -579,8 +578,7 @@ handle_transform(TSCont contp) TSVConn output_conn; TSVIO input_vio; ContData *data; - TSIOBufferReader input_reader; - int toread, avail, psi, toconsume = 0, towrite = 0; + int toread, toconsume = 0, towrite = 0; /* Get the output (downstream) vconnection where we'll write data to. */ output_conn = TSTransformOutputVConnGet(contp); @@ -610,11 +608,12 @@ handle_transform(TSCont contp) toread = TSVIONTodoGet(input_vio); if (toread > 0) { - input_reader = TSVIOReaderGet(input_vio); - avail = TSIOBufferReaderAvail(input_reader); + TSIOBufferReader input_reader = TSVIOReaderGet(input_vio); + int avail = TSIOBufferReaderAvail(input_reader); /* There are some data available for reading. Let's parse it */ if (avail > 0) { + int psi; /* No need to parse data if there are too few bytes left to contain an include command... */ if (toread > (PSI_START_TAG_LEN + PSI_END_TAG_LEN)) { @@ -685,20 +684,13 @@ static int dump_psi(TSCont contp) { ContData *data; - int psi_output_len; - -/* TODO: This is odd, do we need to get the input_vio, but never use it ?? */ -#if 0 - TSVIO input_vio; - input_vio = TSVConnWriteVIOGet(contp); -#endif data = TSContDataGet(contp); TSAssert(data->magic == MAGIC_ALIVE); /* If script exec succeeded, copy its output to the downstream vconn */ if (data->psi_success == 1) { - psi_output_len = TSIOBufferReaderAvail(data->psi_reader); + int psi_output_len = TSIOBufferReaderAvail(data->psi_reader); if (psi_output_len > 0) { data->transform_bytes += psi_output_len; @@ -733,7 +725,6 @@ dump_psi(TSCont contp) static int transform_handler(TSCont contp, TSEvent event, void *edata ATS_UNUSED) { - TSVIO input_vio; ContData *data; int state, retval; @@ -773,6 +764,7 @@ transform_handler(TSCont contp, TSEvent event, void *edata ATS_UNUSED) return 1; } } else { + TSVIO input_vio; switch (event) { case TS_EVENT_ERROR: input_vio = TSVConnWriteVIOGet(contp); @@ -853,9 +845,8 @@ transformable(TSHttpTxn txnp) /* We are only interested in transforming "200 OK" responses with a Content-Type: text/ header and with X-Psi header */ TSMBuffer bufp; - TSMLoc hdr_loc, field_loc; + TSMLoc hdr_loc; TSHttpStatus resp_status; - const char *value; if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc)) { resp_status = TSHttpHdrStatusGet(bufp, hdr_loc); @@ -864,6 +855,7 @@ transformable(TSHttpTxn txnp) return 0; } + TSMLoc field_loc; field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1); if (field_loc == TS_NULL_MLOC) { TSError("[%s] Unable to search Content-Type field", PLUGIN_NAME); @@ -871,7 +863,7 @@ transformable(TSHttpTxn txnp) return 0; } - value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, -1, NULL); + const char *value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, -1, NULL); if ((value == NULL) || (strncasecmp(value, "text/", sizeof("text/") - 1) != 0)) { TSHandleMLocRelease(bufp, hdr_loc, field_loc); TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); diff --git a/example/thread_pool/test/SDKTest/psi_server.c b/example/thread_pool/test/SDKTest/psi_server.c index 2f4a73c5a16..71d21244ce8 100644 --- a/example/thread_pool/test/SDKTest/psi_server.c +++ b/example/thread_pool/test/SDKTest/psi_server.c @@ -134,9 +134,6 @@ TSResponsePut(void **resp_id /* return */, void *resp_buffer /* return */, int * { int i = 0; RequestInfo *rid = *((RequestInfo **)resp_id); - int psi = 0; - int len; - char psi_tag[PSI_TAG_MAX_SIZE]; /* copy the header into the response buffer */ if (!rid->done_sent_header) { @@ -163,7 +160,8 @@ TSResponsePut(void **resp_id /* return */, void *resp_buffer /* return */, int * else { if (rid->psi) { /* generate our psi tag: */ - len = sprintf(psi_tag, PSI_TAG_FORMAT, rand() % 100); + char psi_tag[PSI_TAG_MAX_SIZE]; + int len = sprintf(psi_tag, PSI_TAG_FORMAT, rand() % 100); /* hopefully enough space for our include command */ if (rid->bytes_not_sent >= len) { diff --git a/example/thread_pool/thread.c b/example/thread_pool/thread.c index 79347bcf178..f5e8b536b96 100644 --- a/example/thread_pool/thread.c +++ b/example/thread_pool/thread.c @@ -48,13 +48,10 @@ init_queue(Queue *q) void add_to_queue(Queue *q, void *data) { - Cell *new_cell; - int n; - if (data != NULL) { TSMutexLock(q->mutex); /* Init the new cell */ - new_cell = TSmalloc(sizeof(Cell)); + Cell *new_cell = TSmalloc(sizeof(Cell)); new_cell->magic = MAGIC_ALIVE; new_cell->ptr_data = data; new_cell->ptr_next = q->tail; @@ -71,7 +68,7 @@ add_to_queue(Queue *q, void *data) q->tail->ptr_prev = new_cell; q->tail = new_cell; } - n = q->nb_elem++; + int n = q->nb_elem++; TSMutexUnlock(q->mutex); if (n > MAX_JOBS_ALARM) { @@ -157,12 +154,10 @@ thread_init() void * thread_loop(void *arg ATS_UNUSED) { - Job *job_todo; - /* Infinite loop */ for (;;) { /* returns a job or NULL if no jobs to do */ - job_todo = remove_from_queue(&job_queue); + Job *job_todo = remove_from_queue(&job_queue); if (job_todo != NULL) { TSAssert(job_todo->magic == MAGIC_ALIVE); From e5fc6cb208d9adefe7306f9c90388cd42818fef2 Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Mon, 22 Apr 2019 23:48:59 -0700 Subject: [PATCH 464/526] cppcheck fix for iocore/dns --- iocore/dns/DNS.cc | 14 +++++++------- iocore/dns/P_DNSProcessor.h | 2 +- iocore/dns/SplitDNS.cc | 28 +++++++++++----------------- iocore/dns/test_I_DNS.cc | 1 + iocore/dns/test_P_DNS.cc | 5 ++++- 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index ac9d4e1f239..70a2bc592db 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -673,7 +673,7 @@ DNSHandler::try_primary_named(bool reopen) void DNSHandler::switch_named(int ndx) { - for (DNSEntry *e = entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = entries.head; e; e = static_cast(e->link.next)) { e->written_flag = false; if (e->retries < dns_retries) { ++(e->retries); // give them another chance @@ -759,7 +759,7 @@ DNSHandler::rr_failure(int ndx) Warning("connection to all DNS servers lost, retrying"); // actual retries will be done in retry_named called from mainEvent // mark any outstanding requests as not sent for later retry - for (DNSEntry *e = entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = entries.head; e; e = static_cast(e->link.next)) { e->written_flag = false; if (e->retries < dns_retries) { ++(e->retries); // give them another chance @@ -769,7 +769,7 @@ DNSHandler::rr_failure(int ndx) } } else { // move outstanding requests that were sent to this nameserver to another - for (DNSEntry *e = entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = entries.head; e; e = static_cast(e->link.next)) { if (e->which_ns == ndx) { e->written_flag = false; if (e->retries < dns_retries) { @@ -795,7 +795,7 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) DNSConnection *dnsc = nullptr; ip_text_buffer ipbuff1, ipbuff2; Ptr buf; - while ((dnsc = (DNSConnection *)triggered.dequeue())) { + while ((dnsc = static_cast(triggered.dequeue()))) { while (true) { int res; IpEndpoint from_ip; @@ -967,7 +967,7 @@ DNSHandler::mainEvent(int event, Event *e) inline static DNSEntry * get_dns(DNSHandler *h, uint16_t id) { - for (DNSEntry *e = h->entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = h->entries.head; e; e = static_cast(e->link.next)) { if (e->once_written_flag) { for (int j : e->id) { if (j == id) { @@ -986,7 +986,7 @@ get_dns(DNSHandler *h, uint16_t id) inline static DNSEntry * get_entry(DNSHandler *h, char *qname, int qtype) { - for (DNSEntry *e = h->entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = h->entries.head; e; e = static_cast(e->link.next)) { if (e->qtype == qtype) { if (is_addr_query(qtype)) { if (!strcmp(qname, e->qname)) { @@ -1027,7 +1027,7 @@ write_dns(DNSHandler *h, bool tcp_retry) if (h->in_flight < dns_max_dns_in_flight) { DNSEntry *e = h->entries.head; while (e) { - DNSEntry *n = (DNSEntry *)e->link.next; + DNSEntry *n = static_cast(e->link.next); if (!e->written_flag) { if (dns_ns_rr) { int ns_start = h->name_server; diff --git a/iocore/dns/P_DNSProcessor.h b/iocore/dns/P_DNSProcessor.h index 4eb5045a011..9ff1b4baf60 100644 --- a/iocore/dns/P_DNSProcessor.h +++ b/iocore/dns/P_DNSProcessor.h @@ -66,7 +66,7 @@ extern unsigned int dns_sequence_number; #define DNS_SEQUENCE_NUMBER_RESTART_OFFSET 4000 #define DNS_PRIMARY_RETRY_PERIOD HRTIME_SECONDS(5) #define DNS_PRIMARY_REOPEN_PERIOD HRTIME_SECONDS(60) -#define BAD_DNS_RESULT ((HostEnt *)(uintptr_t)-1) +#define BAD_DNS_RESULT (reinterpret_cast((uintptr_t)-1)) #define DEFAULT_NUM_TRY_SERVER 8 // these are from nameser.h diff --git a/iocore/dns/SplitDNS.cc b/iocore/dns/SplitDNS.cc index 11bef367de9..92240602a50 100644 --- a/iocore/dns/SplitDNS.cc +++ b/iocore/dns/SplitDNS.cc @@ -94,7 +94,7 @@ SplitDNS::~SplitDNS() SplitDNS * SplitDNSConfig::acquire() { - return (SplitDNS *)configProcessor.get(SplitDNSConfig::m_id); + return static_cast(configProcessor.get(SplitDNSConfig::m_id)); } /* -------------------------------------------------------------- @@ -255,7 +255,7 @@ SplitDNS::findServer(RequestData *rdata, SplitDNSResult *result) int res = memcmp(pH, pMatch, pxHL[i].match.size()); if ((0 != res && '!' == cNot) || (0 == res && '!' != cNot)) { - data_ptr = (SplitDNSRecord *)pxHL[i].opaque_data; + data_ptr = static_cast(pxHL[i].opaque_data); data_ptr->UpdateMatch(result, rdata); break; } @@ -298,9 +298,7 @@ SplitDNSRecord::ProcessDNSHosts(char *val) { Tokenizer pTok(",; \t\r"); int numTok; - const char *current; - int port = 0; - char *tmp; + int port = 0; int totsz = 0, sz = 0; numTok = pTok.Initialize(val, SHARE_TOKS); @@ -317,8 +315,8 @@ SplitDNSRecord::ProcessDNSHosts(char *val) set of servers specified ------------------------------------------------ */ for (int i = 0; i < numTok; i++) { - current = pTok[i]; - tmp = (char *)strchr(current, ':'); + const char *current = pTok[i]; + char *tmp = (char *)strchr(current, ':'); // coverity[secure_coding] if (tmp != nullptr && sscanf(tmp + 1, "%d", &port) != 1) { return "Malformed DNS port"; @@ -402,9 +400,8 @@ SplitDNSRecord::ProcessDomainSrchList(char *val) { Tokenizer pTok(",; \t\r"); int numTok; - int cnt = 0, sz = 0; + int sz = 0; char *pSp = nullptr; - const char *current; numTok = pTok.Initialize(val, SHARE_TOKS); @@ -415,8 +412,8 @@ SplitDNSRecord::ProcessDomainSrchList(char *val) pSp = &m_servers.x_domain_srch_list[0]; for (int i = 0; i < numTok; i++) { - current = pTok[i]; - cnt = sz += strlen(current); + const char *current = pTok[i]; + int cnt = sz += strlen(current); if (MAXDNAME - 1 < sz) { break; @@ -440,14 +437,11 @@ Result SplitDNSRecord::Init(matcher_line *line_info) { const char *errPtr = nullptr; - const char *tmp; - char *label; - char *val; this->line_num = line_info->line_num; for (int i = 0; i < MATCHER_MAX_TOKENS; i++) { - label = line_info->line[0][i]; - val = line_info->line[1][i]; + char *label = line_info->line[0][i]; + char *val = line_info->line[1][i]; if (label == nullptr) { continue; @@ -509,7 +503,7 @@ SplitDNSRecord::Init(matcher_line *line_info) Process any modifiers to the directive, if they exist ----------------------------------------------------- */ if (line_info->num_el > 0) { - tmp = ProcessModifiers(line_info); + const char *tmp = ProcessModifiers(line_info); if (tmp != nullptr) { return Result::failure("%s %s at line %d in splitdns.config", modulePrefix, tmp, line_num); } diff --git a/iocore/dns/test_I_DNS.cc b/iocore/dns/test_I_DNS.cc index 87fd8f99b5c..98331bcf7fc 100644 --- a/iocore/dns/test_I_DNS.cc +++ b/iocore/dns/test_I_DNS.cc @@ -26,6 +26,7 @@ #include "diags.i" +int main() { init_diags("net_test", nullptr); diff --git a/iocore/dns/test_P_DNS.cc b/iocore/dns/test_P_DNS.cc index 1c0bee3622e..fecca6d0c8f 100644 --- a/iocore/dns/test_P_DNS.cc +++ b/iocore/dns/test_P_DNS.cc @@ -75,4 +75,7 @@ struct NetTesterSM : public Continuation { } }; -main() {} +int +main() +{ +} From e4398187f1e058f7d45a0923769aa9cb6e140982 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 23 Apr 2019 10:32:32 +0800 Subject: [PATCH 465/526] Removes priorities for AIOs, thanks oknet It also renames the http_aio_todo queue to just aio_todo. This was all dead code, that could not be triggered. --- include/tscore/ink_aiocb.h | 11 +++------ iocore/aio/AIO.cc | 42 ++++++++-------------------------- iocore/aio/I_AIO.h | 4 ---- iocore/aio/P_AIO.h | 15 ++++-------- iocore/cache/Cache.cc | 14 ------------ iocore/cache/CacheDisk.cc | 7 +++--- iocore/cache/I_Cache.h | 10 ++++---- iocore/cache/P_CacheInternal.h | 7 ++---- 8 files changed, 26 insertions(+), 84 deletions(-) diff --git a/include/tscore/ink_aiocb.h b/include/tscore/ink_aiocb.h index ac50fbd9cd0..3a7b30e7505 100644 --- a/include/tscore/ink_aiocb.h +++ b/include/tscore/ink_aiocb.h @@ -46,14 +46,9 @@ struct ink_aiocb { void *aio_buf; /* buffer location */ #endif size_t aio_nbytes; /* length of transfer */ + off_t aio_offset; /* file offset */ - // TODO change to off_t - off_t aio_offset; /* file offset */ - - int aio_reqprio; /* request priority offset */ - // struct sigevent aio_sigevent; /* signal number and offset */ int aio_lio_opcode; /* listio operation */ - // aio_result_t aio_resultp; /* results */ - int aio_state; /* state flag for List I/O */ - int aio__pad[1]; /* extension padding */ + int aio_state; /* state flag for List I/O */ + int aio__pad[1]; /* extension padding */ }; diff --git a/iocore/aio/AIO.cc b/iocore/aio/AIO.cc index 29deaf65e87..e908542668b 100644 --- a/iocore/aio/AIO.cc +++ b/iocore/aio/AIO.cc @@ -206,16 +206,12 @@ struct AIOThreadInfo : public Continuation { } }; -/* priority scheduling */ -/* Have 2 queues per file descriptor - A queue for http requests and another - for non-http (streaming) request. Each file descriptor has a lock - and condition variable associated with it. A dedicated number of threads - (THREADS_PER_DISK) wait on the condition variable associated with the - file descriptor. The cache threads try to put the request in the - appropriate queue. If they fail to acquire the lock, they put the - request in the atomic list. Requests are served in the order of - highest priority first. If both the queues are empty, the aio threads - check if there is any request on the other disks */ +/* + A dedicated number of threads (THREADS_PER_DISK) wait on the condition + variable associated with the file descriptor. The cache threads try to put + the request in the appropriate queue. If they fail to acquire the lock, they + put the request in the atomic list. + */ /* insert an entry for file descriptor fildes into aio_reqs */ static AIO_Reqs * @@ -268,8 +264,7 @@ aio_init_fildes(int fildes, int fromAPI = 0) return request; } -/* insert a request into either aio_todo or http_todo queue. aio_todo - list is kept sorted */ +/* insert a request into aio_todo queue. */ static void aio_insert(AIOCallback *op, AIO_Reqs *req) { @@ -277,22 +272,7 @@ aio_insert(AIOCallback *op, AIO_Reqs *req) num_requests++; req->queued++; #endif - if (op->aiocb.aio_reqprio == AIO_LOWEST_PRIORITY) // http request - { - req->http_aio_todo.enqueue(op); - } else { - AIOCallback *cb = (AIOCallback *)req->aio_todo.tail; - - for (; cb; cb = (AIOCallback *)cb->link.prev) { - if (cb->aiocb.aio_reqprio >= op->aiocb.aio_reqprio) { - req->aio_todo.insert(op, cb); - return; - } - } - - /* Either the queue was empty or this request has the highest priority */ - req->aio_todo.push(op); - } + req->aio_todo.enqueue(op); } /* move the request from the atomic list to the queue */ @@ -466,7 +446,7 @@ aio_thread_main(void *arg) current_req = my_aio_req; /* check if any pending requests on the atomic list */ aio_move(my_aio_req); - if (!(op = my_aio_req->aio_todo.pop()) && !(op = my_aio_req->http_aio_todo.pop())) { + if (!(op = my_aio_req->aio_todo.pop())) { break; } #ifdef AIO_STATS @@ -578,7 +558,6 @@ DiskHandler::mainAIOEvent(int event, Event *e) int ink_aio_read(AIOCallback *op, int /* fromAPI ATS_UNUSED */) { - op->aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; op->aiocb.aio_lio_opcode = IO_CMD_PREAD; op->aiocb.data = op; EThread *t = this_ethread(); @@ -593,7 +572,6 @@ ink_aio_read(AIOCallback *op, int /* fromAPI ATS_UNUSED */) int ink_aio_write(AIOCallback *op, int /* fromAPI ATS_UNUSED */) { - op->aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; op->aiocb.aio_lio_opcode = IO_CMD_PWRITE; op->aiocb.data = op; EThread *t = this_ethread(); @@ -614,7 +592,6 @@ ink_aio_readv(AIOCallback *op, int /* fromAPI ATS_UNUSED */) int sz = 0; while (io) { - io->aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; io->aiocb.aio_lio_opcode = IO_CMD_PREAD; io->aiocb.data = io; #ifdef HAVE_EVENTFD @@ -645,7 +622,6 @@ ink_aio_writev(AIOCallback *op, int /* fromAPI ATS_UNUSED */) int sz = 0; while (io) { - io->aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; io->aiocb.aio_lio_opcode = IO_CMD_PWRITE; io->aiocb.data = io; #ifdef HAVE_EVENTFD diff --git a/iocore/aio/I_AIO.h b/iocore/aio/I_AIO.h index 6b957b6a0fa..8618e59da37 100644 --- a/iocore/aio/I_AIO.h +++ b/iocore/aio/I_AIO.h @@ -72,7 +72,6 @@ struct ink_aiocb { size_t aio_nbytes = 0; /* length of transfer */ off_t aio_offset = 0; /* file offset */ - int aio_reqprio = 0; /* request priority offset */ int aio_lio_opcode = 0; /* listio operation */ int aio_state = 0; /* state flag for List I/O */ int aio__pad[1]; /* extension padding */ @@ -86,9 +85,6 @@ bool ink_aio_thread_num_set(int thread_num); #define AIO_CALLBACK_THREAD_ANY ((EThread *)0) // any regular event thread #define AIO_CALLBACK_THREAD_AIO ((EThread *)-1) -#define AIO_LOWEST_PRIORITY 0 -#define AIO_DEFAULT_PRIORITY AIO_LOWEST_PRIORITY - struct AIOCallback : public Continuation { // set before calling aio_read/aio_write ink_aiocb aiocb; diff --git a/iocore/aio/P_AIO.h b/iocore/aio/P_AIO.h index 06eb15b8fb1..95534c9821b 100644 --- a/iocore/aio/P_AIO.h +++ b/iocore/aio/P_AIO.h @@ -85,24 +85,19 @@ struct AIOCallbackInternal : public AIOCallback { int io_complete(int event, void *data); - AIOCallbackInternal() - { - aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; - SET_HANDLER(&AIOCallbackInternal::io_complete); - } + AIOCallbackInternal() { SET_HANDLER(&AIOCallbackInternal::io_complete); } }; struct AIO_Reqs { - Que(AIOCallback, link) aio_todo; /* queue for holding non-http requests */ - Que(AIOCallback, link) http_aio_todo; /* queue for http requests */ - /* Atomic list to temporarily hold the request if the - lock for a particular queue cannot be acquired */ + Que(AIOCallback, link) aio_todo; /* queue for AIO operations */ + /* Atomic list to temporarily hold the request if the + lock for a particular queue cannot be acquired */ ASLL(AIOCallbackInternal, alink) aio_temp_list; ink_mutex aio_mutex; ink_cond aio_cond; int index = 0; /* position of this struct in the aio_reqs array */ int pending = 0; /* number of outstanding requests on the disk */ - int queued = 0; /* total number of aio_todo and http_todo requests */ + int queued = 0; /* total number of aio_todo requests */ int filedes = 0; /* the file descriptor for the requests */ int requests_queued = 0; }; diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 9d986076c83..8b6ab8d8c30 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -482,26 +482,12 @@ CacheVC::set_pin_in_cache(time_t time_pin) return true; } -bool -CacheVC::set_disk_io_priority(int priority) -{ - ink_assert(priority >= AIO_LOWEST_PRIORITY); - io.aiocb.aio_reqprio = priority; - return true; -} - time_t CacheVC::get_pin_in_cache() { return pin_in_cache; } -int -CacheVC::get_disk_io_priority() -{ - return io.aiocb.aio_reqprio; -} - int Vol::begin_read(CacheVC *cont) { diff --git a/iocore/cache/CacheDisk.cc b/iocore/cache/CacheDisk.cc index 78beffe0310..8e98e309092 100644 --- a/iocore/cache/CacheDisk.cc +++ b/iocore/cache/CacheDisk.cc @@ -61,10 +61,9 @@ CacheDisk::open(char *s, off_t blocks, off_t askip, int ahw_sector_size, int fil skip = askip; start = skip; /* we can't use fractions of store blocks. */ - len = blocks; - io.aiocb.aio_fildes = fd; - io.aiocb.aio_reqprio = 0; - io.action = this; + len = blocks; + io.aiocb.aio_fildes = fd; + io.action = this; // determine header size and hence start point by successive approximation uint64_t l; for (int i = 0; i < 3; i++) { diff --git a/iocore/cache/I_Cache.h b/iocore/cache/I_Cache.h index 8d4447b6f6e..b0fd5441ef4 100644 --- a/iocore/cache/I_Cache.h +++ b/iocore/cache/I_Cache.h @@ -189,12 +189,10 @@ struct CacheVConnection : public VConnection { virtual void set_http_info(CacheHTTPInfo *info) = 0; virtual void get_http_info(CacheHTTPInfo **info) = 0; - virtual bool is_ram_cache_hit() const = 0; - virtual bool set_disk_io_priority(int priority) = 0; - virtual int get_disk_io_priority() = 0; - virtual bool set_pin_in_cache(time_t t) = 0; - virtual time_t get_pin_in_cache() = 0; - virtual int64_t get_object_size() = 0; + virtual bool is_ram_cache_hit() const = 0; + virtual bool set_pin_in_cache(time_t t) = 0; + virtual time_t get_pin_in_cache() = 0; + virtual int64_t get_object_size() = 0; virtual bool is_compressed_in_ram() const { diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h index 54d531485b8..7ba0e240ca4 100644 --- a/iocore/cache/P_CacheInternal.h +++ b/iocore/cache/P_CacheInternal.h @@ -388,8 +388,6 @@ struct CacheVC : public CacheVConnection { bool is_pread_capable() override; bool set_pin_in_cache(time_t time_pin) override; time_t get_pin_in_cache() override; - bool set_disk_io_priority(int priority) override; - int get_disk_io_priority() override; // offsets from the base stat #define CACHE_STAT_ACTIVE 0 @@ -585,9 +583,8 @@ free_CacheVC(CacheVC *cont) cont->io.action.continuation = nullptr; cont->io.action.mutex = nullptr; cont->io.mutex.clear(); - cont->io.aio_result = 0; - cont->io.aiocb.aio_nbytes = 0; - cont->io.aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; + cont->io.aio_result = 0; + cont->io.aiocb.aio_nbytes = 0; cont->request.reset(); cont->vector.clear(); cont->vio.buffer.clear(); From 5d8b02aa6626ebef5bee91b089e48168bbcce35b Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Mon, 22 Apr 2019 18:22:58 -0700 Subject: [PATCH 466/526] correct config name for proxy.config.dns.connection_mode --- iocore/dns/DNS.cc | 2 +- mgmt/RecordsConfig.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 70a2bc592db..41173551891 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -221,7 +221,7 @@ DNSProcessor::start(int, size_t stacksize) REC_ReadConfigStringAlloc(dns_resolv_conf, "proxy.config.dns.resolv_conf"); REC_EstablishStaticConfigInt32(dns_thread, "proxy.config.dns.dedicated_thread"); int dns_conn_mode_i = 0; - REC_EstablishStaticConfigInt32(dns_conn_mode_i, "proxy.config.dns.connection.mode"); + REC_EstablishStaticConfigInt32(dns_conn_mode_i, "proxy.config.dns.connection_mode"); dns_conn_mode = static_cast(dns_conn_mode_i); if (dns_thread > 0) { diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 3e93abf2c62..3003477c03c 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -909,7 +909,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.dns.dedicated_thread", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-1]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.dns.connection.mode", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-2]", RECA_NULL} + {RECT_CONFIG, "proxy.config.dns.connection_mode", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-2]", RECA_NULL} , {RECT_CONFIG, "proxy.config.hostdb.ip_resolve", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , From 2a3f99395ce3b47e511e1b1a5968c190ccf7e233 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 23 Apr 2019 16:59:59 +0800 Subject: [PATCH 467/526] Fixes clang-analyzer error dereferencing nullptr in parent This also eliminates superflous assignments of parent. --- proxy/http/Http1ClientTransaction.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/proxy/http/Http1ClientTransaction.cc b/proxy/http/Http1ClientTransaction.cc index 983ee029c99..d8a78abbfe5 100644 --- a/proxy/http/Http1ClientTransaction.cc +++ b/proxy/http/Http1ClientTransaction.cc @@ -49,15 +49,17 @@ Http1ClientTransaction::release(IOBufferReader *r) void Http1ClientTransaction::set_parent(ProxyClientSession *new_parent) { - parent = new_parent; Http1ClientSession *http1_parent = dynamic_cast(new_parent); + if (http1_parent) { outbound_port = http1_parent->outbound_port; outbound_ip4 = http1_parent->outbound_ip4; outbound_ip6 = http1_parent->outbound_ip6; outbound_transparent = http1_parent->f_outbound_transparent; + super_type::set_parent(new_parent); + } else { + parent = nullptr; } - super_type::set_parent(new_parent); } void From 4b0d979aeb442b0a124e07cb9dfc1b6adb560cf3 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Wed, 24 Apr 2019 13:58:36 +0800 Subject: [PATCH 468/526] cppcheck: minimize variable scopes --- iocore/net/P_UDPNet.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index 99ca6150a01..e54ba6c0b16 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -167,12 +167,12 @@ class PacketQueue void FreeCancelledPackets(int numSlots) { - UDPPacketInternal *p; Queue tempQ; - int i, s; + int i; for (i = 0; i < numSlots; i++) { - s = (now_slot + i) % N_SLOTS; + int s = (now_slot + i) % N_SLOTS; + UDPPacketInternal *p; while (nullptr != (p = bucket[s].dequeue())) { if (IsCancelledPacket(p)) { p->free(); @@ -191,7 +191,6 @@ class PacketQueue advanceNow(ink_hrtime t) { int s = now_slot; - int prev; if (ink_hrtime_to_msec(t - lastPullLongTermQ) >= SLOT_TIME_MSEC * ((N_SLOTS - 1) / 2)) { Queue tempQ; @@ -208,6 +207,8 @@ class PacketQueue } while (!bucket[s].head && (t > delivery_time[s] + SLOT_TIME)) { + int prev; + prev = (s + N_SLOTS - 1) % N_SLOTS; delivery_time[s] = delivery_time[prev] + SLOT_TIME; s = (s + 1) % N_SLOTS; From 09b02b49991dd6675a151703c2c85da21661737b Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Wed, 24 Apr 2019 13:44:44 +0800 Subject: [PATCH 469/526] cppcheck: Remove an unused private function --- iocore/net/P_UDPNet.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index e54ba6c0b16..9368cbba5b2 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -270,12 +270,6 @@ class PacketQueue } return HRTIME_FOREVER; } - -private: - void - kill_cancelled_events() - { - } }; class UDPQueue From 991df532eb42d70571056120abb2a5818945cacd Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 23 Apr 2019 16:42:28 +0800 Subject: [PATCH 470/526] cppcheck: fixes issue found in proxy/IPAllow.cc Variable 'config_file_path' is assigned in constructor body. Consider performing initialization in initialization list. --- proxy/IPAllow.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/proxy/IPAllow.cc b/proxy/IPAllow.cc index 3419258280a..8adadb7a3e4 100644 --- a/proxy/IPAllow.cc +++ b/proxy/IPAllow.cc @@ -133,10 +133,7 @@ IpAllow::match(sockaddr const *ip, match_key_t key) // End API functions // -IpAllow::IpAllow(const char *config_var) -{ - config_file_path = RecConfigReadConfigPath(config_var); -} +IpAllow::IpAllow(const char *config_var) : config_file_path(RecConfigReadConfigPath(config_var)) {} void IpAllow::PrintMap(IpMap *map) From a9267dc953de0246ca5f302418d9b8ac1c0e92da Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 24 Apr 2019 14:51:31 +0800 Subject: [PATCH 471/526] cppcheck: fixes issues fround in example/remap (style) The scope of the variable 'X' can be reduced. (style) C-style pointer casting (style) Local variable len shadows outer variable --- example/remap/remap.cc | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/example/remap/remap.cc b/example/remap/remap.cc index 359e03867fa..c60b1a59391 100644 --- a/example/remap/remap.cc +++ b/example/remap/remap.cc @@ -66,9 +66,8 @@ pthread_mutex_t remap_entry::mutex; /* remap_entry class mutex */ /* ----------------------- remap_entry::remap_entry ------------------------ */ remap_entry::remap_entry(int _argc, char *_argv[]) : next(nullptr), argc(0), argv(nullptr) { - int i; - if (_argc > 0 && _argv && (argv = (char **)TSmalloc(sizeof(char *) * (_argc + 1))) != nullptr) { + int i; argc = _argc; for (i = 0; i < argc; i++) { argv[i] = TSstrdup(_argv[i]); @@ -80,10 +79,8 @@ remap_entry::remap_entry(int _argc, char *_argv[]) : next(nullptr), argc(0), arg /* ---------------------- remap_entry::~remap_entry ------------------------ */ remap_entry::~remap_entry() { - int i; - if (argc && argv) { - for (i = 0; i < argc; i++) { + for (int i = 0; i < argc; i++) { TSfree(argv[i]); } TSfree(argv); @@ -106,10 +103,9 @@ remap_entry::add_to_list(remap_entry *re) void remap_entry::remove_from_list(remap_entry *re) { - remap_entry **rre; if (likely(re && plugin_init_counter)) { pthread_mutex_lock(&mutex); - for (rre = &active_list; *rre; rre = &((*rre)->next)) { + for (remap_entry **rre = &active_list; *rre; rre = &((*rre)->next)) { if (re == *rre) { *rre = re->next; break; @@ -203,7 +199,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s void TSRemapDeleteInstance(void *ih) { - remap_entry *ri = (remap_entry *)ih; + remap_entry *ri = static_cast(ih); TSDebug(PLUGIN_NAME, "enter"); @@ -225,7 +221,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) TSMLoc cfield; uint64_t _processing_counter = processing_counter++; - remap_entry *ri = (remap_entry *)ih; + remap_entry *ri = static_cast(ih); TSDebug(PLUGIN_NAME, "enter"); if (!ri || !rri) { @@ -281,8 +277,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) char *tmp = (char *)TSmalloc(256); static int my_local_counter = 0; - size_t len = snprintf(tmp, 255, "This is very small example of TS API usage!\nIteration %d!\nHTTP return code %d\n", - my_local_counter, TS_HTTP_STATUS_CONTINUE + my_local_counter); + len = snprintf(tmp, 255, "This is very small example of TS API usage!\nIteration %d!\nHTTP return code %d\n", my_local_counter, + TS_HTTP_STATUS_CONTINUE + my_local_counter); TSHttpTxnStatusSet((TSHttpTxn)rh, (TSHttpStatus)((int)TS_HTTP_STATUS_CONTINUE + my_local_counter)); TSHttpTxnErrorBodySet((TSHttpTxn)rh, tmp, len, nullptr); // Defaults to text/html my_local_counter++; From 179306d6890234101a3e3be2b7f0262bb3b7a60b Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 24 Apr 2019 14:46:27 +0800 Subject: [PATCH 472/526] cppcheck: fixes issues found in example/protocol (style) The scope of the variable 'X' can be reduced. (warning) Assignment of function parameter has no effect outside the function. Did you forget dereferencing it? --- example/protocol/TxnSM.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/example/protocol/TxnSM.c b/example/protocol/TxnSM.c index 10d3724cad6..144c501e3c1 100644 --- a/example/protocol/TxnSM.c +++ b/example/protocol/TxnSM.c @@ -209,8 +209,7 @@ state_interface_with_client(TSCont contp, TSEvent event, TSVIO vio) int state_read_request_from_client(TSCont contp, TSEvent event, TSVIO vio ATS_UNUSED) { - int bytes_read, parse_result; - char *temp_buf; + int bytes_read; TxnSM *txn_sm = (TxnSM *)TSContDataGet(contp); @@ -221,7 +220,7 @@ state_read_request_from_client(TSCont contp, TSEvent event, TSVIO vio ATS_UNUSED bytes_read = TSIOBufferReaderAvail(txn_sm->q_client_request_buffer_reader); if (bytes_read > 0) { - temp_buf = (char *)get_info_from_buffer(txn_sm->q_client_request_buffer_reader); + char *temp_buf = get_info_from_buffer(txn_sm->q_client_request_buffer_reader); TSstrlcat(txn_sm->q_client_request, temp_buf, MAX_REQUEST_LENGTH + 1); TSfree(temp_buf); @@ -230,7 +229,8 @@ state_read_request_from_client(TSCont contp, TSEvent event, TSVIO vio ATS_UNUSED temp_buf = (char *)TSmalloc(sizeof(char) * (strlen(txn_sm->q_client_request) + 1)); memcpy(temp_buf, txn_sm->q_client_request, strlen(txn_sm->q_client_request)); temp_buf[strlen(txn_sm->q_client_request)] = '\0'; - parse_result = parse_request(temp_buf, txn_sm->q_server_name, txn_sm->q_file_name); + + int parse_result = parse_request(temp_buf, txn_sm->q_server_name, txn_sm->q_file_name); TSfree(temp_buf); if (parse_result != 1) { @@ -546,8 +546,6 @@ state_send_request_to_server(TSCont contp, TSEvent event, TSVIO vio) TSVIOReenable(vio); break; case TS_EVENT_VCONN_WRITE_COMPLETE: - vio = NULL; - /* Waiting for the incoming response. */ set_handler(txn_sm->q_current_handler, (TxnSMHandler)&state_interface_with_server); txn_sm->q_server_read_vio = TSVConnRead(txn_sm->q_server_vc, contp, txn_sm->q_server_response_buffer, INT64_MAX); @@ -923,8 +921,6 @@ get_info_from_buffer(TSIOBufferReader the_reader) char *info_start; int64_t read_avail, read_done; - TSIOBufferBlock blk; - char *buf; if (!the_reader) { return NULL; @@ -940,8 +936,8 @@ get_info_from_buffer(TSIOBufferReader the_reader) /* Read the data out of the reader */ while (read_avail > 0) { - blk = TSIOBufferReaderStart(the_reader); - buf = (char *)TSIOBufferBlockReadStart(blk, the_reader, &read_done); + TSIOBufferBlock blk = TSIOBufferReaderStart(the_reader); + char *buf = (char *)TSIOBufferBlockReadStart(blk, the_reader, &read_done); memcpy(info, buf, read_done); if (read_done > 0) { TSIOBufferReaderConsume(the_reader, read_done); From b5316c5a2ccbf39645668c41b500a5cd957ba442 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 23 Apr 2019 16:59:34 +0800 Subject: [PATCH 473/526] cppcheck: Fix various issues of Http2ConnectionState Use C++ style pointer casting > [Http2ConnectionState.cc:887]: (style) C-style pointer casting > [Http2ConnectionState.cc:933]: (style) C-style pointer casting Reduce scope > [Http2ConnectionState.cc:1144]: (style) The scope of the variable 'starting_point' can be reduced. Change variable names > [Http2ConnectionState.cc:1523] -> [Http2ConnectionState.cc:1555]: (style) Local variable headers shadows outer variable > [Http2ConnectionState.cc:1632] -> [Http2ConnectionState.cc:1652]: (style) Local variable headers shadows outer variable --- proxy/http2/Http2ConnectionState.cc | 51 +++++++++++++++-------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 9db9cafad4e..afc1d2488e4 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -884,7 +884,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Initialize HTTP/2 Connection case HTTP2_SESSION_EVENT_INIT: { ink_assert(this->ua_session == nullptr); - this->ua_session = (Http2ClientSession *)edata; + this->ua_session = static_cast(edata); REMEMBER(event, this->recursion); // [RFC 7540] 3.5. HTTP/2 Connection Preface. Upon establishment of a TCP connection and @@ -930,7 +930,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Parse received HTTP/2 frames case HTTP2_SESSION_EVENT_RECV: { REMEMBER(event, this->recursion); - const Http2Frame *frame = (Http2Frame *)edata; + const Http2Frame *frame = static_cast(edata); const Http2StreamId stream_id = frame->header().streamid; Http2Error error; @@ -1138,14 +1138,15 @@ Http2ConnectionState::find_stream(Http2StreamId id) const void Http2ConnectionState::restart_streams() { - // This is a static variable, so it is shared in Http2ConnectionState instances and will get incremented in subsequent calls. - // It doesn't need to be initialized with rand() nor time(), and doesn't need to be accessed with a lock, because it doesn't need - // that randomness and accuracy. - static uint16_t starting_point = 0; - - Http2Stream *s = stream_list.head; - Http2Stream *end = s; + Http2Stream *s = stream_list.head; if (s) { + Http2Stream *end = s; + + // This is a static variable, so it is shared in Http2ConnectionState instances and will get incremented in subsequent calls. + // It doesn't need to be initialized with rand() nor time(), and doesn't need to be accessed with a lock, because it doesn't + // need that randomness and accuracy. + static uint16_t starting_point = 0; + // Change the start point randomly for (int i = starting_point % total_client_streams_count; i >= 0; --i) { end = static_cast(end->link.next ? end->link.next : stream_list.head); @@ -1552,14 +1553,14 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) if (sent + payload_length == header_blocks_size) { flags |= HTTP2_FLAGS_CONTINUATION_END_HEADERS; } - Http2Frame headers(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); - headers.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); - http2_write_headers(buf + sent, payload_length, headers.write()); - headers.finalize(payload_length); - stream->change_state(headers.header().type, headers.header().flags); + Http2Frame continuation_frame(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); + continuation_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); + http2_write_headers(buf + sent, payload_length, continuation_frame.write()); + continuation_frame.finalize(payload_length); + stream->change_state(continuation_frame.header().type, continuation_frame.header().flags); // xmit event SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &headers); + this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &continuation_frame); sent += payload_length; } @@ -1629,15 +1630,15 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con payload_length = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_PUSH_PROMISE]) - sizeof(push_promise.promised_streamid); } - Http2Frame headers(HTTP2_FRAME_TYPE_PUSH_PROMISE, stream->get_id(), flags); - headers.alloc(buffer_size_index[HTTP2_FRAME_TYPE_PUSH_PROMISE]); + Http2Frame push_promise_frame(HTTP2_FRAME_TYPE_PUSH_PROMISE, stream->get_id(), flags); + push_promise_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_PUSH_PROMISE]); Http2StreamId id = this->get_latest_stream_id_out() + 2; push_promise.promised_streamid = id; - http2_write_push_promise(push_promise, buf, payload_length, headers.write()); - headers.finalize(sizeof(push_promise.promised_streamid) + payload_length); + http2_write_push_promise(push_promise, buf, payload_length, push_promise_frame.write()); + push_promise_frame.finalize(sizeof(push_promise.promised_streamid) + payload_length); // xmit event SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &headers); + this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &push_promise_frame); sent += payload_length; // Send CONTINUATION frames @@ -1649,13 +1650,13 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con if (sent + payload_length == header_blocks_size) { flags |= HTTP2_FLAGS_CONTINUATION_END_HEADERS; } - Http2Frame headers(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); - headers.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); - http2_write_headers(buf + sent, payload_length, headers.write()); - headers.finalize(payload_length); + Http2Frame continuation_frame(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); + continuation_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); + http2_write_headers(buf + sent, payload_length, continuation_frame.write()); + continuation_frame.finalize(payload_length); // xmit event SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &headers); + this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &continuation_frame); sent += payload_length; } ats_free(buf); From 6cb506a4873f1c56917982a48e42e85e53d15ca0 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 23 Apr 2019 15:36:21 +0800 Subject: [PATCH 474/526] cppcheck: Fix various issues of Http2DependencyTree Fix argument order on declarations > [Http2DependencyTree.h:122] -> [Http2DependencyTree.h:393]: (warning) Function 'reprioritize' argument order different: declaration 'new_parent_id, id, exclusive' definition 'id, new_parent_id, exclusive' > [Http2DependencyTree.h:146] -> [Http2DependencyTree.h:454]: (warning) Function '_change_parent' argument order different: declaration 'new_parent, node, exclusive' definition 'node, new_parent, exclusive' Add explicit specifier > [Http2DependencyTree.h:44]: (style) Class 'Node' has a constructor with 1 argument that is not explicit. > [Http2DependencyTree.h:114]: (style) Class 'Tree < Http2Stream * >' has a constructor with 1 argument that is not explicit. Delete operator= and copy constructor > [Http2DependencyTree.h:52]: (style) Class 'Node' does not have a operator= which is recommended since it has dynamic memory/resource allocation(s). > [Http2DependencyTree.h:52]: (style) Class 'Node' does not have a copy constructor which is recommended since it has dynamic memory/resource allocation(s). --- proxy/http2/Http2DependencyTree.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/proxy/http2/Http2DependencyTree.h b/proxy/http2/Http2DependencyTree.h index 60f2b59f5bb..ec12acb17cf 100644 --- a/proxy/http2/Http2DependencyTree.h +++ b/proxy/http2/Http2DependencyTree.h @@ -41,7 +41,7 @@ namespace Http2DependencyTree class Node { public: - Node(void *t = nullptr) : t(t) + explicit Node(void *t = nullptr) : t(t) { entry = new PriorityQueueEntry(this); queue = new PriorityQueue(); @@ -71,6 +71,9 @@ class Node } } + Node(const Node &) = delete; + Node &operator=(const Node &) = delete; + LINK(Node, link); bool @@ -78,6 +81,7 @@ class Node { return point < n.point; } + bool operator>(const Node &n) const { @@ -111,7 +115,7 @@ class Node template class Tree { public: - Tree(uint32_t max_concurrent_streams) : _max_depth(MIN(max_concurrent_streams, HTTP2_DEPENDENCY_TREE_MAX_DEPTH)) + explicit Tree(uint32_t max_concurrent_streams) : _max_depth(MIN(max_concurrent_streams, HTTP2_DEPENDENCY_TREE_MAX_DEPTH)) { _ancestors.resize(_max_ancestors); } @@ -119,7 +123,7 @@ template class Tree Node *find(uint32_t id, bool *is_max_leaf = nullptr); Node *find_shadow(uint32_t id, bool *is_max_leaf = nullptr); Node *add(uint32_t parent_id, uint32_t id, uint32_t weight, bool exclusive, T t, bool shadow = false); - Node *reprioritize(uint32_t new_parent_id, uint32_t id, bool exclusive); + Node *reprioritize(uint32_t id, uint32_t new_parent_id, bool exclusive); Node *reprioritize(Node *node, uint32_t id, bool exclusive); Node *top(); void remove(Node *node); @@ -143,7 +147,7 @@ template class Tree void _dump(Node *node, std::ostream &output) const; Node *_find(Node *node, uint32_t id, uint32_t depth = 1, bool *is_max_leaf = nullptr); Node *_top(Node *node); - void _change_parent(Node *new_parent, Node *node, bool exclusive); + void _change_parent(Node *node, Node *new_parent, bool exclusive); bool in_parent_chain(Node *maybe_parent, Node *target); Node *_root = new Node(this); From b129f2b3385d63c621fcf41fbb6c7a015159fa98 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 23 Apr 2019 14:49:40 +0800 Subject: [PATCH 475/526] cppcheck: change to C++ style pointer casting --- proxy/http2/Http2ClientSession.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index a48ad3d260c..2bfd121cc08 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -338,7 +338,7 @@ Http2ClientSession::main_event_handler(int event, void *edata) } case HTTP2_SESSION_EVENT_XMIT: { - Http2Frame *frame = (Http2Frame *)edata; + Http2Frame *frame = static_cast(edata); total_write_len += frame->size(); write_vio->nbytes = total_write_len; frame->xmit(this->write_buffer); From 72311cd7e41477fba0d90a10e3f2d28ca3e66d2d Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Tue, 23 Apr 2019 14:57:50 +0800 Subject: [PATCH 476/526] cppcheck: Reduce the scope of the variable 'out_buf' --- proxy/http2/Http2ClientSession.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index 2bfd121cc08..eb644e1c1a4 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -213,9 +213,9 @@ Http2ClientSession::set_upgrade_context(HTTPHdr *h) int svlen; const char *sv = settings->value_get(&svlen); - // Maybe size of data decoded by Base64URL is lower than size of encoded data. - unsigned char out_buf[svlen]; if (sv && svlen > 0) { + // Maybe size of data decoded by Base64URL is lower than size of encoded data. + unsigned char out_buf[svlen]; size_t decoded_len; ats_base64_decode(sv, svlen, out_buf, svlen, &decoded_len); for (size_t nbytes = 0; nbytes < decoded_len; nbytes += HTTP2_SETTINGS_PARAMETER_LEN) { From ddb9524026139002af0c7fa47b6ef97ca7447cee Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 23 Apr 2019 16:49:25 +0800 Subject: [PATCH 477/526] cppcheck: Fixes issue found in DiagsConfig.cc Member variable 'DiagsConfig::diags' is not initialized in the constructor. --- proxy/shared/DiagsConfig.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/proxy/shared/DiagsConfig.cc b/proxy/shared/DiagsConfig.cc index ebdedeb70f6..98848ea4a24 100644 --- a/proxy/shared/DiagsConfig.cc +++ b/proxy/shared/DiagsConfig.cc @@ -265,14 +265,11 @@ DiagsConfig::RegisterDiagConfig() } DiagsConfig::DiagsConfig(const char *prefix_string, const char *filename, const char *tags, const char *actions, bool use_records) - : diags_log(nullptr) + : callbacks_established(false), diags_log(nullptr), diags(nullptr) { char diags_logpath[PATH_NAME_MAX]; ats_scoped_str logpath; - callbacks_established = false; - diags = nullptr; - //////////////////////////////////////////////////////////////////// // If we aren't using the manager records for configuation // // just build the tables based on command line parameters and // From 42e31a1f091d21c106332964a0c596ed0e1055cf Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 23 Apr 2019 15:50:28 +0800 Subject: [PATCH 478/526] cppcheck: Fixes various issues under proxy/http/remap Including: Reducing variable scope Check for less than zero on unsigned variable Adding explicit constructor --- proxy/http/remap/RemapConfig.cc | 17 +++++++++-------- proxy/http/remap/RemapPluginInfo.h | 2 +- proxy/http/remap/RemapProcessor.cc | 16 ++++++++-------- proxy/http/remap/UrlMapping.cc | 7 +++---- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/proxy/http/remap/RemapConfig.cc b/proxy/http/remap/RemapConfig.cc index 6b5a5e990f1..8d5948f7796 100644 --- a/proxy/http/remap/RemapConfig.cc +++ b/proxy/http/remap/RemapConfig.cc @@ -163,7 +163,6 @@ parse_define_directive(const char *directive, BUILD_TABLE_INFO *bti, char *errbu { bool flg; acl_filter_rule *rp; - acl_filter_rule **rpp; const char *cstr = nullptr; if (bti->paramc < 2) { @@ -181,6 +180,7 @@ parse_define_directive(const char *directive, BUILD_TABLE_INFO *bti, char *errbu // coverity[alloc_arg] if ((cstr = remap_validate_filter_args(&rp, (const char **)bti->argv, bti->argc, errbuf, errbufsize)) == nullptr && rp) { if (flg) { // new filter - add to list + acl_filter_rule **rpp = nullptr; Debug("url_rewrite", "[parse_directive] new rule \"%s\" was created", bti->paramv[1]); for (rpp = &bti->rules_list; *rpp; rpp = &((*rpp)->next)) { ; @@ -400,7 +400,7 @@ remap_parse_directive(BUILD_TABLE_INFO *bti, char *errbuf, size_t errbufsize) const char *directive = nullptr; // Check arguments - if (unlikely(!bti || !errbuf || errbufsize <= 0 || !bti->paramc || (directive = bti->paramv[0]) == nullptr)) { + if (unlikely(!bti || !errbuf || errbufsize == 0 || !bti->paramc || (directive = bti->paramv[0]) == nullptr)) { Debug("url_rewrite", "[parse_directive] Invalid argument(s)"); return "Invalid argument(s)"; } @@ -420,8 +420,6 @@ const char * remap_validate_filter_args(acl_filter_rule **rule_pp, const char **argv, int argc, char *errStrBuf, size_t errStrBufSize) { acl_filter_rule *rule; - unsigned long ul; - const char *argptr; src_ip_info_t *ipi; int i, j; bool new_rule_flg = false; @@ -450,8 +448,10 @@ remap_validate_filter_args(acl_filter_rule **rule_pp, const char **argv, int arg } for (i = 0; i < argc; i++) { + unsigned long ul; bool hasarg; + const char *argptr; if ((ul = remap_check_option((const char **)&argv[i], 1, 0, nullptr, &argptr)) == 0) { Debug("url_rewrite", "[validate_filter_args] Unknown remap option - %s", argv[i]); snprintf(errStrBuf, errStrBufSize, "Unknown option - \"%s\"", argv[i]); @@ -718,8 +718,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in char *c, *err, tmpbuf[2048], default_path[PATH_NAME_MAX]; const char *new_argv[1024]; char *parv[1024]; - int idx = 0, retcode = 0; - int parc = 0; + int idx = 0; *plugin_found_at = 0; @@ -804,6 +803,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in pi->do_remap_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_DO_REMAP)); pi->os_response_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_OS_RESPONSE)); + int retcode = 0; if (!pi->init_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s")", TSREMAP_FUNCNAME_INIT, c); retcode = -10; @@ -851,6 +851,8 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in snprintf(errbuf, errbufsize, "Can't load fromURL from URL class"); return -7; } + + int parc = 0; parv[parc++] = ats_strdup(err); ats_free(err); @@ -1150,10 +1152,9 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) new_mapping->map_id = 0; if ((bti->remap_optflg & REMAP_OPTFLG_MAP_ID) != 0) { int idx = 0; - char *c; int ret = remap_check_option((const char **)bti->argv, bti->argc, REMAP_OPTFLG_MAP_ID, &idx); if (ret & REMAP_OPTFLG_MAP_ID) { - c = strchr(bti->argv[idx], (int)'='); + char *c = strchr(bti->argv[idx], (int)'='); new_mapping->map_id = (unsigned int)atoi(++c); } } diff --git a/proxy/http/remap/RemapPluginInfo.h b/proxy/http/remap/RemapPluginInfo.h index a9ac0b76420..7802b5c252d 100644 --- a/proxy/http/remap/RemapPluginInfo.h +++ b/proxy/http/remap/RemapPluginInfo.h @@ -74,7 +74,7 @@ class RemapPluginInfo Do_Remap_F *do_remap_cb = nullptr; OS_Response_F *os_response_cb = nullptr; - RemapPluginInfo(ts::file::path &&library_path); + explicit RemapPluginInfo(ts::file::path &&library_path); ~RemapPluginInfo(); static self_type *find_by_path(std::string_view library_path); diff --git a/proxy/http/remap/RemapProcessor.cc b/proxy/http/remap/RemapProcessor.cc index b8776df27c9..05f47236e8e 100644 --- a/proxy/http/remap/RemapProcessor.cc +++ b/proxy/http/remap/RemapProcessor.cc @@ -144,11 +144,10 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) HTTPHdr *request_header = &s->hdr_info.client_request; URL *request_url = request_header->url_get(); char **redirect_url = &s->remap_redirect; - char host_hdr_buf[TS_MAX_HOST_NAME_LEN], tmp_referer_buf[4096], tmp_redirect_buf[4096], tmp_buf[2048], *c; + char tmp_referer_buf[4096], tmp_redirect_buf[4096], tmp_buf[2048]; const char *remapped_host; - int remapped_host_len, remapped_port, tmp; + int remapped_host_len, tmp; int from_len; - bool remap_found = false; referer_info *ri; map = s->url_map.getMapping(); @@ -189,8 +188,9 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) if ((s->filter_mask & URL_REMAP_FILTER_REDIRECT_FMT) != 0 && map->redir_chunk_list) { redirect_tag_str *rc; tmp_redirect_buf[(tmp = 0)] = 0; + for (rc = map->redir_chunk_list; rc; rc = rc->next) { - c = nullptr; + char *c = nullptr; switch (rc->type) { case 's': c = rc->chunk_str; @@ -233,8 +233,6 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) } } - remap_found = true; - // We also need to rewrite the "Host:" header if it exists and // pristine host hdr is not enabled int host_len; @@ -253,8 +251,10 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) // temporary buffer has adequate length // remapped_host = request_url->host_get(&remapped_host_len); - remapped_port = request_url->port_get_raw(); + int remapped_port = request_url->port_get_raw(); + + char host_hdr_buf[TS_MAX_HOST_NAME_LEN]; if (TS_MAX_HOST_NAME_LEN > remapped_host_len) { tmp = remapped_host_len; memcpy(host_hdr_buf, remapped_host, remapped_host_len); @@ -280,7 +280,7 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) request_header->mark_target_dirty(); - return remap_found; + return true; } Action * diff --git a/proxy/http/remap/UrlMapping.cc b/proxy/http/remap/UrlMapping.cc index c675abd2ca7..782b3e2ac56 100644 --- a/proxy/http/remap/UrlMapping.cc +++ b/proxy/http/remap/UrlMapping.cc @@ -132,12 +132,12 @@ redirect_tag_str * redirect_tag_str::parse_format_redirect_url(char *url) { char *c; - redirect_tag_str *r, **rr; + redirect_tag_str *r; redirect_tag_str *list = nullptr; - char type = 0; if (url && *url) { - for (rr = &list; *(c = url) != 0;) { + for (redirect_tag_str **rr = &list; *(c = url) != 0;) { + char type = 0; for (type = 's'; *c; c++) { if (c[0] == '%') { char tmp_type = (char)tolower((int)c[1]); @@ -162,7 +162,6 @@ redirect_tag_str::parse_format_redirect_url(char *url) } (*rr = r)->next = nullptr; rr = &(r->next); - // printf("\t***********'%c' - '%s'*******\n",r->type,r->chunk_str ? r->chunk_str : ""); } else { break; /* memory allocation error */ } From fde18999612520f71c6ee7742602d3f5e327cd07 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 24 Apr 2019 15:07:37 +0800 Subject: [PATCH 479/526] Don't assign if this and other are the same object --- src/tscore/ink_uuid.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/tscore/ink_uuid.cc b/src/tscore/ink_uuid.cc index e2a56f0c033..80940ad96b8 100644 --- a/src/tscore/ink_uuid.cc +++ b/src/tscore/ink_uuid.cc @@ -54,9 +54,11 @@ ATSUuid::initialize(TSUuidVersion v) ATSUuid & ATSUuid::operator=(const ATSUuid &other) { - memcpy(_uuid.data, other._uuid.data, sizeof(_uuid.data)); - memcpy(_string, other._string, sizeof(_string)); - _version = other._version; + if (this != &other) { // Self assignment guard + memcpy(_uuid.data, other._uuid.data, sizeof(_uuid.data)); + memcpy(_string, other._string, sizeof(_string)); + _version = other._version; + } return *this; } From 6d55813162a900cb3cc6d967695858da4ee8763c Mon Sep 17 00:00:00 2001 From: Vijay Mamidi Date: Wed, 24 Apr 2019 17:26:15 +0800 Subject: [PATCH 480/526] Revert RS-374 to avoid deadlocks --- iocore/hostdb/HostDB.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index fc876bd8702..9f4110a83ac 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -610,7 +610,6 @@ HostDBContinuation::insert(unsigned int attl) Action * HostDBProcessor::getby(Continuation *cont, cb_process_result_pfn cb_process_result, HostDBHash &hash, Options const &opt) { - bool trylock = (cb_process_result == nullptr); bool force_dns = false; EThread *thread = this_ethread(); Ptr mutex = thread->mutex; @@ -651,10 +650,6 @@ HostDBProcessor::getby(Continuation *cont, cb_process_result_pfn cb_process_resu // find the partition lock Ptr bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); MUTEX_TRY_LOCK(lock2, bucket_mutex, thread); - if (!lock2.is_locked() && !trylock) { - // Refer to: [TS-374](http://issues.apache.org/jira/browse/TS-374) - lock2.acquire(thread); - } if (lock2.is_locked()) { // If we can get the lock and a level 1 probe succeeds, return Ptr r = probe(bucket_mutex, hash, false); From 60abcd3936a179116aee8a98d896abac88a3fdb3 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 24 Apr 2019 16:43:40 +0800 Subject: [PATCH 481/526] cppcheck: fixes issues in cppcheck Assert statement calls a function which may have desired side effects: 'getDispatchController'. Assert statement calls a function which may have desired side effects: 'isAlive'. C-style pointer casting Class 'X' has a constructor with 1 argument that is not explicit. Local variable url shadows outer variable The scope of the variable 'X' can be reduced. Variable 'X' is assigned a value that is never used. --- example/append_transform/append_transform.c | 21 +++++++------------ example/blacklist_0/blacklist_0.c | 3 +-- example/bnull_transform/bnull_transform.c | 3 +-- example/cache_scan/cache_scan.cc | 6 ++---- .../cppapi/async_http_fetch/AsyncHttpFetch.cc | 17 ++++++++++----- example/cppapi/boom/boom.cc | 7 +++---- .../CustomErrorRemapPlugin.cc | 3 ++- .../GzipTransformationPlugin.cc | 2 +- example/cppapi/intercept/intercept.cc | 2 +- .../MultipleTransactionHookPlugins.cc | 4 ++-- example/cppapi/post_buffer/PostBuffer.cc | 2 +- example/cppapi/remap_plugin/RemapPlugin.cc | 3 ++- .../transactionhook/TransactionHookPlugin.cc | 3 ++- example/cppapi/websocket/WebSocket.h | 2 +- example/file_1/file_1.c | 3 +-- example/null_transform/null_transform.c | 3 +-- example/passthru/passthru.cc | 2 +- example/query_remap/query_remap.c | 7 +++---- example/redirect_1/redirect_1.c | 6 ++---- example/remap_header_add/remap_header_add.cc | 2 +- example/response_header_1/response_header_1.c | 4 ++-- example/secure_link/secure_link.c | 2 +- example/server_push/server_push.c | 8 +++---- example/server_transform/server_transform.c | 13 +++++------- 24 files changed, 59 insertions(+), 69 deletions(-) diff --git a/example/append_transform/append_transform.c b/example/append_transform/append_transform.c index 873d1c530fe..aca2fdbe34b 100644 --- a/example/append_transform/append_transform.c +++ b/example/append_transform/append_transform.c @@ -92,7 +92,6 @@ handle_transform(TSCont contp) TSVIO write_vio; MyData *data; int64_t towrite; - int64_t avail; /* Get the output connection where we'll write data to. */ output_conn = TSTransformOutputVConnGet(contp); @@ -146,7 +145,7 @@ handle_transform(TSCont contp) if (towrite > 0) { /* The amount of data left to read needs to be truncated by the amount of data actually in the read buffer. */ - avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); + int64_t avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); if (towrite > avail) { towrite = avail; } @@ -247,9 +246,6 @@ transformable(TSHttpTxn txnp) { TSMBuffer bufp; TSMLoc hdr_loc; - TSMLoc field_loc; - TSHttpStatus resp_status; - const char *value; int val_length; if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc)) { @@ -257,16 +253,16 @@ transformable(TSHttpTxn txnp) * We are only interested in "200 OK" responses. */ - if (TS_HTTP_STATUS_OK == (resp_status = TSHttpHdrStatusGet(bufp, hdr_loc))) { + if (TS_HTTP_STATUS_OK == TSHttpHdrStatusGet(bufp, hdr_loc)) { /* We only want to do the transformation on documents that have a content type of "text/html". */ - field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Content-Type", 12); + TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Content-Type", 12); if (!field_loc) { ASSERT_SUCCESS(TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc)); return 0; } - value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, -1, &val_length); + const char *value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, -1, &val_length); if (value && (strncasecmp(value, "text/html", sizeof("text/html") - 1) == 0)) { ASSERT_SUCCESS(TSHandleMLocRelease(bufp, hdr_loc, field_loc)); ASSERT_SUCCESS(TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc)); @@ -315,10 +311,7 @@ static int load(const char *filename) { TSFile fp; - TSIOBufferBlock blk; - char *p; int64_t avail; - int err; fp = TSfopen(filename, "r"); if (!fp) { @@ -329,10 +322,10 @@ load(const char *filename) append_buffer_reader = TSIOBufferReaderAlloc(append_buffer); for (;;) { - blk = TSIOBufferStart(append_buffer); - p = TSIOBufferBlockWriteStart(blk, &avail); + TSIOBufferBlock blk = TSIOBufferStart(append_buffer); + char *p = TSIOBufferBlockWriteStart(blk, &avail); - err = TSfread(fp, p, avail); + int err = TSfread(fp, p, avail); if (err > 0) { TSIOBufferProduce(append_buffer, err); } else { diff --git a/example/blacklist_0/blacklist_0.c b/example/blacklist_0/blacklist_0.c index aa86c255142..5ca5179e1fb 100644 --- a/example/blacklist_0/blacklist_0.c +++ b/example/blacklist_0/blacklist_0.c @@ -150,7 +150,6 @@ blacklist_plugin(TSCont contp, TSEvent event, void *edata) void TSPluginInit(int argc, const char *argv[]) { - int i; TSPluginRegistrationInfo info; info.plugin_name = PLUGIN_NAME; @@ -165,7 +164,7 @@ TSPluginInit(int argc, const char *argv[]) if (nsites > 0) { sites = (char **)TSmalloc(sizeof(char *) * nsites); - for (i = 0; i < nsites; i++) { + for (int i = 0; i < nsites; i++) { sites[i] = TSstrdup(argv[i + 1]); } diff --git a/example/bnull_transform/bnull_transform.c b/example/bnull_transform/bnull_transform.c index 09af6218704..55dd7fdd3ec 100644 --- a/example/bnull_transform/bnull_transform.c +++ b/example/bnull_transform/bnull_transform.c @@ -82,7 +82,6 @@ handle_buffering(TSCont contp, MyData *data) { TSVIO write_vio; int towrite; - int avail; /* Get the write VIO for the write operation that was performed on ourself. This VIO contains the buffer that we are to read from @@ -118,7 +117,7 @@ handle_buffering(TSCont contp, MyData *data) /* The amount of data left to read needs to be truncated by the amount of data actually in the read buffer. */ - avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); + int avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); if (towrite > avail) { towrite = avail; } diff --git a/example/cache_scan/cache_scan.cc b/example/cache_scan/cache_scan.cc index f889fb880a3..9d1d362496f 100644 --- a/example/cache_scan/cache_scan.cc +++ b/example/cache_scan/cache_scan.cc @@ -65,7 +65,6 @@ using cache_scan_state = struct cache_scan_state_t; static int handle_scan(TSCont contp, TSEvent event, void *edata) { - TSCacheHttpInfo cache_infop; cache_scan_state *cstate = (cache_scan_state *)TSContDataGet(contp); if (event == TS_EVENT_CACHE_REMOVE) { @@ -120,19 +119,18 @@ handle_scan(TSCont contp, TSEvent event, void *edata) if (cstate->done) { return TS_CACHE_SCAN_RESULT_DONE; } - cache_infop = (TSCacheHttpInfo)edata; + TSCacheHttpInfo cache_infop = (TSCacheHttpInfo)edata; TSMBuffer req_bufp, resp_bufp; TSMLoc req_hdr_loc, resp_hdr_loc; TSMLoc url_loc; - char *url; int url_len; const char s1[] = "URL: ", s2[] = "\n"; cstate->total_bytes += TSIOBufferWrite(cstate->resp_buffer, s1, sizeof(s1) - 1); TSCacheHttpInfoReqGet(cache_infop, &req_bufp, &req_hdr_loc); if (TS_SUCCESS == TSHttpHdrUrlGet(req_bufp, req_hdr_loc, &url_loc)) { - url = TSUrlStringGet(req_bufp, url_loc, &url_len); + char *url = TSUrlStringGet(req_bufp, url_loc, &url_len); cstate->total_bytes += TSIOBufferWrite(cstate->resp_buffer, url, url_len); cstate->total_bytes += TSIOBufferWrite(cstate->resp_buffer, s2, sizeof(s2) - 1); diff --git a/example/cppapi/async_http_fetch/AsyncHttpFetch.cc b/example/cppapi/async_http_fetch/AsyncHttpFetch.cc index b6622183057..291d650fb0f 100644 --- a/example/cppapi/async_http_fetch/AsyncHttpFetch.cc +++ b/example/cppapi/async_http_fetch/AsyncHttpFetch.cc @@ -42,7 +42,7 @@ GlobalPlugin *plugin; class AsyncHttpFetch2 : public AsyncHttpFetch { public: - AsyncHttpFetch2(const string &request) : AsyncHttpFetch(request){}; + explicit AsyncHttpFetch2(const string &request) : AsyncHttpFetch(request){}; }; class AsyncHttpFetch3 : public AsyncHttpFetch @@ -66,7 +66,9 @@ class DelayedAsyncHttpFetch : public AsyncHttpFetch, public AsyncReceiverdispatch()); + if (!getDispatchController()->dispatch()) { + TS_ERROR(TAG, "Failed to dispatch()"); + } delete this; } bool @@ -88,7 +90,7 @@ class TransactionHookPlugin : public TransactionPlugin, public AsyncReceiver { public: - TransactionHookPlugin(Transaction &transaction) + explicit TransactionHookPlugin(Transaction &transaction) : TransactionPlugin(transaction), transaction_(transaction), num_fetches_pending_(0), post_request_(nullptr) { TS_DEBUG(TAG, "Constructed TransactionHookPlugin, saved a reference to this transaction."); @@ -118,9 +120,14 @@ class TransactionHookPlugin : public TransactionPlugin, // canceling right after starting in this case, but cancel() can be called any time TS_DEBUG(TAG, "Will cancel delayed fetch"); - assert(delayed_provider->isAlive()); + if (!delayed_provider->isAlive()) { + TS_ERROR(TAG, "provider is NOT alive!"); + } + delayed_provider->cancel(); - assert(!delayed_provider->isAlive()); + if (delayed_provider->isAlive()) { + TS_ERROR(TAG, "provider is alive!"); + } } void diff --git a/example/cppapi/boom/boom.cc b/example/cppapi/boom/boom.cc index be58fca7d6a..8db6059625b 100644 --- a/example/cppapi/boom/boom.cc +++ b/example/cppapi/boom/boom.cc @@ -108,7 +108,7 @@ class IsRewritableCode : public std::unary_function std::string current_code_string_; public: - IsRewritableCode(int current_code) : current_code_(current_code) + explicit IsRewritableCode(int current_code) : current_code_(current_code) { std::ostringstream oss; oss << current_code_; @@ -218,12 +218,11 @@ BoomResponseRegistry::populate_error_responses(const std::string &base_directory // Filename (sans the .html suffix) becomes the entry to the // registry lookup table DIR *pDIR = nullptr; - struct dirent *entry; pDIR = opendir(base_error_directory_.c_str()); if (pDIR != nullptr) { while (true) { - entry = readdir(pDIR); + struct dirent *entry = readdir(pDIR); if (entry == nullptr) { break; } @@ -382,7 +381,7 @@ class BoomGlobalPlugin : public atscppapi::GlobalPlugin BoomResponseRegistry *response_registry_; public: - BoomGlobalPlugin(BoomResponseRegistry *response_registry) : response_registry_(response_registry) + explicit BoomGlobalPlugin(BoomResponseRegistry *response_registry) : response_registry_(response_registry) { TS_DEBUG(TAG, "Creating BoomGlobalHook %p", this); registerHook(HOOK_READ_RESPONSE_HEADERS); diff --git a/example/cppapi/custom_error_remap_plugin/CustomErrorRemapPlugin.cc b/example/cppapi/custom_error_remap_plugin/CustomErrorRemapPlugin.cc index f6cb20c7425..727c3cf79db 100644 --- a/example/cppapi/custom_error_remap_plugin/CustomErrorRemapPlugin.cc +++ b/example/cppapi/custom_error_remap_plugin/CustomErrorRemapPlugin.cc @@ -31,7 +31,8 @@ RemapPlugin *plugin; class MyRemapPlugin : public RemapPlugin { public: - MyRemapPlugin(void **instance_handle) : RemapPlugin(instance_handle) {} + explicit MyRemapPlugin(void **instance_handle) : RemapPlugin(instance_handle) {} + Result doRemap(const Url &map_from_url, const Url &map_to_url, Transaction &transaction, bool &redirect) override { diff --git a/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc b/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc index a2d7ba0c15c..fe191a57e79 100644 --- a/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc +++ b/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc @@ -89,7 +89,7 @@ class Helpers class SomeTransformationPlugin : public TransformationPlugin { public: - SomeTransformationPlugin(Transaction &transaction) + explicit SomeTransformationPlugin(Transaction &transaction) : TransformationPlugin(transaction, RESPONSE_TRANSFORMATION), transaction_(transaction) { registerHook(HOOK_SEND_RESPONSE_HEADERS); diff --git a/example/cppapi/intercept/intercept.cc b/example/cppapi/intercept/intercept.cc index ec3c747ea64..da615cb077b 100644 --- a/example/cppapi/intercept/intercept.cc +++ b/example/cppapi/intercept/intercept.cc @@ -35,7 +35,7 @@ GlobalPlugin *plugin; class Intercept : public InterceptPlugin { public: - Intercept(Transaction &transaction) : InterceptPlugin(transaction, InterceptPlugin::SERVER_INTERCEPT) {} + explicit Intercept(Transaction &transaction) : InterceptPlugin(transaction, InterceptPlugin::SERVER_INTERCEPT) {} void consume(const string &data, InterceptPlugin::RequestDataType type) override; void handleInputComplete() override; ~Intercept() override { cout << "Shutting down" << endl; } diff --git a/example/cppapi/multiple_transaction_hooks/MultipleTransactionHookPlugins.cc b/example/cppapi/multiple_transaction_hooks/MultipleTransactionHookPlugins.cc index d96712f94f9..fde3ea36657 100644 --- a/example/cppapi/multiple_transaction_hooks/MultipleTransactionHookPlugins.cc +++ b/example/cppapi/multiple_transaction_hooks/MultipleTransactionHookPlugins.cc @@ -31,7 +31,7 @@ GlobalPlugin *plugin; class MultipleTransactionHookPluginsOne : public atscppapi::TransactionPlugin { public: - MultipleTransactionHookPluginsOne(Transaction &transaction) : TransactionPlugin(transaction) + explicit MultipleTransactionHookPluginsOne(Transaction &transaction) : TransactionPlugin(transaction) { TransactionPlugin::registerHook(HOOK_SEND_RESPONSE_HEADERS); std::cout << "Constructed MultipleTransactionHookPluginsOne!" << std::endl; @@ -49,7 +49,7 @@ class MultipleTransactionHookPluginsOne : public atscppapi::TransactionPlugin class MultipleTransactionHookPluginsTwo : public atscppapi::TransactionPlugin { public: - MultipleTransactionHookPluginsTwo(Transaction &transaction) : TransactionPlugin(transaction) + explicit MultipleTransactionHookPluginsTwo(Transaction &transaction) : TransactionPlugin(transaction) { TransactionPlugin::registerHook(HOOK_SEND_REQUEST_HEADERS); TransactionPlugin::registerHook(HOOK_SEND_RESPONSE_HEADERS); diff --git a/example/cppapi/post_buffer/PostBuffer.cc b/example/cppapi/post_buffer/PostBuffer.cc index 1d67f8b1908..a5064c9b064 100644 --- a/example/cppapi/post_buffer/PostBuffer.cc +++ b/example/cppapi/post_buffer/PostBuffer.cc @@ -36,7 +36,7 @@ GlobalPlugin *plugin; class PostBufferTransformationPlugin : public TransformationPlugin { public: - PostBufferTransformationPlugin(Transaction &transaction) + explicit PostBufferTransformationPlugin(Transaction &transaction) : TransformationPlugin(transaction, REQUEST_TRANSFORMATION), transaction_(transaction) { buffer_.reserve(1024); // not required, this is an optimization to start the buffer at a slightly higher value. diff --git a/example/cppapi/remap_plugin/RemapPlugin.cc b/example/cppapi/remap_plugin/RemapPlugin.cc index aed34a91d37..508ce3b0b0e 100644 --- a/example/cppapi/remap_plugin/RemapPlugin.cc +++ b/example/cppapi/remap_plugin/RemapPlugin.cc @@ -36,7 +36,8 @@ RemapPlugin *plugin; class MyRemapPlugin : public RemapPlugin { public: - MyRemapPlugin(void **instance_handle) : RemapPlugin(instance_handle) {} + explicit MyRemapPlugin(void **instance_handle) : RemapPlugin(instance_handle) {} + Result doRemap(const Url &map_from_url, const Url &map_to_url, Transaction &transaction, bool &redirect) override { diff --git a/example/cppapi/transactionhook/TransactionHookPlugin.cc b/example/cppapi/transactionhook/TransactionHookPlugin.cc index d2b09a183f8..8cd8da370df 100644 --- a/example/cppapi/transactionhook/TransactionHookPlugin.cc +++ b/example/cppapi/transactionhook/TransactionHookPlugin.cc @@ -31,12 +31,13 @@ GlobalPlugin *plugin; class TransactionHookPlugin : public atscppapi::TransactionPlugin { public: - TransactionHookPlugin(Transaction &transaction) : TransactionPlugin(transaction) + explicit TransactionHookPlugin(Transaction &transaction) : TransactionPlugin(transaction) { char_ptr_ = new char[100]; TransactionPlugin::registerHook(HOOK_SEND_RESPONSE_HEADERS); std::cout << "Constructed!" << std::endl; } + ~TransactionHookPlugin() override { delete[] char_ptr_; // cleanup diff --git a/example/cppapi/websocket/WebSocket.h b/example/cppapi/websocket/WebSocket.h index ad4adccbefd..01528fa531a 100644 --- a/example/cppapi/websocket/WebSocket.h +++ b/example/cppapi/websocket/WebSocket.h @@ -40,7 +40,7 @@ using atscppapi::GlobalPlugin; class WebSocket : public InterceptPlugin { public: - WebSocket(Transaction &transaction); + explicit WebSocket(Transaction &transaction); ~WebSocket() override; void consume(const std::string &data, InterceptPlugin::RequestDataType type) override; diff --git a/example/file_1/file_1.c b/example/file_1/file_1.c index 21ee9400720..d1a4023e6cf 100644 --- a/example/file_1/file_1.c +++ b/example/file_1/file_1.c @@ -41,7 +41,6 @@ void TSPluginInit(int argc, const char *argv[]) { - TSFile filep; char buf[4096]; int i; TSPluginRegistrationInfo info; @@ -55,7 +54,7 @@ TSPluginInit(int argc, const char *argv[]) } for (i = 1; i < argc; i++) { - filep = TSfopen(argv[i], "r"); + TSFile filep = TSfopen(argv[i], "r"); if (!filep) { continue; } diff --git a/example/null_transform/null_transform.c b/example/null_transform/null_transform.c index e180f5c10cc..b350c3f076f 100644 --- a/example/null_transform/null_transform.c +++ b/example/null_transform/null_transform.c @@ -67,7 +67,6 @@ handle_transform(TSCont contp) TSVIO input_vio; MyData *data; int64_t towrite; - int64_t avail; TSDebug(PLUGIN_NAME, "Entering handle_transform()"); /* Get the output (downstream) vconnection where we'll write data to. */ @@ -126,7 +125,7 @@ handle_transform(TSCont contp) /* The amount of data left to read needs to be truncated by * the amount of data actually in the read buffer. */ - avail = TSIOBufferReaderAvail(TSVIOReaderGet(input_vio)); + int64_t avail = TSIOBufferReaderAvail(TSVIOReaderGet(input_vio)); TSDebug(PLUGIN_NAME, "\tavail is %" PRId64 "", avail); if (towrite > avail) { towrite = avail; diff --git a/example/passthru/passthru.cc b/example/passthru/passthru.cc index 587f35a0c6c..8912e2961be 100644 --- a/example/passthru/passthru.cc +++ b/example/passthru/passthru.cc @@ -182,7 +182,7 @@ static int PassthruSessionEvent(TSCont cont, TSEvent event, void *edata) { EventArgument arg(edata); - PassthruSession *sp = (PassthruSession *)TSContDataGet(cont); + PassthruSession *sp = static_cast(TSContDataGet(cont)); PassthruSessionDebug(sp, "session event on vconn=%p event=%d (%s)", TSVIOVConnGet(arg.vio), event, TSHttpEventNameLookup(event)); diff --git a/example/query_remap/query_remap.c b/example/query_remap/query_remap.c index 8bd6910179a..9e850bb7299 100644 --- a/example/query_remap/query_remap.c +++ b/example/query_remap/query_remap.c @@ -91,7 +91,6 @@ void TSRemapDeleteInstance(void *ih) { /* Release instance memory allocated in TSRemapNewInstance */ - int i; TSDebug(PLUGIN_NAME, "deleting instance %p", ih); if (ih) { @@ -100,7 +99,7 @@ TSRemapDeleteInstance(void *ih) TSfree(qri->param_name); } if (qri->hosts) { - for (i = 0; i < qri->num_hosts; ++i) { + for (int i = 0; i < qri->num_hosts; ++i) { TSfree(qri->hosts[i]); } TSfree(qri->hosts); @@ -112,7 +111,6 @@ TSRemapDeleteInstance(void *ih) TSRemapStatus TSRemapDoRemap(void *ih, TSHttpTxn rh ATS_UNUSED, TSRemapRequestInfo *rri) { - int hostidx = -1; query_remap_info *qri = (query_remap_info *)ih; if (!qri || !rri) { @@ -125,7 +123,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh ATS_UNUSED, TSRemapRequestInfo *rri) if (req_query && req_query_len > 0) { char *q, *key; - char *s = NULL; + char *s = NULL; + int hostidx = -1; /* make a copy of the query, as it is read only */ q = (char *)TSstrndup(req_query, req_query_len + 1); diff --git a/example/redirect_1/redirect_1.c b/example/redirect_1/redirect_1.c index 677ef716e14..45b83408700 100644 --- a/example/redirect_1/redirect_1.c +++ b/example/redirect_1/redirect_1.c @@ -246,7 +246,6 @@ update_redirected_method_stats(TSMBuffer bufp, TSMLoc hdr_loc) { const char *txn_method; int length; - int64_t tempint; txn_method = TSHttpHdrMethodGet(bufp, hdr_loc, &length); @@ -264,7 +263,7 @@ update_redirected_method_stats(TSMBuffer bufp, TSMLoc hdr_loc) } else if (0 == strncmp(txn_method, TS_HTTP_METHOD_OPTIONS, length)) { // This is a bad idea in a real plugin because it causes a race condition // with other transactions, but is here for illustrative purposes. - tempint = TSStatIntGet(redirect_count_options); + int64_t tempint = TSStatIntGet(redirect_count_options); ++tempint; TSStatIntSet(redirect_count_options, tempint); } else if (0 == strncmp(txn_method, TS_HTTP_METHOD_POST, length)) { @@ -289,7 +288,6 @@ void TSPluginInit(int argc, const char *argv[]) { const char prefix[] = "http://"; - int uri_len; TSPluginRegistrationInfo info; info.plugin_name = PLUGIN_NAME; @@ -308,7 +306,7 @@ TSPluginInit(int argc, const char *argv[]) */ url_redirect = TSstrdup(argv[2]); - uri_len = strlen(prefix) + strlen(url_redirect) + 1; + int uri_len = strlen(prefix) + strlen(url_redirect) + 1; uri_redirect = TSmalloc(uri_len); TSstrlcpy(uri_redirect, prefix, uri_len); TSstrlcat(uri_redirect, url_redirect, uri_len); diff --git a/example/remap_header_add/remap_header_add.cc b/example/remap_header_add/remap_header_add.cc index d1152914423..f538fae264e 100644 --- a/example/remap_header_add/remap_header_add.cc +++ b/example/remap_header_add/remap_header_add.cc @@ -98,7 +98,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *, int) // print all arguments for this particular remapping - rl = (remap_line *)TSmalloc(sizeof(remap_line)); + rl = static_cast(TSmalloc(sizeof(remap_line))); rl->argc = argc; rl->argv = argv; rl->nvc = argc - 2; // the first two are the remap from and to diff --git a/example/response_header_1/response_header_1.c b/example/response_header_1/response_header_1.c index a96afaa9fe3..65f173a8b5d 100644 --- a/example/response_header_1/response_header_1.c +++ b/example/response_header_1/response_header_1.c @@ -68,7 +68,6 @@ modify_header(TSHttpTxn txnp) TSHttpStatus resp_status; TSMLoc new_field_loc; TSMLoc cached_field_loc; - time_t recvd_time; const char *chkptr; int chklength; @@ -114,7 +113,8 @@ modify_header(TSHttpTxn txnp) TSDebug(PLUGIN_NAME, "Created new resp field with loc %p", new_field_loc); TSMimeHdrFieldAppend(resp_bufp, resp_loc, new_field_loc); TSMimeHdrFieldNameSet(resp_bufp, resp_loc, new_field_loc, mimehdr2_name, strlen(mimehdr2_name)); - recvd_time = time(NULL); + + time_t recvd_time = time(NULL); TSMimeHdrFieldValueDateInsert(resp_bufp, resp_loc, new_field_loc, recvd_time); TSHandleMLocRelease(resp_bufp, resp_loc, new_field_loc); diff --git a/example/secure_link/secure_link.c b/example/secure_link/secure_link.c index 9bf8506fba7..871e7617e67 100644 --- a/example/secure_link/secure_link.c +++ b/example/secure_link/secure_link.c @@ -151,7 +151,6 @@ TSReturnCode TSRemapNewInstance(int argc, char **argv, void **ih, char *errbuf, int errbuf_size) { int i; - char *ptr; secure_link_info *sli; // squash unused variable warnings ... @@ -163,6 +162,7 @@ TSRemapNewInstance(int argc, char **argv, void **ih, char *errbuf, int errbuf_si sli->strict = 0; for (i = 2; i < argc; i++) { + char *ptr; if ((ptr = strchr(argv[i], ':')) != NULL) { *ptr++ = '\0'; if (strcmp(argv[i], "secret") == 0) { diff --git a/example/server_push/server_push.c b/example/server_push/server_push.c index 717d373bcf0..41203f532db 100644 --- a/example/server_push/server_push.c +++ b/example/server_push/server_push.c @@ -47,16 +47,16 @@ bool should_push(TSHttpTxn txnp) { TSMBuffer mbuf; - TSMLoc hdr, url; + TSMLoc hdr, in_url; if (TSHttpTxnClientReqGet(txnp, &mbuf, &hdr) != TS_SUCCESS) { return false; } - if (TSHttpHdrUrlGet(mbuf, hdr, &url) != TS_SUCCESS) { + if (TSHttpHdrUrlGet(mbuf, hdr, &in_url) != TS_SUCCESS) { return false; } int len; - TSUrlHttpQueryGet(mbuf, url, &len); - TSHandleMLocRelease(mbuf, hdr, url); + TSUrlHttpQueryGet(mbuf, in_url, &len); + TSHandleMLocRelease(mbuf, hdr, in_url); TSHandleMLocRelease(mbuf, TS_NULL_MLOC, hdr); if (len > 0) { return true; diff --git a/example/server_transform/server_transform.c b/example/server_transform/server_transform.c index ec93e5303e4..825c1fedb5f 100644 --- a/example/server_transform/server_transform.c +++ b/example/server_transform/server_transform.c @@ -285,7 +285,6 @@ transform_buffer_event(TSCont contp, TransformData *data, TSEvent event ATS_UNUS { TSVIO write_vio; int towrite; - int avail; if (!data->input_buf) { data->input_buf = TSIOBufferCreate(); @@ -314,7 +313,7 @@ transform_buffer_event(TSCont contp, TransformData *data, TSEvent event ATS_UNUS if (towrite > 0) { /* The amount of data left to read needs to be truncated by the amount of data actually in the read buffer. */ - avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); + int avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); if (towrite > avail) { towrite = avail; } @@ -404,18 +403,16 @@ transform_read_status_event(TSCont contp, TransformData *data, TSEvent event, vo return transform_bypass(contp, data); case TS_EVENT_VCONN_READ_COMPLETE: if (TSIOBufferReaderAvail(data->output_reader) == sizeof(int)) { - TSIOBufferBlock blk; - char *buf; void *buf_ptr; int64_t avail; int64_t read_nbytes = sizeof(int); - int64_t read_ndone = 0; buf_ptr = &data->content_length; while (read_nbytes > 0) { - blk = TSIOBufferReaderStart(data->output_reader); - buf = (char *)TSIOBufferBlockReadStart(blk, data->output_reader, &avail); - read_ndone = (avail >= read_nbytes) ? read_nbytes : avail; + TSIOBufferBlock blk = TSIOBufferReaderStart(data->output_reader); + char *buf = (char *)TSIOBufferBlockReadStart(blk, data->output_reader, &avail); + int64_t read_ndone = (avail >= read_nbytes) ? read_nbytes : avail; + memcpy(buf_ptr, buf, read_ndone); if (read_ndone > 0) { TSIOBufferReaderConsume(data->output_reader, read_ndone); From a0e9f8ff26536ec87d8a231bec6d580d90c41530 Mon Sep 17 00:00:00 2001 From: Jan van Doorn Date: Tue, 23 Apr 2019 16:51:11 +0800 Subject: [PATCH 482/526] cppcheck: fixes issues found for plugins/authproxy --- plugins/authproxy/authproxy.cc | 22 ++++++++++------------ plugins/authproxy/utils.h | 3 +-- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/plugins/authproxy/authproxy.cc b/plugins/authproxy/authproxy.cc index 8ce13be763e..80b4567363f 100644 --- a/plugins/authproxy/authproxy.cc +++ b/plugins/authproxy/authproxy.cc @@ -148,10 +148,10 @@ struct AuthRequestContext { const StateTransition *state = nullptr; - AuthRequestContext() : hparser(TSHttpParserCreate()), rheader(), iobuf(TS_IOBUFFER_SIZE_INDEX_4K) + AuthRequestContext() + : cont(TSContCreate(dispatch, TSMutexCreate())), hparser(TSHttpParserCreate()), rheader(), iobuf(TS_IOBUFFER_SIZE_INDEX_4K) { - this->cont = TSContCreate(dispatch, TSMutexCreate()); TSContDataSet(this->cont, this); } @@ -170,7 +170,7 @@ struct AuthRequestContext { { AuthOptions *opt; - opt = (AuthOptions *)TSHttpTxnArgGet(this->txn, AuthTaggedRequestArg); + opt = static_cast(TSHttpTxnArgGet(this->txn, AuthTaggedRequestArg)); return opt ? opt : AuthGlobalOptions; } @@ -198,7 +198,7 @@ AuthRequestContext::destroy(AuthRequestContext *auth) int AuthRequestContext::dispatch(TSCont cont, TSEvent event, void *edata) { - AuthRequestContext *auth = (AuthRequestContext *)TSContDataGet(cont); + AuthRequestContext *auth = static_cast(TSContDataGet(cont)); const StateTransition *s; pump: @@ -415,7 +415,6 @@ static TSEvent StateAuthProxyCompleteHeaders(AuthRequestContext *auth, void * /* edata ATS_UNUSED */) { TSHttpStatus status; - unsigned nbytes; HttpDebugHeader(auth->rheader.buffer, auth->rheader.header); @@ -436,7 +435,7 @@ StateAuthProxyCompleteHeaders(AuthRequestContext *auth, void * /* edata ATS_UNUS } else { // OK, we have a non-chunked response. If there's any content, let's go and // buffer it so that we can send it on to the client. - nbytes = HttpGetContentLength(auth->rheader.buffer, auth->rheader.header); + unsigned nbytes = HttpGetContentLength(auth->rheader.buffer, auth->rheader.header); if (nbytes > 0) { AuthLogDebug("content length is %u", nbytes); return TS_EVENT_HTTP_CONTINUE; @@ -628,7 +627,6 @@ AuthRequestIsTagged(TSHttpTxn txn) static int AuthProxyGlobalHook(TSCont /* cont ATS_UNUSED */, TSEvent event, void *edata) { - AuthRequestContext *auth; TSHttpTxn txn = (TSHttpTxn)edata; AuthLogDebug("handling event=%d edata=%p", (int)event, edata); @@ -653,9 +651,9 @@ AuthProxyGlobalHook(TSCont /* cont ATS_UNUSED */, TSEvent event, void *edata) // Hook this request if we are in global authorization mode or if a // remap rule tagged it. if (AuthGlobalOptions != nullptr || AuthRequestIsTagged(txn)) { - auth = AuthRequestContext::allocate(); - auth->state = StateTableInit; - auth->txn = txn; + AuthRequestContext *auth = AuthRequestContext::allocate(); + auth->state = StateTableInit; + auth->txn = txn; return AuthRequestContext::dispatch(auth->cont, event, edata); } // fallthru @@ -778,14 +776,14 @@ TSRemapNewInstance(int argc, char *argv[], void **instance, char * /* err ATS_UN void TSRemapDeleteInstance(void *instance) { - AuthOptions *options = (AuthOptions *)instance; + AuthOptions *options = static_cast(instance); AuthDelete(options); } TSRemapStatus TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri ATS_UNUSED */) { - AuthOptions *options = (AuthOptions *)instance; + AuthOptions *options = static_cast(instance); TSHttpTxnArgSet(txn, AuthTaggedRequestArg, options); TSHttpTxnHookAdd(txn, TS_HTTP_POST_REMAP_HOOK, AuthOsDnsContinuation); diff --git a/plugins/authproxy/utils.h b/plugins/authproxy/utils.h index a95e2fe860b..72a5830ada3 100644 --- a/plugins/authproxy/utils.h +++ b/plugins/authproxy/utils.h @@ -43,9 +43,8 @@ struct HttpIoBuffer { TSIOBufferReader reader; explicit HttpIoBuffer(TSIOBufferSizeIndex size = TS_IOBUFFER_SIZE_INDEX_32K) + : buffer(TSIOBufferSizedCreate(size)), reader(TSIOBufferReaderAlloc(buffer)) { - this->buffer = TSIOBufferSizedCreate(size); - this->reader = TSIOBufferReaderAlloc(this->buffer); } ~HttpIoBuffer() From 969a41ce590fa725d7560cd07b9ec53af30a26ef Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Wed, 24 Apr 2019 17:05:36 +0800 Subject: [PATCH 483/526] Password can not be nullptr here, so don't check --- src/traffic_server/InkAPITest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index a89b9c1b71a..5e574991284 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -3836,7 +3836,7 @@ REGRESSION_TEST(SDK_API_TSUrl)(RegressionTest *test, int /* atype ATS_UNUSED */, SDK_RPRINT(test, "TSUrlPasswordSet", "TestCase1", TC_FAIL, "Returned TS_ERROR"); } else { password_get = TSUrlPasswordGet(bufp1, url_loc1, &length); - if (((password_get == nullptr) && (password == nullptr)) || (strncmp(password_get, password, length) == 0)) { + if ((password_get == nullptr) || (strncmp(password_get, password, length) == 0)) { SDK_RPRINT(test, "TSUrlPasswordSet&Get", "TestCase1", TC_PASS, "ok"); test_passed_password = true; } else { From 4a9e35056fe7954903edae32153462009a791089 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Wed, 24 Apr 2019 15:38:57 +0800 Subject: [PATCH 484/526] cppcheck: Declare a member variable with an initial value [iocore/net/P_UDPNet.h:66]: (performance) Variable 'lastPullLongTermQ' is assigned in constructor body. Consider performing initialization in initialization list. --- iocore/net/P_UDPNet.h | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index 9368cbba5b2..1be6d86dbdb 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -61,15 +61,11 @@ extern UDPNetProcessorInternal udpNetInternal; class PacketQueue { public: - PacketQueue() - { - lastPullLongTermQ = 0; - init(); - } + PacketQueue() { init(); } virtual ~PacketQueue() {} - int nPackets = 0; - ink_hrtime lastPullLongTermQ; + int nPackets = 0; + ink_hrtime lastPullLongTermQ = 0; Queue longTermQ; Queue bucket[N_SLOTS]; ink_hrtime delivery_time[N_SLOTS]; From 1a3c257f71f30c936a12e4925960b9b4cfbcf4a4 Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Wed, 24 Apr 2019 15:39:53 +0800 Subject: [PATCH 485/526] cppcheck: Minimize a variable scope [iocore/net/P_UDPPacket.h:192]: (style) The scope of the variable 'body' can be reduced. --- iocore/net/P_UDPPacket.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/iocore/net/P_UDPPacket.h b/iocore/net/P_UDPPacket.h index 21fbfe7269d..38364e699b5 100644 --- a/iocore/net/P_UDPPacket.h +++ b/iocore/net/P_UDPPacket.h @@ -189,7 +189,6 @@ new_UDPPacket(struct sockaddr const *to, ink_hrtime when, IOBufferBlock *buf, in { (void)len; UDPPacketInternal *p = udpPacketAllocator.alloc(); - IOBufferBlock *body; p->in_the_priority_queue = 0; p->in_heap = 0; @@ -197,7 +196,7 @@ new_UDPPacket(struct sockaddr const *to, ink_hrtime when, IOBufferBlock *buf, in ats_ip_copy(&p->to, to); while (buf) { - body = buf->clone(); + IOBufferBlock *body = buf->clone(); p->append_block(body); buf = buf->next.get(); } From 7bef557ab8b46fa87eb8e9b4d9e906e3d6fd9145 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 23 Apr 2019 02:33:31 -0500 Subject: [PATCH 486/526] cppcheck: Removed problematic move operators for FixedBufferWriter. --- include/tscore/BufferWriter.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/tscore/BufferWriter.h b/include/tscore/BufferWriter.h index f5fce06db73..57b6c570f67 100644 --- a/include/tscore/BufferWriter.h +++ b/include/tscore/BufferWriter.h @@ -207,6 +207,9 @@ class BufferWriter }; /** A @c BufferWrite concrete subclass to write to a fixed size buffer. + * + * Copies and moves are forbidden because that leaves the original in a potentially bad state. An + * instance is cheap to construct and should be done explicitly when needed. */ class FixedBufferWriter : public BufferWriter { @@ -231,10 +234,8 @@ class FixedBufferWriter : public BufferWriter FixedBufferWriter(const FixedBufferWriter &) = delete; FixedBufferWriter &operator=(const FixedBufferWriter &) = delete; - /// Move constructor. - FixedBufferWriter(FixedBufferWriter &&) = default; - /// Move assignment. - FixedBufferWriter &operator=(FixedBufferWriter &&) = default; + FixedBufferWriter(FixedBufferWriter &&) = delete; + FixedBufferWriter &operator=(FixedBufferWriter &&) = delete; FixedBufferWriter(MemSpan &span) : _buf(span.begin()), _capacity(static_cast(span.size())) {} From d0753e21cd76d85ef4a9d3cb9da540edcbe3c2be Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 25 Apr 2019 11:36:55 +0800 Subject: [PATCH 487/526] Adds assert, albeit not needed, makes CA happy --- mgmt/utils/MgmtMarshall.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/mgmt/utils/MgmtMarshall.cc b/mgmt/utils/MgmtMarshall.cc index eb2358600c3..35263c34331 100644 --- a/mgmt/utils/MgmtMarshall.cc +++ b/mgmt/utils/MgmtMarshall.cc @@ -42,6 +42,7 @@ data_is_nul_terminated(const MgmtMarshallData *data) { const char *str = (const char *)(data->ptr); + ink_assert(str); if (str[data->len - 1] != '\0') { return false; } From 00237a141cdb6d9f2268080f03cb5f58ee32c7ec Mon Sep 17 00:00:00 2001 From: Masakazu Kitajo Date: Thu, 18 Apr 2019 22:27:49 +0900 Subject: [PATCH 488/526] Don't read frames after sending GOAWAY with an error code --- proxy/http2/Http2ClientSession.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index eb644e1c1a4..35d60acb660 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -543,6 +543,11 @@ Http2ClientSession::state_process_frame_read(int event, VIO *vio, bool inside_fr } while (this->sm_reader->read_avail() >= (int64_t)HTTP2_FRAME_HEADER_LEN) { + // Cancel reading if there was an error + if (connection_state.tx_error_code.code != static_cast(Http2ErrorCode::HTTP2_ERROR_NO_ERROR)) { + Http2SsnDebug("reading a frame has been canceled (%u)", connection_state.tx_error_code.code); + break; + } // Return if there was an error Http2ErrorCode err; if (do_start_frame_read(err) < 0) { From 83e5b28cfb7612640f462543f47f7080d676da6e Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Thu, 25 Apr 2019 13:28:31 +0800 Subject: [PATCH 489/526] Move the delete to please clang-analyzer --- proxy/PluginVC.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc index 79abc43663a..569ea851127 100644 --- a/proxy/PluginVC.cc +++ b/proxy/PluginVC.cc @@ -1283,7 +1283,7 @@ class PVCTestDriver : public NetTestDriver ~PVCTestDriver() override; void start_tests(RegressionTest *r_arg, int *pstatus_arg); - void run_next_test(); + bool run_next_test(); int main_handler(int event, void *data); private: @@ -1307,12 +1307,12 @@ PVCTestDriver::start_tests(RegressionTest *r_arg, int *pstatus_arg) r = r_arg; pstatus = pstatus_arg; - run_next_test(); - - SET_HANDLER(&PVCTestDriver::main_handler); + if (run_next_test()) { + SET_HANDLER(&PVCTestDriver::main_handler); + } } -void +bool PVCTestDriver::run_next_test() { unsigned a_index = i * 2; @@ -1326,7 +1326,7 @@ PVCTestDriver::run_next_test() *pstatus = REGRESSION_TEST_FAILED; } delete this; - return; + return false; } completions_received = 0; i++; @@ -1341,6 +1341,8 @@ PVCTestDriver::run_next_test() PluginVC *a_vc = core->connect(); a->init_test(NET_VC_TEST_ACTIVE, this, a_vc, r, &netvc_tests_def[a_index], "PluginVC", "pvc_test_detail"); + + return true; } int From 0eb7810fba923cd66f837991eac9c8f2ef659b42 Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Tue, 23 Apr 2019 13:42:07 +0800 Subject: [PATCH 490/526] cppcheck: Changed from C casting to C++ casting --- iocore/net/SSLConfig.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 491bd784627..ca61712e4ca 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -475,7 +475,7 @@ SSLConfig::reconfigure() SSLConfigParams * SSLConfig::acquire() { - return ((SSLConfigParams *)configProcessor.get(configid)); + return static_cast(configProcessor.get(configid)); } void @@ -544,7 +544,7 @@ SSLCertificateConfig::reconfigure() SSLCertLookup * SSLCertificateConfig::acquire() { - return (SSLCertLookup *)configProcessor.get(configid); + return static_cast(configProcessor.get(configid)); } void From 19d1733f555bc6cbf0d55e12dbf6eeb81fdd0519 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Thu, 25 Apr 2019 11:26:34 +0800 Subject: [PATCH 491/526] cppcheck: fix comparison issues found in plugins (style) Condition 'TS_EVENT_VCONN_WRITE_COMPLETE' is always true --- plugins/experimental/acme/acme.c | 2 +- plugins/healthchecks/healthchecks.c | 2 +- plugins/stats_over_http/stats_over_http.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/experimental/acme/acme.c b/plugins/experimental/acme/acme.c index 32c899d61a3..65507d9c3bd 100644 --- a/plugins/experimental/acme/acme.c +++ b/plugins/experimental/acme/acme.c @@ -208,7 +208,7 @@ acme_process_write(TSCont contp, TSEvent event, AcmeState *my_state) TSVIONBytesSet(my_state->write_vio, my_state->output_bytes); TSVIOReenable(my_state->write_vio); - } else if (TS_EVENT_VCONN_WRITE_COMPLETE) { + } else if (event == TS_EVENT_VCONN_WRITE_COMPLETE) { cleanup(contp, my_state); } else if (event == TS_EVENT_ERROR) { TSError("[%s] acme_process_write: Received TS_EVENT_ERROR", PLUGIN_NAME); diff --git a/plugins/healthchecks/healthchecks.c b/plugins/healthchecks/healthchecks.c index c2fe0955a5b..3fe44b95de6 100644 --- a/plugins/healthchecks/healthchecks.c +++ b/plugins/healthchecks/healthchecks.c @@ -458,7 +458,7 @@ hc_process_write(TSCont contp, TSEvent event, HCState *my_state) } TSVIONBytesSet(my_state->write_vio, my_state->output_bytes); TSVIOReenable(my_state->write_vio); - } else if (TS_EVENT_VCONN_WRITE_COMPLETE) { + } else if (event == TS_EVENT_VCONN_WRITE_COMPLETE) { cleanup(contp, my_state); } else if (event == TS_EVENT_ERROR) { TSError("[healthchecks] hc_process_write: Received TS_EVENT_ERROR"); diff --git a/plugins/stats_over_http/stats_over_http.c b/plugins/stats_over_http/stats_over_http.c index 919c394f62d..3f44b688678 100644 --- a/plugins/stats_over_http/stats_over_http.c +++ b/plugins/stats_over_http/stats_over_http.c @@ -205,7 +205,7 @@ stats_process_write(TSCont contp, TSEvent event, stats_state *my_state) TSVIONBytesSet(my_state->write_vio, my_state->output_bytes); } TSVIOReenable(my_state->write_vio); - } else if (TS_EVENT_VCONN_WRITE_COMPLETE) { + } else if (event == TS_EVENT_VCONN_WRITE_COMPLETE) { stats_cleanup(contp, my_state); } else if (event == TS_EVENT_ERROR) { TSError("[%s] stats_process_write: Received TS_EVENT_ERROR", PLUGIN_NAME); From b42fb6000d5b226d237fe40d8fbdbb683e393881 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Mon, 22 Apr 2019 12:52:47 +0000 Subject: [PATCH 492/526] Add slice plugin config options to either pace or disable block stitch error logs. --- doc/admin-guide/plugins/slice.en.rst | 91 +++++++-- plugins/experimental/slice/Config.cc | 188 +++++++++++------- plugins/experimental/slice/Config.h | 26 ++- plugins/experimental/slice/Data.h | 9 +- plugins/experimental/slice/Makefile.inc | 6 +- plugins/experimental/slice/README.md | 75 ++++++- plugins/experimental/slice/client.cc | 8 +- plugins/experimental/slice/server.cc | 61 +++--- plugins/experimental/slice/slice.cc | 11 +- plugins/experimental/slice/slice.h | 4 +- .../slice/unit-tests/test_config.cc | 22 +- .../gold_tests/pluginTest/slice/slice.test.py | 2 +- .../pluginTest/slice/slice_error.test.py | 4 +- 13 files changed, 340 insertions(+), 167 deletions(-) diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst index 86ad1a622d0..58cc610cdd8 100644 --- a/doc/admin-guide/plugins/slice.en.rst +++ b/doc/admin-guide/plugins/slice.en.rst @@ -61,39 +61,87 @@ In this case, the plugin will use the default behaviour: Plugin Options -------------- -Slice block sizes can specified using the blockbytes parameter:: +The slice plugin supports the following options:: - @plugin=slice.so @pparam=blockbytes:1000000 @cache_range_requests.so + --blockbytes= (optional) + Default is 1m or 1048576 bytes + -b for short. + Suffix k,m,g supported + Limited to 32k and 32m inclusive. -In adition to bytes, 'k', 'm' and 'g' suffixes may be used for -kilobytes, megabytes and gigabytes:: + --test-blockbytes= (optional) + Suffix k,m,g supported + -t for short. + Limited to any positive number. + Ignored if --blockbytes provided. - @plugin=slice.so @pparam=blockbytes:5m @cache_range_requests.so - @plugin=slice.so @pparam=blockbytes:512k @cache_range_requests.so - @plugin=slice.so @pparam=blockbytes:32m @cache_range_requests.so + --pace-errorlog= (optional) + Limit stitching error logs to every 'n' second(s) -paramater ``blockbytes`` is checked to be between 32kb and 32mb -inclusive. + --disable-errorlog (optional) + Disable writing block stitch errors to the error log. -For testing and extreme purposes the parameter ``bytesover`` may +Examples:: + + @plugin=slice.so @pparam=--blockbytes=1000000 @plugin=cache_range_requests.so + +Or alternatively:: + + @plugin=slice.so @pparam=-b @pparam=1000000 @plugin=cache_range_requests.so + +Byte suffix examples:: + + slice.so --blockbytes=5m + slice.so -b 512k + slice.so --blockbytes=32m + +For testing and extreme purposes the parameter ``test-blockbytes`` may be used instead which is unchecked:: - @plugin=slice.so @pparam=bytesover:1G @cache_range_requests.so - @plugin=slice.so @pparam=bytesover:13 @cache_range_requests.so + slice.so --test-blockbytes=1G + slice.so -t 13 + +Because the slice plugin is susceptible to errors during block stitching +extra logs related to stitching are written to ``diags.log``. Worst case +an error log entry could be generated for every transaction. The +following options are provided to help with log overrun:: + + slice.so --pace-errorlog=5 + slice.so -p 1 + slice.so --disable-errorlog After modifying :file:`remap.config`, restart or reload traffic server (sudo traffic_ctl config reload) or (sudo traffic_ctl server restart) to activate the new configuration values. +Debug Options +------------- + +While the current slice plugin is able to detect block consistency +errors during the block stitching process, it can only abort the +client connection. A CDN can only "fix" these by issuing an appropriate +content revalidation. + +Under normal logging these slice block errors tend to show up as:: + + pscl value 0 + crc value ERR_READ_ERROR + +By default more detailed stitching errors are written to ``diags.log``. +An example is as follows:: + +[Apr 19 20:26:13.639] [ET_NET 17] ERROR: [slice] 1555705573.639 reason="Non 206 internal block response" uri="http://localhost:18080/%7Ep.tex/%7Es.50M/%7Eui.20000/" uas="curl/7.29.0" req_range="bytes=1000000-" norm_range="bytes 1000000-52428799/52428800" etag_exp="%221603934496%22" lm_exp="Fri, 19 Apr 2019 18:53:20 GMT" blk_range="21000000-21999999" status_got="400" cr_got="" etag_got="" lm_got="" cc="no-store" via="" + +Whether or how often these detailed log entries are written are +configurable plugin options. + Implementation Notes ==================== -This slice plugin is by no means a best solution for adding -blocking support to ATS. - -The slice plugin as is designed to provide a basic capability to block -requests for arbitrary range requests and for blocking large assets for -ease of caching. +This slice plugin is a stop gap plugin for handling special cases +involving very large assets that may be range requested. Hopefully +the slice plugin is deprecated in the future when partial object +caching is finally implemented. Slice *ONLY* handles slicing up requests into blocks, it delegates actual caching and fetching to the cache_range_requests.so plugin. @@ -133,7 +181,7 @@ the block fetch to ensure the block is cached. Important Notes =============== -This plugin also assumes that the content requested is cacheable. +This plugin assumes that the content requested is cacheable. Any first block server response that is not a 206 is passed directly down to the client. If that response is a '200' only the first @@ -145,7 +193,7 @@ remove these headers. The only 416 response that this plugin handles itself is if the requested range is inside the last slice block but past the end of -the asset contents. Other 416 responses are handled by the parents. +the asset contents. Other 416 responses are handled by the parent. If a client aborts mid transaction the current slice block continues to be read from the server until it is complete to ensure that the block @@ -166,5 +214,4 @@ check for the existence of any headers they may have created the first time they have were visited. Since the Slice plugin is written as an intercept handler it loses the -ability to use state machine hooks and transaction states. - +ability to use normal state machine hooks and transaction states. diff --git a/plugins/experimental/slice/Config.cc b/plugins/experimental/slice/Config.cc index 27ec5aa89ab..43d1a7fc2d0 100644 --- a/plugins/experimental/slice/Config.cc +++ b/plugins/experimental/slice/Config.cc @@ -18,22 +18,23 @@ #include "Config.h" -#include #include #include -#include -#include +#include +#include +#include + +#include "ts/experimental.h" int64_t -Config::bytesFrom(std::string const &valstr) +Config::bytesFrom(char const *const valstr) { - char const *const nptr = valstr.c_str(); - char *endptr = nullptr; - int64_t blockbytes = strtoll(nptr, &endptr, 10); + char *endptr = nullptr; + int64_t blockbytes = strtoll(valstr, &endptr, 10); - if (nullptr != endptr && nptr < endptr) { - size_t const dist = endptr - nptr; - if (dist < valstr.size() && 0 <= blockbytes) { + if (nullptr != endptr && valstr < endptr) { + size_t const dist = endptr - valstr; + if (dist < strlen(valstr) && 0 <= blockbytes) { switch (tolower(*endptr)) { case 'g': blockbytes *= ((int64_t)1024 * (int64_t)1024 * (int64_t)1024); @@ -58,96 +59,135 @@ Config::bytesFrom(std::string const &valstr) } bool -Config::fromArgs(int const argc, char const *const argv[], char *const errbuf, int const errbuf_size) +Config::fromArgs(int const argc, char const *const argv[]) { -#if !defined(SLICE_UNIT_TEST) DEBUG_LOG("Number of arguments: %d", argc); for (int index = 0; index < argc; ++index) { DEBUG_LOG("args[%d] = %s", index, argv[index]); } -#endif - std::map keyvals; + // current "best" blockbytes from configuration + int64_t blockbytes = 0; - static std::string const bbstr(blockbytesstr); - static std::string const bostr(bytesoverstr); - - // collect all args + // backwards compat: look for blockbytes for (int index = 0; index < argc; ++index) { - std::string const argstr = argv[index]; + std::string_view const argstr = argv[index]; std::size_t const spos = argstr.find_first_of(':'); - if (spos != std::string::npos) { - std::string key = argstr.substr(0, spos); - std::string val = argstr.substr(spos + 1); - - if (!key.empty()) { - std::for_each(key.begin(), key.end(), [](char &ch) { ch = tolower(ch); }); - - // blockbytes and bytesover collide - if (bbstr == key) { - keyvals.erase(bostr); - } else if (bostr == key) { - keyvals.erase(bbstr); - } + if (spos != std::string_view::npos) { + std::string_view const key = argstr.substr(0, spos); + std::string_view const val = argstr.substr(spos + 1); - keyvals[std::move(key)] = std::move(val); + if (!key.empty() && !val.empty()) { + char const *const valstr = val.data(); // inherits argv's null + int64_t const bytesread = bytesFrom(valstr); + + if (blockbytesmin <= bytesread && bytesread <= blockbytesmax) { + DEBUG_LOG("Found deprecated blockbytes %" PRId64, bytesread); + blockbytes = bytesread; + } } } } - std::map::const_iterator itfind; + // standard parsing + constexpr const struct option longopts[] = { + {const_cast("blockbytes"), required_argument, nullptr, 'b'}, + {const_cast("test-blockbytes"), required_argument, nullptr, 't'}, + {const_cast("pace-errorlog"), required_argument, nullptr, 'p'}, + {const_cast("disable-errorlog"), no_argument, nullptr, 'd'}, + {nullptr, 0, nullptr, 0}, + }; + + // getopt assumes args start at '1' so this hack is needed + char *const *argvp = ((char *const *)argv - 1); + + for (;;) { + int const opt = getopt_long(argc + 1, argvp, "b:t:p:d", longopts, nullptr); + if (-1 == opt) { + break; + } - // blockbytes checked range string - itfind = keyvals.find(bbstr); - if (keyvals.end() != itfind) { - std::string val = itfind->second; - if (!val.empty()) { - int64_t const blockbytes = bytesFrom(val); + DEBUG_LOG("processing '%c' %s", (char)opt, argvp[optind - 1]); - if (blockbytes < blockbytesmin || blockbytesmax < blockbytes) { -#if !defined(SLICE_UNIT_TEST) - DEBUG_LOG("Block Bytes %" PRId64 " outside checked limits %" PRId64 "-%" PRId64, blockbytes, blockbytesmin, blockbytesmax); - DEBUG_LOG("Block Bytes kept at %" PRId64, m_blockbytes); -#endif + switch (opt) { + case 'b': { + int64_t const bytesread = bytesFrom(optarg); + if (blockbytesmin <= bytesread && bytesread <= blockbytesmax) { + DEBUG_LOG("Using blockbytes %" PRId64, bytesread); + blockbytes = bytesread; } else { -#if !defined(SLICE_UNIT_TEST) - DEBUG_LOG("Block Bytes set to %" PRId64, blockbytes); -#endif - m_blockbytes = blockbytes; + ERROR_LOG("Invalid blockbytes: %s", optarg); + } + } break; + case 't': + if (0 == blockbytes) { + int64_t const bytesread = bytesFrom(optarg); + if (0 < bytesread) { + DEBUG_LOG("Using blockbytestest %" PRId64, bytesread); + blockbytes = bytesread; + } else { + ERROR_LOG("Invalid blockbytestest: %s", optarg); + } + } else { + DEBUG_LOG("Skipping blockbytestest in favor of blockbytes"); } + break; + case 'p': { + int const secsread = atoi(optarg); + if (0 < secsread) { + m_paceerrsecs = std::min(secsread, 60); + } else { + DEBUG_LOG("Ignoring pace-errlog argument"); + } + } break; + case 'd': + m_paceerrsecs = -1; + break; + default: + break; } + } - keyvals.erase(itfind); + if (0 < blockbytes) { + DEBUG_LOG("Using configured blockbytes %" PRId64, blockbytes); + m_blockbytes = blockbytes; + } else { + DEBUG_LOG("Using default blockbytes %" PRId64, m_blockbytes); } - // bytesover unchecked range string - itfind = keyvals.find(bostr); - if (keyvals.end() != itfind) { - std::string val = itfind->second; - if (!val.empty()) { - int64_t const bytesover = bytesFrom(val); - - if (bytesover <= 0) { -#if !defined(SLICE_UNIT_TEST) - DEBUG_LOG("Bytes Over %" PRId64 " <= 0", bytesover); - DEBUG_LOG("Block Bytes kept at %" PRId64, m_blockbytes); -#endif - } else { -#if !defined(SLICE_UNIT_TEST) - DEBUG_LOG("Block Bytes set to %" PRId64, bytesover); -#endif - m_blockbytes = bytesover; - } - } - keyvals.erase(itfind); + if (m_paceerrsecs < 0) { + DEBUG_LOG("Block stitching error logs disabled"); + } else if (0 == m_paceerrsecs) { + DEBUG_LOG("Block stitching error logs enabled"); + } else { + DEBUG_LOG("Block stitching error logs at most every %d sec(s)", m_paceerrsecs); } - for (std::map::const_iterator itkv(keyvals.cbegin()); keyvals.cend() != itkv; ++itkv) { -#if !defined(SLICE_UNIT_TEST) - ERROR_LOG("Unhandled pparam %s", itkv->first.c_str()); -#endif + return true; +} + +bool +Config::canLogError() +{ + std::lock_guard const guard(m_mutex); + + if (m_paceerrsecs < 0) { + return false; + } else if (0 == m_paceerrsecs) { + return true; } +#if !defined(UNITTEST) + TSHRTime const timenow = TShrtime(); + if (timenow < m_nextlogtime) { + return false; + } + + m_nextlogtime = timenow + TS_HRTIME_SECONDS(m_paceerrsecs); +#else + m_nextlogtime = 0; // thanks clang +#endif + return true; } diff --git a/plugins/experimental/slice/Config.h b/plugins/experimental/slice/Config.h index e107f3f0167..8c4ab2498fa 100644 --- a/plugins/experimental/slice/Config.h +++ b/plugins/experimental/slice/Config.h @@ -20,21 +20,27 @@ #include "slice.h" -#include +#include // Data Structures and Classes struct Config { - static int64_t const blockbytesmin = 1024 * 256; // 256KB - static int64_t const blockbytesmax = 1024 * 1024 * 32; // 32MB - static int64_t const blockbytesdefault = 1024 * 1024; // 1MB - - static constexpr char const *const blockbytesstr = "blockbytes"; - static constexpr char const *const bytesoverstr = "bytesover"; + static constexpr int64_t const blockbytesmin = 1024 * 256; // 256KB + static constexpr int64_t const blockbytesmax = 1024 * 1024 * 32; // 32MB + static constexpr int64_t const blockbytesdefault = 1024 * 1024; // 1MB int64_t m_blockbytes{blockbytesdefault}; + int m_paceerrsecs{0}; // -1 disable logging, 0 no pacing, max 60s + + // Convert optarg to bytes + static int64_t bytesFrom(char const *const valstr); + + // Parse from args, ast one wins + bool fromArgs(int const argc, char const *const argv[]); - // Last one wins - bool fromArgs(int const argc, char const *const argv[], char *const errbuf, int const errbuf_size); + // Check if the error should can be logged, if sucessful may update m_nexttime + bool canLogError(); - static int64_t bytesFrom(std::string const &valstr); +private: + TSHRTime m_nextlogtime{0}; // next time to log in ns + std::mutex m_mutex; }; diff --git a/plugins/experimental/slice/Data.h b/plugins/experimental/slice/Data.h index f7707196c72..45383522103 100644 --- a/plugins/experimental/slice/Data.h +++ b/plugins/experimental/slice/Data.h @@ -20,10 +20,10 @@ #include "ts/ts.h" +#include "Config.h" #include "HttpHeader.h" #include "Range.h" #include "Stage.h" -#include "slice.h" #include @@ -35,7 +35,8 @@ struct Data { Data(Data const &) = delete; Data &operator=(Data const &) = delete; - int64_t const m_blockbytes_config; // configured slice block size + Config *const m_config; + sockaddr_storage m_client_ip; // for pristine url coming in @@ -76,8 +77,8 @@ struct Data { TSHttpParser m_http_parser{nullptr}; //!< cached for reuse - explicit Data(int64_t const blockbytes) - : m_blockbytes_config(blockbytes), + explicit Data(Config *const config) + : m_config(config), m_client_ip(), m_urlbuffer(nullptr), m_urlloc(nullptr), diff --git a/plugins/experimental/slice/Makefile.inc b/plugins/experimental/slice/Makefile.inc index 2f0f8f4b458..dbac02b4940 100644 --- a/plugins/experimental/slice/Makefile.inc +++ b/plugins/experimental/slice/Makefile.inc @@ -43,21 +43,21 @@ experimental_slice_slice_la_SOURCES = \ check_PROGRAMS += experimental/slice/test_content_range -experimental_slice_test_content_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DSLICE_UNIT_TEST +experimental_slice_test_content_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST experimental_slice_test_content_range_SOURCES = \ experimental/slice/unit-tests/test_content_range.cc \ experimental/slice/ContentRange.cc check_PROGRAMS += experimental/slice/test_range -experimental_slice_test_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DSLICE_UNIT_TEST +experimental_slice_test_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST experimental_slice_test_range_SOURCES = \ experimental/slice/unit-tests/test_range.cc \ experimental/slice/Range.cc check_PROGRAMS += experimental/slice/test_config -experimental_slice_test_config_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DSLICE_UNIT_TEST +experimental_slice_test_config_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST experimental_slice_test_config_SOURCES = \ experimental/slice/unit-tests/test_config.cc \ experimental/slice/Config.cc diff --git a/plugins/experimental/slice/README.md b/plugins/experimental/slice/README.md index dc56997ce56..ec3d1bc420f 100644 --- a/plugins/experimental/slice/README.md +++ b/plugins/experimental/slice/README.md @@ -18,24 +18,86 @@ To enable the plugin, specify the plugin library via @plugin at the end of a remap line as follows (2MB slice in this example): ``` -map http://ats-cache/ http://parent/ @plugin=slice.so @pparam=blockbytes:2097152 @plugin=cache_range_requests.so +map http://ats-cache/ http://parent/ @plugin=slice.so @pparam=--blockbytes=2M @plugin=cache_range_requests.so +``` + +alternatively + +``` +map http://ats-cache/ http://parent/ @plugin=slice.so @pparam=-b @pparam=2M @plugin=cache_range_requests.so ``` for global plugins. ``` -slice.so blockbytes:2097152 +slice.so --blockbytes=2097152 cache_range_requests.so ``` -**Note**: cache_range_requests **MUST** follow slice.so Put these plugins at the end of the plugin list -**Note**: blockbytes is defined in bytes. 1048576 (1MB) is the default. +alternatively: + +``` +slice.so -b 2M +cache_range_requests.so +``` + +Options for the slice plugin (typically last one wins): +``` +--blockbytes= (optional) + Slice block size. + Default is 1m or 1048576 bytes. + also -b + Suffix k,m,g supported. + Limited to 32k and 32m inclusive. + For backwards compatibility blockbytes: is also supported. + +--test-blockbytes= (optional) + Slice block size for testing. + also -t + Suffix k,m,g supported. + Limited to any positive number. + Ignored if --blockbytes is provided. + +--pace-errorlog= (optional) + Limit stitching error logs to every 'n' second(s) + Default is to log all errors (no pacing). + also -p + +--disable-errorlog (optional) + Disable writing stitching errors to the error log. + also -d +``` + +**Note**: cache_range_requests **MUST** follow slice.so Put these plugins +at the end of the plugin list + +**Note**: blockbytes is defined in bytes. Postfix for 'K', 'M' and 'G' +may be used. 1048576 (1MB) is the default. For testing purposes an unchecked value of "bytesover" is also available. -Debug output can be enable by setting the debug tag: **slice** +Debug output can be enable by setting the debug tag: **slice**. If debug +is enabled all block stitch errors will log to diags.log + +The slice plugin is susceptible to block stitching errors caused by +mismatched blocks. For these cases special detailed error logs are +provided to help with debugging. Below is a sample error log entry:: -Debug messages related to object instance construction/deconstruction, see slice.h. +``` +[Apr 19 20:26:13.639] [ET_NET 17] ERROR: [slice] 1555705573.639 reason="Non 206 internal block response" uri="http://localhost:18080/%7Ep.tex/%7Es.50M/%7Eui.20000/" uas="curl/7.29.0" req_range="bytes=1000000-" norm_range="bytes 1000000-52428799/52428800" etag_exp="%221603934496%22" lm_exp="Fri, 19 Apr 2019 18:53:20 GMT" blk_range="21000000-21999999" status_got="400" cr_got="" etag_got="" lm_got="" cc="no-store" via="" +``` + +Current error types logged: +``` + Mismatch block Etag + Mismatch block Last-Modified + Non 206 internal block response + Mismatch/Bad block Content-Range +``` + + +With slice error logs disabled these type errors can typically be detected +by observing crc=ERR_READ_ERROR and pscl=0 in normal logs. At the current time only single range requests or the first part of a multi part range request of the forms: @@ -46,7 +108,6 @@ Range: bytes=- ``` are supported as multi part range responses are non trivial to implement. This matches with the cache_range_requests.so plugin capability. - --- Important things to note: diff --git a/plugins/experimental/slice/client.cc b/plugins/experimental/slice/client.cc index 75bbe8e434a..7076f6dcd45 100644 --- a/plugins/experimental/slice/client.cc +++ b/plugins/experimental/slice/client.cc @@ -34,8 +34,8 @@ shutdown(TSCont const contp, Data *const data) bool requestBlock(TSCont contp, Data *const data) { - int64_t const blockbeg = (data->m_blockbytes_config * data->m_blocknum); - Range blockbe(blockbeg, blockbeg + data->m_blockbytes_config); + int64_t const blockbeg = (data->m_config->m_blockbytes * data->m_blocknum); + Range blockbe(blockbeg, blockbeg + data->m_config->m_blockbytes); char rangestr[1024]; int rangelen = sizeof(rangestr); @@ -131,7 +131,7 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) data->m_statustype = TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE; // First block will give Content-Length - rangebe = Range(0, data->m_blockbytes_config); + rangebe = Range(0, data->m_config->m_blockbytes); } } else { DEBUG_LOG("Full content request"); @@ -143,7 +143,7 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) } // set to the first block in range - data->m_blocknum = rangebe.firstBlockFor(data->m_blockbytes_config); + data->m_blocknum = rangebe.firstBlockFor(data->m_config->m_blockbytes); data->m_req_range = rangebe; // remove ATS keys to avoid 404 loop diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc index 9371469e367..37016de753e 100644 --- a/plugins/experimental/slice/server.cc +++ b/plugins/experimental/slice/server.cc @@ -186,6 +186,15 @@ handleFirstServerHeader(Data *const data, TSCont const contp) void logSliceError(char const *const message, Data const *const data, HttpHeader const &header_resp) { + Config *const config = data->m_config; + + bool const logToError = config->canLogError(); + + // always write block stitch errors while in debug mode + if (!logToError && !TSIsDebugTagSet(PLUGIN_NAME)) { + return; + } + HttpHeader const header_req(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); TSHRTime const timenowus = TShrtime(); @@ -222,8 +231,8 @@ logSliceError(char const *const message, Data const *const data, HttpHeader cons crange.toStringClosed(normstr, &normlen); // block range request - int64_t const blockbeg = data->m_blocknum * data->m_blockbytes_config; - int64_t const blockend = std::min(blockbeg + data->m_blockbytes_config, data->m_contentlen); + int64_t const blockbeg = data->m_blocknum * data->m_config->m_blockbytes; + int64_t const blockend = std::min(blockbeg + data->m_config->m_blockbytes, data->m_contentlen); // Block response data TSHttpStatus const statusgot = header_resp.status(); @@ -261,26 +270,28 @@ logSliceError(char const *const message, Data const *const data, HttpHeader cons size_t etaggotlen = sizeof(etaggotstr); TSStringPercentEncode(etagstr, etaglen, etaggotstr, etaggotlen, &etaggotlen, nullptr); - TSError("[%s] %" PRId64 ".%" PRId64 " reason=\"%s\"" - " uri=\"%.*s\"" - " uas=\"%.*s\"" - " req_range=\"%.*s\"" - " norm_range=\"%.*s\"" - - " etag_exp=\"%.*s\"" - " lm_exp=\"%.*s\"" - - " blk_range=\"%" PRId64 "-%" PRId64 "\"" - - " status_got=\"%d\"" - " cr_got=\"%.*s\"" - " etag_got=\"%.*s\"" - " lm_got=\"%.*s\"" - " cc=\"%.*s\"" - " via=\"%.*s\"", - PLUGIN_NAME, secs, ms, message, (int)urlplen, urlpstr, uaslen, uasstr, rangelen, rangestr, normlen, normstr, - (int)etagexplen, etagexpstr, data->m_lastmodifiedlen, data->m_lastmodified, blockbeg, blockend - 1, statusgot, crlen, - crstr, (int)etaggotlen, etaggotstr, lmlen, lmstr, cclen, ccstr, vialen, viastr); + DEBUG_LOG("Logging Block Stitch error"); + + ERROR_LOG("%" PRId64 ".%" PRId64 " reason=\"%s\"" + " uri=\"%.*s\"" + " uas=\"%.*s\"" + " req_range=\"%.*s\"" + " norm_range=\"%.*s\"" + + " etag_exp=\"%.*s\"" + " lm_exp=\"%.*s\"" + + " blk_range=\"%" PRId64 "-%" PRId64 "\"" + + " status_got=\"%d\"" + " cr_got=\"%.*s\"" + " etag_got=\"%.*s\"" + " lm_got=\"%.*s\"" + " cc=\"%.*s\"" + " via=\"%.*s\"", + secs, ms, message, (int)urlplen, urlpstr, uaslen, uasstr, rangelen, rangestr, normlen, normstr, (int)etagexplen, + etagexpstr, data->m_lastmodifiedlen, data->m_lastmodified, blockbeg, blockend - 1, statusgot, crlen, crstr, + (int)etaggotlen, etaggotstr, lmlen, lmstr, cclen, ccstr, vialen, viastr); } bool @@ -378,7 +389,7 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) } // how much to fast forward into this data block - data->m_blockskip = data->m_req_range.skipBytesForBlock(data->m_blockbytes_config, data->m_blocknum); + data->m_blockskip = data->m_req_range.skipBytesForBlock(data->m_config->m_blockbytes, data->m_blocknum); } transfer_content_bytes(data); @@ -424,13 +435,13 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) // issues a speculative request for the first block // in that case fast forward to the real first in range block // Btw this isn't implemented yet, to be handled - int64_t const firstblock(data->m_req_range.firstBlockFor(data->m_blockbytes_config)); + int64_t const firstblock(data->m_req_range.firstBlockFor(data->m_config->m_blockbytes)); if (data->m_blocknum < firstblock) { data->m_blocknum = firstblock; } // done processing blocks? - if (!data->m_req_range.blockIsInside(data->m_blockbytes_config, data->m_blocknum)) { + if (!data->m_req_range.blockIsInside(data->m_config->m_blockbytes, data->m_blocknum)) { data->m_blocknum = -1; // signal value no more blocks } } else { diff --git a/plugins/experimental/slice/slice.cc b/plugins/experimental/slice/slice.cc index b0c0e1aba50..aab8b7d3f4c 100644 --- a/plugins/experimental/slice/slice.cc +++ b/plugins/experimental/slice/slice.cc @@ -33,7 +33,7 @@ namespace Config globalConfig; bool -read_request(TSHttpTxn txnp, Config const *const config) +read_request(TSHttpTxn txnp, Config *const config) { DEBUG_LOG("slice read_request"); TxnHdrMgr hdrmgr; @@ -55,7 +55,8 @@ read_request(TSHttpTxn txnp, Config const *const config) return false; } - Data *const data = new Data(config->m_blockbytes); + TSAssert(nullptr != config); + Data *const data = new Data(config); // set up feedback connect if (AF_INET == ip->sa_family) { @@ -150,11 +151,11 @@ TSRemapOSResponse(void *ih, TSHttpTxn rh, int os_response_type) SLICE_EXPORT TSReturnCode -TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) +TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int /* errbuf_size */) { Config *const config = new Config; if (2 < argc) { - config->fromArgs(argc - 2, argv + 2, errbuf, errbuf_size); + config->fromArgs(argc - 2, argv + 2); } *ih = static_cast(config); return TS_SUCCESS; @@ -195,7 +196,7 @@ TSPluginInit(int argc, char const *argv[]) } if (1 < argc) { - globalConfig.fromArgs(argc - 1, argv + 1, nullptr, 0); + globalConfig.fromArgs(argc - 1, argv + 1); } TSCont const contp(TSContCreate(global_read_request_hook, nullptr)); diff --git a/plugins/experimental/slice/slice.h b/plugins/experimental/slice/slice.h index 0cb2523b18d..682c0168c5a 100644 --- a/plugins/experimental/slice/slice.h +++ b/plugins/experimental/slice/slice.h @@ -30,10 +30,9 @@ #define PLUGIN_NAME "slice" #endif -#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) - #if !defined(UNITTEST) +#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) #define DEBUG_LOG(fmt, ...) \ TSDebug(PLUGIN_NAME, "[%s:%04d] %s(): " fmt, __FILENAME__, __LINE__, __func__, \ ##__VA_ARGS__) /* \ @@ -48,6 +47,7 @@ TSDebug(PLUGIN_NAME, "[%s:%04d] %s(): " fmt, __FILENAME__, __LINE__, __func__, ##__VA_ARGS__) #else + #define DEBUG_LOG(fmt, ...) #define ERROR_LOG(fmt, ...) diff --git a/plugins/experimental/slice/unit-tests/test_config.cc b/plugins/experimental/slice/unit-tests/test_config.cc index e7467cc2701..fcd1b8e051c 100644 --- a/plugins/experimental/slice/unit-tests/test_config.cc +++ b/plugins/experimental/slice/unit-tests/test_config.cc @@ -25,6 +25,8 @@ #include "../Config.h" #include "catch.hpp" /* catch unit-test framework */ +#include + TEST_CASE("config default", "[AWS][slice][utility]") { Config const config; @@ -34,9 +36,13 @@ TEST_CASE("config default", "[AWS][slice][utility]") TEST_CASE("config bytesfrom valid parsing", "[AWS][slice][utility]") { - std::vector const teststrings = {"1000", "1m", "5g", "2k", "3kb", "1z"}; + static std::array const teststrings = { + "1000", "1m", "5g", "2k", "3kb", "1z", + }; - std::vector const expvals = {1000, 1024 * 1024, int64_t(1024) * 1024 * 1024 * 5, 1024 * 2, 1024 * 3, 1}; + constexpr std::array const expvals = { + 1000, 1024 * 1024, int64_t(1024) * 1024 * 1024 * 5, 1024 * 2, 1024 * 3, 1, + }; for (size_t index = 0; index < teststrings.size(); ++index) { std::string const &teststr = teststrings[index]; @@ -52,12 +58,12 @@ TEST_CASE("config bytesfrom valid parsing", "[AWS][slice][utility]") TEST_CASE("config bytesfrom invalid parsing", "[AWS][slice][utility]") { - std::vector const badstrings = { - "abc", // alpha - "g00", // giga - "M00", // mega - "k00", // kilo - "-500" // negative + static std::array const badstrings = { + "abc", // alpha + "g00", // giga + "M00", // mega + "k00", // kilo + "-500", // negative }; for (std::string const &badstr : badstrings) { diff --git a/tests/gold_tests/pluginTest/slice/slice.test.py b/tests/gold_tests/pluginTest/slice/slice.test.py index a3e299a29d5..8b57d48f526 100644 --- a/tests/gold_tests/pluginTest/slice/slice.test.py +++ b/tests/gold_tests/pluginTest/slice/slice.test.py @@ -117,7 +117,7 @@ remap_config_path = ts.Disk.remap_config.Name tr.Disk.File(remap_config_path, typename="ats:config").AddLines([ 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + - ' @plugin=slice.so @pparam=bytesover:{}'.format(block_bytes) + ' @plugin=slice.so @pparam=--test-blockbytes={}'.format(block_bytes) ]) tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/slice/slice_error.test.py b/tests/gold_tests/pluginTest/slice/slice_error.test.py index f6fd4d50627..53cf194f26d 100644 --- a/tests/gold_tests/pluginTest/slice/slice_error.test.py +++ b/tests/gold_tests/pluginTest/slice/slice_error.test.py @@ -245,12 +245,12 @@ # set up whole asset fetch into cache ts.Disk.remap_config.AddLine( 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + - ' @plugin=slice.so @pparam=bytesover:{}'.format(blockbytes) + ' @plugin=slice.so @pparam=--test-blockbytes={}'.format(blockbytes) ) # minimal configuration ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.enabled': 0, 'proxy.config.diags.debug.tags': 'slice', 'proxy.config.http.cache.http': 0, 'proxy.config.http.wait_for_cache': 0, From 82d564486e058ead160bfb8ad15e40b0507ff8aa Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 22 Apr 2019 14:11:02 +0800 Subject: [PATCH 493/526] Purges log collation feature This implements the deprecation noted in 26b30870d656eb26d752b7f7f841dbfb6e93c970 --- doc/.tx/config | 5 - doc/admin-guide/files/logging.yaml.en.rst | 7 - doc/admin-guide/files/records.config.en.rst | 99 +-- doc/admin-guide/logging/destinations.en.rst | 137 +--- doc/admin-guide/logging/formatting.en.rst | 23 - doc/admin-guide/logging/rotation.en.rst | 3 +- doc/admin-guide/performance/index.en.rst | 1 - include/ts/ts.h | 3 +- iocore/cache/Makefile.am | 1 - iocore/eventsystem/I_Event.h | 1 - lib/perl/lib/Apache/TS/AdminClient.pm | 7 - mgmt/RecordsConfig.cc | 22 +- proxy/README-stats.otl | 1 - proxy/logging/Log.cc | 263 +------ proxy/logging/Log.h | 12 +- proxy/logging/LogBuffer.cc | 1 - proxy/logging/LogCollationAccept.cc | 105 --- proxy/logging/LogCollationAccept.h | 42 -- proxy/logging/LogCollationBase.h | 45 -- proxy/logging/LogCollationClientSM.cc | 716 ------------------- proxy/logging/LogCollationClientSM.h | 116 ---- proxy/logging/LogCollationHostSM.cc | 527 -------------- proxy/logging/LogCollationHostSM.h | 109 --- proxy/logging/LogConfig.cc | 196 +----- proxy/logging/LogConfig.h | 29 +- proxy/logging/LogFile.cc | 2 - proxy/logging/LogFile.h | 1 - proxy/logging/LogFilter.cc | 1 - proxy/logging/LogFormat.cc | 27 - proxy/logging/LogHost.cc | 466 ------------- proxy/logging/LogHost.h | 165 ----- proxy/logging/LogObject.cc | 107 +-- proxy/logging/LogObject.h | 61 +- proxy/logging/LogSock.cc | 734 -------------------- proxy/logging/LogSock.h | 128 ---- proxy/logging/Makefile.am | 17 +- proxy/logging/YamlLogConfig.cc | 56 +- proxy/shared/UglyLogStubs.cc | 16 - src/traffic_logcat/logcat.cc | 2 - src/traffic_server/InkAPI.cc | 7 +- src/traffic_server/Makefile.inc | 1 - 41 files changed, 97 insertions(+), 4165 deletions(-) delete mode 100644 proxy/logging/LogCollationAccept.cc delete mode 100644 proxy/logging/LogCollationAccept.h delete mode 100644 proxy/logging/LogCollationBase.h delete mode 100644 proxy/logging/LogCollationClientSM.cc delete mode 100644 proxy/logging/LogCollationClientSM.h delete mode 100644 proxy/logging/LogCollationHostSM.cc delete mode 100644 proxy/logging/LogCollationHostSM.h delete mode 100644 proxy/logging/LogHost.cc delete mode 100644 proxy/logging/LogHost.h delete mode 100644 proxy/logging/LogSock.cc delete mode 100644 proxy/logging/LogSock.h diff --git a/doc/.tx/config b/doc/.tx/config index be18c3c2c94..6640ca1228c 100644 --- a/doc/.tx/config +++ b/doc/.tx/config @@ -162,11 +162,6 @@ file_filter = locale//LC_MESSAGES/admin-guide/monitoring/logging/log-build source_file = _build/locale/pot/admin-guide/monitoring/logging/log-builder.en.pot source_lang = en -[apache-traffic-server-6x.admin-guide--monitoring--logging--log-collation_en] -file_filter = locale//LC_MESSAGES/admin-guide/monitoring/logging/log-collation.en.po -source_file = _build/locale/pot/admin-guide/monitoring/logging/log-collation.en.pot -source_lang = en - [apache-traffic-server-6x.admin-guide--monitoring--logging--log-formats_en] file_filter = locale//LC_MESSAGES/admin-guide/monitoring/logging/log-formats.en.po source_file = _build/locale/pot/admin-guide/monitoring/logging/log-formats.en.pot diff --git a/doc/admin-guide/files/logging.yaml.en.rst b/doc/admin-guide/files/logging.yaml.en.rst index a301aa5982c..96a6fe3d12a 100644 --- a/doc/admin-guide/files/logging.yaml.en.rst +++ b/doc/admin-guide/files/logging.yaml.en.rst @@ -288,13 +288,6 @@ rolling_min_count number Specifies the minimum number of rolled logs t filters array of The optional list of filter objects which filters restrict the individual events logged. The array may only contain one accept filter. -collation_hosts array of If present, one or more strings specifying the - strings log collation hosts to which logs should be - delivered, each in the form of ":". - :ref:`admin-logging-collation` for more - information. NOTE: This is a deprecated feature, - which will be removed in ATS v9.0.0. See the - logging documentation (above) for more details. ====================== =========== ================================================= Enabling log rolling may be done globally in :file:`records.config`, or on a diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index d9120b21710..080a99bff17 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -2765,25 +2765,7 @@ Logging Configuration .. note:: All files in the logging directory contribute to the space used, - even if they are not log files. In collation client mode, if - there is no local disk logging, or - :ts:cv:`proxy.config.log.max_space_mb_for_orphan_logs` is set - to a higher value than :ts:cv:`proxy.config.log.max_space_mb_for_logs`, - |TS| will take :ts:cv:`proxy.config.log.max_space_mb_for_orphan_logs` - for maximum allowed log space. - -.. ts:cv:: CONFIG proxy.config.log.max_space_mb_for_orphan_logs INT 25 - :units: megabytes - :reloadable: - - The amount of space allocated to the logging directory (in MB) if this node is acting as a collation client. - -.. note:: - - When max_space_mb_for_orphan_logs is take as the maximum allowed log space in the logging system, the same rule apply - to proxy.config.log.max_space_mb_for_logs also apply to proxy.config.log.max_space_mb_for_orphan_logs, ie: All files - in the logging directory contribute to the space used, even if they are not log files. you may need to consider this - when you enable full remote logging, and bump to the same size as proxy.config.log.max_space_mb_for_logs. + even if they are not log files. .. ts:cv:: CONFIG proxy.config.log.max_space_mb_headroom INT 1000 :units: megabytes @@ -2826,85 +2808,6 @@ Logging Configuration others, even if specified in the configuration file. Permissions for existing log files are not changed when the configuration is modified. -.. ts:cv:: LOCAL proxy.local.log.collation_mode INT 0 - :reloadable: - :deprecated: - - Set the log collation mode. - - ===== ====================================================================== - Value Effect - ===== ====================================================================== - ``0`` Log collation is disabled. - ``1`` This host is a log collation server. - ``2`` This host is a collation client and sends entries using standard - formats to the collation server. - ``3`` This host is a collation client and sends entries using the - traditional custom formats to the collation server. - ``4`` This host is a collation client and sends entries that use both the - standard and traditional custom formats to the collation server. - ===== ====================================================================== - - For information on sending custom formats to the collation server, - refer to :ref:`admin-logging-collating-custom-formats` and - :file:`logging.yaml`. - -.. note:: - - Log collation is a *deprecated* feature as of ATS v8.0.0, and will be - removed in ATS v9.0.0. Our recommendation is to use one of the many existing - log collection tools, such as Kafka, LogStash, FileBeat, Fluentd or even - syslog / syslog-ng. - -.. ts:cv:: CONFIG proxy.config.log.collation_host STRING NULL - :deprecated: - - The hostname of the log collation server. - -.. ts:cv:: CONFIG proxy.config.log.collation_port INT 8085 - :reloadable: - :deprecated: - - The port used for communication between the collation server and client. - -.. ts:cv:: CONFIG proxy.config.log.collation_secret STRING foobar - :reloadable: - :deprecated: - - The password used to validate logging data and prevent the exchange of unauthorized information when a collation server is being used. - -.. ts:cv:: CONFIG proxy.config.log.collation_host_tagged INT 0 - :reloadable: - :deprecated: - - When enabled (``1``), configures |TS| to include the hostname of the collation client that generated the log entry in each entry. - -.. ts:cv:: CONFIG proxy.config.log.collation_retry_sec INT 5 - :reloadable: - :deprecated: - - The number of seconds between collation server connection retries. - -.. ts:cv:: CONFIG proxy.config.log.collation_host_timeout INT 86390 - :deprecated: - - The number of seconds before inactivity time-out events for the host side. - This setting over-rides the default set with proxy.config.net.default_inactivity_timeout - for log collation connections. - - The default is set for 10s less on the host side to help prevent any possible race - conditions. If the host disconnects first, the client will see the disconnect - before its own time-out and re-connect automatically. If the client does not see - the disconnect, i.e., connection is "locked-up" for some reason, it will disconnect - when it reaches its own time-out and then re-connect automatically. - -.. ts:cv:: CONFIG proxy.config.log.collation_client_timeout INT 86400 - :deprecated: - - The number of seconds before inactivity time-out events for the client side. - This setting over-rides the default set with proxy.config.net.default_inactivity_timeout - for log collation connections. - .. ts:cv:: CONFIG proxy.config.log.rolling_enabled INT 1 :reloadable: diff --git a/doc/admin-guide/logging/destinations.en.rst b/doc/admin-guide/logging/destinations.en.rst index f8521dea2b9..5b70913ad64 100644 --- a/doc/admin-guide/logging/destinations.en.rst +++ b/doc/admin-guide/logging/destinations.en.rst @@ -31,8 +31,7 @@ Two classes of destinations are provided by |TS| currently: local and remote. Local logging involves storing log data onto filesystems locally mounted on the same system as the |TS| processes themselves and are covered below in :ref:`admin-logging-destinations-local`, while remote logging options involving -:manpage:`syslog` and built-in |TS| log collation, are covered below in -:ref:`admin-logging-destinations-remote`. +:manpage:`syslog`, are covered below in :ref:`admin-logging-destinations-remote`. .. _admin-logging-destinations-local: @@ -44,9 +43,8 @@ Local Logging Log Directory Configuration --------------------------- -All local logging output (including incoming collation logs on a |TS| instance -configured as a :ref:`admin-logging-collation-server`) are stored within a -single base directory. Individual log file configurations may optionally append +All local logging output is stored within a single base directory. +Individual log file configurations may optionally append subdirectories to this base path. This location is adjusted with :ts:cv:`proxy.config.log.logfile_dir` in :file:`records.config`. @@ -196,132 +194,3 @@ system and emergency logs. Sending custom event or transaction error logs to syslog is not directly supported. You may use external log aggregation tools, such as Logstash, to accomplish this by having them handle the ingestion of |TS| local log files and forwarding to whatever receivers you wish. - -.. _admin-logging-collation: - -Log Collation -------------- - -.. note:: - - Log collation is a *deprecated* feature as of ATS v8.0.0, and will be - removed in ATS v9.0.0. Our recommendation is to use one of the many existing - log collection tools, such as Kafka, LogStash, FileBeat, Fluentd or even - syslog / syslog-ng (see above). - -|TS| offers remote log shipping natively through the log collation feature, -which allows one or more |TS| instances handling regular traffic to transmit -their log data to one or more |TS| instances acting as collation servers. - -This allows you to centralize your |TS| logging for (potentially) easier -analysis and reporting in environments with many |TS| instances. Collation -servers may be |TS| instance running a stripped down configuration aimed -at log collation only (and omitting any configuration for actual traffic -proxying or caching). - -When a |TS| node generates a buffer of event log entries, it first determines -if it is the collation server or a collation client. The collation server node -writes all log buffers to its local disk (as per its :file:`logging.yaml` -configuration), just as it would if log collation was not enabled. The -collation client nodes prepare their log buffers for transfer across the -network and send the buffers to the configured log collation server. - -If log clients cannot contact their log collation server, then they write their -log buffers to their local disks, into orphan log files. Orphaned log files -require manual collation. - -.. important:: - - Log collation can have an impact on network performance. Because all nodes - are forwarding their log data buffers to the single collation server, a - bottleneck can occur. In addition, collated log files contain timestamp - information for each entry, but entries in the files do not appear in strict - chronological order. You may want to sort collated log files before doing - analysis. - -.. _admin-logging-collation-client: - -Collation Client -~~~~~~~~~~~~~~~~ - -To configure a |TS| node to be a collation client, follow the steps below. - -#. In the :file:`records.config` file, edit the following variables: - - - :ts:cv:`proxy.local.log.collation_mode`: ``2`` to configure this node as - log collation client and send standard formatted log entries to the - collation server. For custom log entries, see :file:`logging.yaml`. - - :ts:cv:`proxy.config.log.collation_host` - - :ts:cv:`proxy.config.log.collation_port` - - :ts:cv:`proxy.config.log.collation_secret` - - :ts:cv:`proxy.config.log.collation_host_tagged` - - :ts:cv:`proxy.config.log.max_space_mb_for_orphan_logs` - -#. Run the command :option:`traffic_ctl config reload` to apply the configuration - changes. - -.. note:: - - If you modify the collation port or secret after connections between the - collation server and collation clients have been established, you must - restart |TS| on all nodes. - -.. _admin-logging-collation-server: - -Collation Server -~~~~~~~~~~~~~~~~ - -To configure a |TS| node to be a collation server, perform the following -configuration adjustments in :file:`records.config`: - -#. Set :ts:cv:`proxy.local.log.collation_mode` to ``1`` to indicate this node - will be a server. :: - - CONFIG proxy.local.log.collation_mode INT 1 - -#. Configure the port on which the server will listen to incoming collation - transfers from clients, using :ts:cv:`proxy.config.log.collation_port`. If - omitted, this defaults to port ``8085``. :: - - CONFIG proxy.config.log.collation_port INT 8085 - -#. Configure the shared secret used by collation clients to authenticate their - sessions, using :ts:cv:`proxy.config.log.collation_secret`. :: - - CONFIG proxy.config.log.collation_secret STRING seekrit - -#. Run the command :option:`traffic_ctl config reload` to apply the configuration - changes. - -.. note:: - - If you modify the collation port or secret after connections between the - collation server and collation clients have been established, you must - restart |TS| on all nodes. - -.. _admin-logging-collating-custom-formats: - -Collating Custom Logs -~~~~~~~~~~~~~~~~~~~~~ - -If you use custom event log files, then you must edit :file:`logging.yaml`, -in addition to configuring a collation server and collation clients. - -To collate custom event log files: - -#. On each collation client, edit :file:`logging.yaml` and add the - ``CollationHosts`` attribute to the relevant logs. For example, adding two - collation hosts to an ASCII log that uses the Squid format would look like: - - .. code:: yaml - - logs: - - mode: ascii - format: squid - filename: squid - collationhosts: - - 192.168.1.100:4567 - - 192.168.1.101:4567 - -#. Run the command :option:`traffic_ctl config reload` to restart |TS|. - diff --git a/doc/admin-guide/logging/formatting.en.rst b/doc/admin-guide/logging/formatting.en.rst index 1217663d629..bb5ccb79760 100644 --- a/doc/admin-guide/logging/formatting.en.rst +++ b/doc/admin-guide/logging/formatting.en.rst @@ -89,7 +89,6 @@ down into the following broad categories for (hopefully) easier reference: - :ref:`admin-logging-fields-methods` - :ref:`admin-logging-fields-ids` - :ref:`admin-logging-fields-lengths` -- :ref:`admin-logging-fields-collation` - :ref:`admin-logging-fields-network` - :ref:`admin-logging-fields-plugin` - :ref:`admin-logging-fields-proto` @@ -447,28 +446,6 @@ ssql Origin Response Content body and header length combined of the origin server response to |TS|. ===== ====================== ================================================== -.. _admin-logging-fields-collation: - -Log Collation -~~~~~~~~~~~~~ - -.. _phn: -.. _phi: - -Logging fields related to :ref:`admin-logging-collation`. - -===== ====== ================================================================== -Field Source Description -===== ====== ================================================================== -phn Proxy Hostname of the |TS| node which generated the collated log entry. -phi Proxy IP of the |TS| node which generated the collated log entry. -===== ====== ================================================================== - -.. note:: - - Log collation is a *deprecated* feature as of ATS v8.0.0, and will be - removed in ATS v9.0.0. - .. _admin-logging-fields-network: Network Addresses, Ports, and Interfaces diff --git a/doc/admin-guide/logging/rotation.en.rst b/doc/admin-guide/logging/rotation.en.rst index 01441c86299..eaed409ff48 100644 --- a/doc/admin-guide/logging/rotation.en.rst +++ b/doc/admin-guide/logging/rotation.en.rst @@ -67,8 +67,7 @@ the following information: - The original log file's name (such as ``access.log``). -- The hostname of the |TS| node that generated the log file (useful in |TS| - log collation configurations). +- The hostname of the |TS| node that generated the log file. - Two timestamps separated by a hyphen (``-``). The first timestamp is a *lower bound* for the timestamp of the first record in the log diff --git a/doc/admin-guide/performance/index.en.rst b/doc/admin-guide/performance/index.en.rst index 18f05dc6fc1..acd5c3d542c 100644 --- a/doc/admin-guide/performance/index.en.rst +++ b/doc/admin-guide/performance/index.en.rst @@ -521,7 +521,6 @@ Logging Configuration binary vs. ascii output multiple log formats (netscape+squid+custom vs. just custom) - overhead to log collation using direct writes vs. syslog target Plugin Tuning diff --git a/include/ts/ts.h b/include/ts/ts.h index d5ee8b0dc54..b15641449c4 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -2087,8 +2087,7 @@ tsapi void TSRecordDump(int rec_type, TSRecordDumpCb callback, void *edata); Creates a new custom log file that your plugin can write to. You can design the fields and inputs to the log file using the TSTextLogObjectWrite() function. The logs you create are treated - like ordinary logs; they are rolled if log rolling is enabled. (Log - collation is not supported though). + like ordinary logs; they are rolled if log rolling is enabled. @param filename new log file being created. The new log file is created in the logs directory. You can specify a path to a diff --git a/iocore/cache/Makefile.am b/iocore/cache/Makefile.am index b5d7c5d9729..b2161f0b322 100644 --- a/iocore/cache/Makefile.am +++ b/iocore/cache/Makefile.am @@ -99,7 +99,6 @@ test_LDADD = \ $(top_builddir)/iocore/hostdb/libinkhostdb.a \ $(top_builddir)/proxy/logging/liblogging.a \ $(top_builddir)/proxy/hdrs/libhdrs.a \ - $(top_builddir)/proxy/logging/liblogcollation.a \ $(top_builddir)/proxy/shared/libdiagsconfig.a \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/iocore/utils/libinkutils.a \ diff --git a/iocore/eventsystem/I_Event.h b/iocore/eventsystem/I_Event.h index 83c1bd0a3b6..3f708db0f16 100644 --- a/iocore/eventsystem/I_Event.h +++ b/iocore/eventsystem/I_Event.h @@ -78,7 +78,6 @@ #define RAFT_EVENT_EVENTS_START 3200 #define SIMPLE_EVENT_EVENTS_START 3300 #define UPDATE_EVENT_EVENTS_START 3500 -#define LOG_COLLATION_EVENT_EVENTS_START 3800 #define AIO_EVENT_EVENTS_START 3900 #define BLOCK_CACHE_EVENT_EVENTS_START 4000 #define UTILS_EVENT_EVENTS_START 5000 diff --git a/lib/perl/lib/Apache/TS/AdminClient.pm b/lib/perl/lib/Apache/TS/AdminClient.pm index 937d5ce310d..9892a909579 100644 --- a/lib/perl/lib/Apache/TS/AdminClient.pm +++ b/lib/perl/lib/Apache/TS/AdminClient.pm @@ -520,12 +520,6 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.local_state_dir proxy.config.log.ascii_buffer_size proxy.config.log.auto_delete_rolled_files - proxy.config.log.collation_host - proxy.config.log.collation_host_tagged - proxy.config.log.collation_max_send_buffers - proxy.config.log.collation_port - proxy.config.log.collation_retry_sec - proxy.config.log.collation_secret proxy.config.log.file_stat_frequency proxy.config.log.hostname proxy.config.log.log_buffer_size @@ -535,7 +529,6 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.log.max_line_size proxy.config.log.max_secs_per_buffer proxy.config.log.max_space_mb_for_logs - proxy.config.log.max_space_mb_for_orphan_logs proxy.config.log.max_space_mb_headroom proxy.config.log.overspill_report_count proxy.config.log.rolling_enabled diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 3003477c03c..8e8b5a0b8ea 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1010,8 +1010,6 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.log.max_space_mb_for_logs", RECD_INT, "25000", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.log.max_space_mb_for_orphan_logs", RECD_INT, "25", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} - , {RECT_CONFIG, "proxy.config.log.max_space_mb_headroom", RECD_INT, "1000", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.log.hostname", RECD_STRING, "localhost", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} @@ -1022,23 +1020,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.log.config.filename", RECD_STRING, "logging.yaml", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.log.collation_host", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_port", RECD_INT, "8085", RECU_DYNAMIC, RR_REQUIRED, RECC_INT, "[0-65535]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_secret", RECD_STRING, "foobar", RECU_DYNAMIC, RR_NULL, RECC_STR, ".*", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_host_tagged", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_retry_sec", RECD_INT, "5", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_max_send_buffers", RECD_INT, "16", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_preproc_threads", RECD_INT, "1", RECU_DYNAMIC, RR_REQUIRED, RECC_INT, "[1-128]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_host_timeout", RECD_INT, "86390", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_client_timeout", RECD_INT, "86400", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.log.preproc_threads", RECD_INT, "1", RECU_DYNAMIC, RR_REQUIRED, RECC_INT, "[1-128]", RECA_NULL} , {RECT_CONFIG, "proxy.config.log.rolling_enabled", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-4]", RECA_NULL} , @@ -1339,8 +1321,6 @@ static const RecordElement RecordsConfig[] = , {RECT_LOCAL, "proxy.local.outgoing_ip_to_bind", RECD_STRING, nullptr, RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_LOCAL, "proxy.local.log.collation_mode", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-4]", RECA_NULL} - , //# Librecords based stats system (new as of v2.1.3) {RECT_CONFIG, "proxy.config.stat_api.max_stats_allowed", RECD_INT, "256", RECU_RESTART_TS, RR_NULL, RECC_INT, "[256-1000]", RECA_NULL} diff --git a/proxy/README-stats.otl b/proxy/README-stats.otl index 80648346d0c..e57263fc0a1 100644 --- a/proxy/README-stats.otl +++ b/proxy/README-stats.otl @@ -521,7 +521,6 @@ Adam Beguelin wrote: > o Cache/Freshness > o Cache/Variable Content > o HostDB page -> o Logging/Log Collation --------------------------------------------------------------------------------- --------------------------------------------------------------------------------- --------------------------------------------------------------------------------- diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc index 9762dae7ea5..090b6a5c695 100644 --- a/proxy/logging/Log.cc +++ b/proxy/logging/Log.cc @@ -44,13 +44,11 @@ #include "LogFilter.h" #include "LogFormat.h" #include "LogFile.h" -#include "LogHost.h" #include "LogObject.h" #include "LogConfig.h" #include "LogBuffer.h" #include "LogUtils.h" #include "Log.h" -#include "LogSock.h" #include "tscore/SimpleTokenizer.h" #include "tscore/ink_apidefs.h" @@ -69,14 +67,8 @@ EventNotify *Log::preproc_notify; EventNotify *Log::flush_notify; InkAtomicList *Log::flush_data_list; -// Collate thread stuff -EventNotify Log::collate_notify; -ink_thread Log::collate_thread; -int Log::collation_accept_file_descriptor; -int Log::collation_preproc_threads; -int Log::collation_port; - // Log private objects +int Log::preproc_threads; int Log::init_status = 0; int Log::config_flags = 0; bool Log::logging_mode_changed = false; @@ -227,7 +219,7 @@ Log::periodic_tasks(long time_now) // so that log objects are flushed // change_configuration(); - } else if (logging_mode > LOG_MODE_NONE || config->collation_mode == Log::COLLATION_HOST || config->has_api_objects()) { + } else if (logging_mode > LOG_MODE_NONE || config->has_api_objects()) { Debug("log-periodic", "Performing periodic tasks"); Debug("log-periodic", "Periodic task interval = %d", periodic_tasks_interval); @@ -300,17 +292,6 @@ struct LoggingFlushContinuation : public Continuation { } }; -struct LoggingCollateContinuation : public Continuation { - int - mainEvent(int /* event ATS_UNUSED */, void * /* data ATS_UNUSED */) - { - Log::collate_thread_main(nullptr); - return 0; - } - - LoggingCollateContinuation() : Continuation(nullptr) { SET_HANDLER(&LoggingCollateContinuation::mainEvent); } -}; - /*------------------------------------------------------------------------- Log::init_fields @@ -935,8 +916,7 @@ Log::handle_periodic_tasks_int_change(const char * /* name ATS_UNUSED */, RecDat void Log::init(int flags) { - collation_preproc_threads = 1; - collation_accept_file_descriptor = NO_FD; + preproc_threads = 1; // store the configuration flags // @@ -957,33 +937,28 @@ Log::init(int flags) LogConfig::register_stat_callbacks(); config->read_configuration_variables(); - collation_port = config->collation_port; - collation_preproc_threads = config->collation_preproc_threads; - - if (config_flags & STANDALONE_COLLATOR) { - logging_mode = LOG_MODE_TRANSACTIONS; + preproc_threads = config->preproc_threads; + + int val = (int)REC_ConfigReadInteger("proxy.config.log.logging_enabled"); + if (val < LOG_MODE_NONE || val > LOG_MODE_FULL) { + logging_mode = LOG_MODE_FULL; + Warning("proxy.config.log.logging_enabled has an invalid " + "value, setting it to %d", + logging_mode); } else { - int val = (int)REC_ConfigReadInteger("proxy.config.log.logging_enabled"); - if (val < LOG_MODE_NONE || val > LOG_MODE_FULL) { - logging_mode = LOG_MODE_FULL; - Warning("proxy.config.log.logging_enabled has an invalid " - "value, setting it to %d", - logging_mode); - } else { - logging_mode = (LoggingMode)val; - } - // periodic task interval are set on a per instance basis - MgmtInt pti = REC_ConfigReadInteger("proxy.config.log.periodic_tasks_interval"); - if (pti <= 0) { - Error("proxy.config.log.periodic_tasks_interval = %" PRId64 " is invalid", pti); - Note("falling back to default periodic tasks interval = %d", PERIODIC_TASKS_INTERVAL_FALLBACK); - periodic_tasks_interval = PERIODIC_TASKS_INTERVAL_FALLBACK; - } else { - periodic_tasks_interval = static_cast(pti); - } - - REC_RegisterConfigUpdateFunc("proxy.config.log.periodic_tasks_interval", &Log::handle_periodic_tasks_int_change, nullptr); + logging_mode = (LoggingMode)val; } + // periodic task interval are set on a per instance basis + MgmtInt pti = REC_ConfigReadInteger("proxy.config.log.periodic_tasks_interval"); + if (pti <= 0) { + Error("proxy.config.log.periodic_tasks_interval = %" PRId64 " is invalid", pti); + Note("falling back to default periodic tasks interval = %d", PERIODIC_TASKS_INTERVAL_FALLBACK); + periodic_tasks_interval = PERIODIC_TASKS_INTERVAL_FALLBACK; + } else { + periodic_tasks_interval = static_cast(pti); + } + + REC_RegisterConfigUpdateFunc("proxy.config.log.periodic_tasks_interval", &Log::handle_periodic_tasks_int_change, nullptr); } // if remote management is enabled, do all necessary initialization to @@ -992,8 +967,6 @@ Log::init(int flags) if (!(config_flags & NO_REMOTE_MANAGEMENT)) { REC_RegisterConfigUpdateFunc("proxy.config.log.logging_enabled", &Log::handle_logging_mode_change, nullptr); - REC_RegisterConfigUpdateFunc("proxy.local.log.collation_mode", &Log::handle_logging_mode_change, nullptr); - // Clear any stat values that need to be reset on startup // RecSetRawStatSum(log_rsb, log_stat_log_files_open_stat, 0); @@ -1003,9 +976,6 @@ Log::init(int flags) init_fields(); if (!(config_flags & LOGCAT)) { Debug("log-config", "Log::init(): logging_mode = %d init status = %d", logging_mode, init_status); - if (config_flags & STANDALONE_COLLATOR) { - config->collation_mode = Log::COLLATION_HOST; - } config->init(); init_when_enabled(); } @@ -1018,26 +988,24 @@ Log::init_when_enabled() ink_release_assert(config->initialized == true); if (!(init_status & FULLY_INITIALIZED)) { - if (!(config_flags & STANDALONE_COLLATOR)) { - // register callbacks - // - if (!(config_flags & NO_REMOTE_MANAGEMENT)) { - LogConfig::register_config_callbacks(); - } - - LogConfig::register_mgmt_callbacks(); + // register callbacks + // + if (!(config_flags & NO_REMOTE_MANAGEMENT)) { + LogConfig::register_config_callbacks(); } + + LogConfig::register_mgmt_callbacks(); // setup global scrap object // global_scrap_format = MakeTextLogFormat(); global_scrap_object = new LogObject(global_scrap_format, Log::config->logfile_dir, "scrapfile.log", LOG_FILE_BINARY, nullptr, - Log::config->rolling_enabled, Log::config->collation_preproc_threads, Log::config->rolling_interval_sec, + Log::config->rolling_enabled, Log::config->preproc_threads, Log::config->rolling_interval_sec, Log::config->rolling_offset_hr, Log::config->rolling_size_mb); - // create the flush thread and the collation thread + // create the flush thread create_threads(); - eventProcessor.schedule_every(new PeriodicWakeup(collation_preproc_threads, 1), HRTIME_SECOND, ET_CALL); + eventProcessor.schedule_every(new PeriodicWakeup(preproc_threads, 1), HRTIME_SECOND, ET_CALL); init_status |= FULLY_INITIALIZED; } @@ -1052,7 +1020,7 @@ void Log::create_threads() { char desc[64]; - preproc_notify = new EventNotify[collation_preproc_threads]; + preproc_notify = new EventNotify[preproc_threads]; size_t stacksize; REC_ReadConfigInteger(stacksize, "proxy.config.thread.default.stacksize"); @@ -1061,7 +1029,7 @@ Log::create_threads() // // no need for the conditional var since it will be relying on // on the event system. - for (int i = 0; i < collation_preproc_threads; i++) { + for (int i = 0; i < preproc_threads; i++) { Continuation *preproc_cont = new LoggingPreprocContinuation(i); sprintf(desc, "[LOG_PREPROC %d]", i); eventProcessor.spawn_thread(preproc_cont, desc, stacksize); @@ -1400,166 +1368,3 @@ Log::flush_thread_main(void * /* args ATS_UNUSED */) Log::flush_notify->unlock(); return nullptr; } - -/*------------------------------------------------------------------------- - Log::collate_thread_main - - This function defines the functionality of the log collation thread, - whose purpose is to collate log buffers from other nodes. - -------------------------------------------------------------------------*/ - -void * -Log::collate_thread_main(void * /* args ATS_UNUSED */) -{ - LogSock *sock; - LogBufferHeader *header; - LogFormat *format; - LogObject *obj; - int bytes_read; - int sock_id; - int new_client; - - Debug("log-thread", "Log collation thread is alive ..."); - - Log::collate_notify.lock(); - - while (true) { - ink_assert(Log::config != nullptr); - - // wait on the collation condition variable until we're sure that - // we're a collation host. The while loop guards against spurious - // wake-ups. - // - while (!Log::config->am_collation_host()) { - Log::collate_notify.wait(); - } - - // Ok, at this point we know we're a log collation host, so get to - // work. We still need to keep checking whether we're a collation - // host to account for a reconfiguration. - // - Debug("log-sock", "collation thread starting, creating LogSock"); - sock = new LogSock(LogSock::LS_CONST_MAX_CONNS); - ink_assert(sock != nullptr); - - if (sock->listen(Log::config->collation_port) != 0) { - LogUtils::manager_alarm(LogUtils::LOG_ALARM_ERROR, "Collation server error; could not listen on port %d", - Log::config->collation_port); - Warning("Collation server error; could not listen on port %d", Log::config->collation_port); - delete sock; - // - // go to sleep ... - // - Log::collate_notify.wait(); - continue; - } - - while (true) { - if (!Log::config->am_collation_host()) { - break; - } - - if (sock->pending_connect(0)) { - Debug("log-sock", "pending connection ..."); - if ((new_client = sock->accept()) < 0) { - Debug("log-sock", "error accepting new collation client"); - } else { - Debug("log-sock", "connection %d accepted", new_client); - if (!sock->authorized_client(new_client, Log::config->collation_secret)) { - Warning("Unauthorized client connecting to " - "log collation port; connection refused."); - sock->close(new_client); - } - } - } - - sock->check_connections(); - - if (!sock->pending_message_any(&sock_id, 0)) { - continue; - } - - Debug("log-sock", "pending message ..."); - header = static_cast(sock->read_alloc(sock_id, &bytes_read)); - if (!header) { - Debug("log-sock", "Error reading LogBuffer from collation client"); - continue; - } - - if (header->version != LOG_SEGMENT_VERSION) { - Note("Invalid LogBuffer received; invalid version - buffer = %u, current = %u", header->version, LOG_SEGMENT_VERSION); - delete[] header; - continue; - } - - Debug("log-sock", "message accepted, size = %d", bytes_read); - - obj = match_logobject(header); - if (!obj) { - Note("LogObject not found with fieldlist id; " - "writing LogBuffer to scrap file"); - obj = global_scrap_object; - } - - format = obj->m_format; - Debug("log-sock", "Using format '%s'", format->name()); - - delete[] header; - } - - Debug("log", "no longer collation host, deleting LogSock"); - delete sock; - } - - /* NOTREACHED */ - Log::collate_notify.unlock(); - return nullptr; -} - -/*------------------------------------------------------------------------- - Log::match_logobject - - This routine matches the given buffer with the local list of LogObjects. - If a match cannot be found, then we'll try to construct a local LogObject - using the information provided in the header. If all else fails, we - return NULL. - -------------------------------------------------------------------------*/ - -LogObject * -Log::match_logobject(LogBufferHeader *header) -{ - if (!header) { - return nullptr; - } - - LogObject *obj; - obj = Log::config->log_object_manager.get_object_with_signature(header->log_object_signature); - - if (!obj) { - // object does not exist yet, create it - // - LogFormat fmt("__collation_format__", header->fmt_fieldlist(), header->fmt_printf()); - - if (fmt.valid()) { - LogFileFormat file_format = (header->log_object_flags & LogObject::BINARY) ? - LOG_FILE_BINARY : - ((header->log_object_flags & LogObject::WRITES_TO_PIPE) ? LOG_FILE_PIPE : LOG_FILE_ASCII); - - obj = new LogObject(&fmt, Log::config->logfile_dir, header->log_filename(), file_format, nullptr, - (Log::RollingEnabledValues)Log::config->rolling_enabled, Log::config->collation_preproc_threads, - Log::config->rolling_interval_sec, Log::config->rolling_offset_hr, Log::config->rolling_size_mb, true); - - obj->set_remote_flag(); - - if (Log::config->log_object_manager.manage_object(obj)) { - // object manager can't solve filename conflicts - // delete the object and return NULL - // - delete obj; - obj = nullptr; - } - } - } - - return obj; -} diff --git a/proxy/logging/Log.h b/proxy/logging/Log.h index d266019a2d6..01700a60818 100644 --- a/proxy/logging/Log.h +++ b/proxy/logging/Log.h @@ -132,12 +132,9 @@ class Log enum ConfigFlags { NO_REMOTE_MANAGEMENT = 1, - STANDALONE_COLLATOR = 2, LOGCAT = 4, }; - enum CollationMode { NO_COLLATION = 0, COLLATION_HOST, N_COLLATION_MODES }; - enum RollingEnabledValues { NO_ROLLING = 0, ROLL_ON_TIME_ONLY, @@ -195,14 +192,7 @@ class Log static InkAtomicList *flush_data_list; static void *flush_thread_main(void *args); - // collation thread stuff - static EventNotify collate_notify; - static ink_thread collate_thread; - static int collation_preproc_threads; - static int collation_accept_file_descriptor; - static int collation_port; - static void *collate_thread_main(void *args); - static LogObject *match_logobject(LogBufferHeader *header); + static int preproc_threads; // reconfiguration stuff static void change_configuration(); diff --git a/proxy/logging/LogBuffer.cc b/proxy/logging/LogBuffer.cc index 21b4c87955d..b35669406cb 100644 --- a/proxy/logging/LogBuffer.cc +++ b/proxy/logging/LogBuffer.cc @@ -38,7 +38,6 @@ #include "LogFormat.h" #include "LogUtils.h" #include "LogFile.h" -#include "LogHost.h" #include "LogObject.h" #include "LogAccess.h" #include "LogConfig.h" diff --git a/proxy/logging/LogCollationAccept.cc b/proxy/logging/LogCollationAccept.cc deleted file mode 100644 index 3e9414396ad..00000000000 --- a/proxy/logging/LogCollationAccept.cc +++ /dev/null @@ -1,105 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -//------------------------------------------------------------------------- -// include files -//------------------------------------------------------------------------- - -#include "tscore/ink_platform.h" -#include "P_EventSystem.h" - -#include "Log.h" -#include "LogCollationAccept.h" -#include "LogCollationHostSM.h" - -//------------------------------------------------------------------------- -// LogCollationAccept::LogCollationAccept -//------------------------------------------------------------------------- - -LogCollationAccept::LogCollationAccept(int port) : Continuation(new_ProxyMutex()), m_port(port) -{ - NetProcessor::AcceptOptions opt; - SET_HANDLER((LogCollationAcceptHandler)&LogCollationAccept::accept_event); - // work around for iocore problem where _pre_fetch_buffer can get - // appended to itself if multiple do_io_reads are called requesting - // small amounts of data. Most arguments are default except for the - // last one which we will set to true. - // [amc] That argument is ignored so I dropped it. - opt.local_port = m_port; - opt.ip_family = AF_INET; - opt.accept_threads = 0; - m_accept_action = netProcessor.accept(this, opt); - ink_assert(nullptr != m_accept_action); -} - -//------------------------------------------------------------------------- -// LogCollationAccept::~LogCollationAccept -//------------------------------------------------------------------------- - -LogCollationAccept::~LogCollationAccept() -{ - Debug("log-collation", "LogCollationAccept::~LogCollationAccept"); - - // stop the netProcessor - if (m_accept_action) { - m_accept_action->cancel(); - m_accept_action = nullptr; - - Debug("log-collation", - "closing Log::collation_accept_file_descriptor " - "(%d)", - Log::collation_accept_file_descriptor); - if (::close(Log::collation_accept_file_descriptor) < 0) { - Error("error closing collate listen file descriptor [%d]: %s", Log::collation_accept_file_descriptor, strerror(errno)); - } else { - Log::collation_accept_file_descriptor = NO_FD; - } - } else { - ink_assert(!"[ERROR] m_accept_action is NULL"); - } - - // stop the eventProcessor - // ... but what if there's more than one pending? - if (m_pending_event && (m_pending_event != ACTION_RESULT_DONE)) { - m_pending_event->cancel(); - } -} - -//------------------------------------------------------------------------- -// LogCollationAccept::accept_event -//------------------------------------------------------------------------- - -int -LogCollationAccept::accept_event(int event, NetVConnection *net_vc) -{ - switch (event) { - case NET_EVENT_ACCEPT: - new LogCollationHostSM(net_vc); - break; - - default: - ink_assert(!"[ERROR] Unexpected Event"); - } - - return EVENT_CONT; -} diff --git a/proxy/logging/LogCollationAccept.h b/proxy/logging/LogCollationAccept.h deleted file mode 100644 index 315ca8905ce..00000000000 --- a/proxy/logging/LogCollationAccept.h +++ /dev/null @@ -1,42 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -#include "P_EventSystem.h" -#include "P_Net.h" - -struct LogCollationAccept : public Continuation { -public: - LogCollationAccept(int port); - ~LogCollationAccept() override; - - int accept_event(int event, NetVConnection *net_vc); - -private: - int m_port; - Action *m_accept_action = nullptr; - Event *m_pending_event = nullptr; -}; - -typedef int (LogCollationAccept::*LogCollationAcceptHandler)(int, void *); diff --git a/proxy/logging/LogCollationBase.h b/proxy/logging/LogCollationBase.h deleted file mode 100644 index 17b783a4595..00000000000 --- a/proxy/logging/LogCollationBase.h +++ /dev/null @@ -1,45 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -//------------------------------------------------------------------------- -// LogCollationBase -//------------------------------------------------------------------------- - -class LogCollationBase -{ -protected: - // ToDo: Can we we use the stuff from LogSock.h instead?? - struct NetMsgHeader { - int msg_bytes; // length of the following message - }; - - enum LogCollEvent { - LOG_COLL_EVENT_NULL = LOG_COLLATION_EVENT_EVENTS_START, - LOG_COLL_EVENT_SWITCH, - LOG_COLL_EVENT_READ_COMPLETE, - LOG_COLL_EVENT_WRITE_COMPLETE, - LOG_COLL_EVENT_ERROR - }; -}; diff --git a/proxy/logging/LogCollationClientSM.cc b/proxy/logging/LogCollationClientSM.cc deleted file mode 100644 index 73051ccec94..00000000000 --- a/proxy/logging/LogCollationClientSM.cc +++ /dev/null @@ -1,716 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -//------------------------------------------------------------------------- -// include files -//------------------------------------------------------------------------- - -#include "tscore/ink_platform.h" - -#include -#include -#include -#include -#include - -#include "P_EventSystem.h" -#include "P_Net.h" - -#include "LogUtils.h" -#include "LogSock.h" -#include "LogField.h" -#include "LogFile.h" -#include "LogFormat.h" -#include "LogBuffer.h" -#include "LogHost.h" -#include "LogObject.h" -#include "LogConfig.h" -#include "Log.h" - -#include "LogCollationClientSM.h" - -//------------------------------------------------------------------------- -// statics -//------------------------------------------------------------------------- - -int LogCollationClientSM::ID = 0; - -//------------------------------------------------------------------------- -// LogCollationClientSM::LogCollationClientSM -//------------------------------------------------------------------------- - -LogCollationClientSM::LogCollationClientSM(LogHost *log_host) : Continuation(new_ProxyMutex()), m_log_host(log_host), m_id(ID++) -{ - Debug("log-coll", "[%d]client::constructor", m_id); - - ink_assert(m_log_host != nullptr); - - // allocate send_list before we do anything - // we can accept logs to send before we're fully initialized - m_buffer_send_list = new LogBufferList(); - ink_assert(m_buffer_send_list != nullptr); - - SET_HANDLER((LogCollationClientSMHandler)&LogCollationClientSM::client_handler); - client_init(LOG_COLL_EVENT_SWITCH, nullptr); -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::~LogCollationClientSM -//------------------------------------------------------------------------- - -LogCollationClientSM::~LogCollationClientSM() -{ - Debug("log-coll", "[%d]client::destructor", m_id); - - ink_mutex_acquire(&(mutex->the_mutex)); - client_done(LOG_COLL_EVENT_SWITCH, nullptr); - ink_mutex_release(&(mutex->the_mutex)); -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// handler -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_handler -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_handler(int event, void *data) -{ - switch (m_client_state) { - case LOG_COLL_CLIENT_AUTH: - return client_auth(event, (VIO *)data); - case LOG_COLL_CLIENT_DNS: - return client_dns(event, (HostDBInfo *)data); - case LOG_COLL_CLIENT_DONE: - return client_done(event, data); - case LOG_COLL_CLIENT_FAIL: - return client_fail(event, data); - case LOG_COLL_CLIENT_IDLE: - return client_idle(event, data); - case LOG_COLL_CLIENT_INIT: - return client_init(event, data); - case LOG_COLL_CLIENT_OPEN: - return client_open(event, (NetVConnection *)data); - case LOG_COLL_CLIENT_SEND: - return client_send(event, (VIO *)data); - default: - ink_assert(!"unexpcted state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// pubic interface -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationClientSM::send -//------------------------------------------------------------------------- - -int -LogCollationClientSM::send(LogBuffer *log_buffer) -{ - ip_port_text_buffer ipb; - - // take lock (can block on call because we're on our own thread) - ink_mutex_acquire(&(mutex->the_mutex)); - - Debug("log-coll", "[%d]client::send", m_id); - - // deny if state is DONE or FAIL - if (m_client_state == LOG_COLL_CLIENT_DONE || m_client_state == LOG_COLL_CLIENT_FAIL) { - Debug("log-coll", "[%d]client::send - DONE/FAIL state; rejecting", m_id); - ink_mutex_release(&(mutex->the_mutex)); - return 0; - } - // only allow send if m_flow is ALLOW - if (m_flow == LOG_COLL_FLOW_DENY) { - Debug("log-coll", "[%d]client::send - m_flow = DENY; rejecting", m_id); - ink_mutex_release(&(mutex->the_mutex)); - return 0; - } - // add log_buffer to m_buffer_send_list - ink_assert(log_buffer != nullptr); - ink_assert(m_buffer_send_list != nullptr); - m_buffer_send_list->add(log_buffer); - Debug("log-coll", "[%d]client::send - new log_buffer to send_list", m_id); - - // disable m_flow if there's too much work to do now - ink_assert(m_flow == LOG_COLL_FLOW_ALLOW); - if (m_buffer_send_list->get_size() >= Log::config->collation_max_send_buffers) { - Debug("log-coll", "[%d]client::send - m_flow = DENY", m_id); - Note("[log-coll] send-queue full; orphaning logs " - "[%s:%u]", - m_log_host->ip_addr().toString(ipb, sizeof(ipb)), m_log_host->port()); - m_flow = LOG_COLL_FLOW_DENY; - } - // compute return value - // must be done before call to client_send. log_buffer may - // be converted to network order during that call. - LogBufferHeader *log_buffer_header = log_buffer->header(); - ink_assert(log_buffer_header != nullptr); - int bytes_to_write = log_buffer_header->byte_count; - - // re-initiate sending if currently idle - if (m_client_state == LOG_COLL_CLIENT_IDLE) { - m_client_state = LOG_COLL_CLIENT_SEND; - ink_assert(m_pending_event == nullptr); - m_pending_event = eventProcessor.schedule_imm(this); - // eventProcessor.schedule_imm(this); - // client_send(LOG_COLL_EVENT_SWITCH, NULL); - } - - ink_mutex_release(&(mutex->the_mutex)); - return bytes_to_write; -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// client states -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_auth -// next: client_fail || client_send -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_auth(int event, VIO * /* vio ATS_UNUSED */) -{ - ip_port_text_buffer ipb; - - Debug("log-coll", "[%d]client::client_auth", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: { - Debug("log-coll", "[%d]client::client_auth - SWITCH", m_id); - m_client_state = LOG_COLL_CLIENT_AUTH; - - NetMsgHeader nmh; - int bytes_to_send = (int)strlen(Log::config->collation_secret); - nmh.msg_bytes = bytes_to_send; - - // memory copies, I know... but it happens rarely!!! ^_^ - ink_assert(m_auth_buffer != nullptr); - m_auth_buffer->write((char *)&nmh, sizeof(NetMsgHeader)); - m_auth_buffer->write(Log::config->collation_secret, bytes_to_send); - bytes_to_send += sizeof(NetMsgHeader); - - Debug("log-coll", "[%d]client::client_auth - do_io_write(%d)", m_id, bytes_to_send); - ink_assert(m_host_vc != nullptr); - m_host_vio = m_host_vc->do_io_write(this, bytes_to_send, m_auth_reader); - ink_assert(m_host_vio != nullptr); - - return EVENT_CONT; - } - - case VC_EVENT_WRITE_READY: - Debug("log-coll", "[%d]client::client_auth - WRITE_READY", m_id); - return EVENT_CONT; - - case VC_EVENT_WRITE_COMPLETE: - Debug("log-coll", "[%d]client::client_auth - WRITE_COMPLETE", m_id); - - Note("[log-coll] host up [%s:%u]", m_log_host->ip_addr().toString(ipb, sizeof(ipb)), m_log_host->port()); - m_host_is_up = true; - - return client_send(LOG_COLL_EVENT_SWITCH, nullptr); - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: { - Debug("log-coll", "[%d]client::client_auth - TIMEOUT|EOS|ERROR", m_id); - int64_t read_avail = m_auth_reader->read_avail(); - - if (read_avail > 0) { - Debug("log-coll", "[%d]client::client_auth - consuming unsent data", m_id); - m_auth_reader->consume(read_avail); - } - - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - } - - default: - ink_assert(!"unexpected event"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_dns -// next: client_open || client_done -//------------------------------------------------------------------------- -int -LogCollationClientSM::client_dns(int event, HostDBInfo *hostdb_info) -{ - Debug("log-coll", "[%d]client::client_dns", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_client_state = LOG_COLL_CLIENT_DNS; - if (m_log_host->m_name == nullptr) { - return client_done(LOG_COLL_EVENT_SWITCH, nullptr); - } - hostDBProcessor.getbyname_re(this, m_log_host->m_name, 0, - HostDBProcessor::Options().setFlags(HostDBProcessor::HOSTDB_FORCE_DNS_RELOAD)); - return EVENT_CONT; - - case EVENT_HOST_DB_LOOKUP: - if (hostdb_info == nullptr) { - return client_done(LOG_COLL_EVENT_SWITCH, nullptr); - } - m_log_host->m_ip.assign(hostdb_info->ip()); - m_log_host->m_ip.toString(m_log_host->m_ipstr, sizeof(m_log_host->m_ipstr)); - - return client_open(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected event"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_done -// next: -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_done(int event, void * /* data ATS_UNUSED */) -{ - ip_port_text_buffer ipb; - - Debug("log-coll", "[%d]client::client_done", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_client_state = LOG_COLL_CLIENT_DONE; - - Note("[log-coll] client shutdown [%s:%u]", m_log_host->ip_addr().toString(ipb, sizeof(ipb)), m_log_host->port()); - - // close connections - if (m_host_vc) { - Debug("log-coll", "[%d]client::client_done - disconnecting!", m_id); - // do I need to delete this??? - m_host_vc->do_io_close(0); - m_host_vc = nullptr; - } - // flush unsent logs to orphan - flush_to_orphan(); - - // cancel any pending events/actions - if (m_pending_action != nullptr) { - m_pending_action->cancel(); - } - if (m_pending_event != nullptr) { - m_pending_event->cancel(); - } - // free memory - if (m_auth_buffer) { - if (m_auth_reader) { - m_auth_buffer->dealloc_reader(m_auth_reader); - } - free_MIOBuffer(m_auth_buffer); - } - if (m_send_buffer) { - if (m_send_reader) { - m_send_buffer->dealloc_reader(m_send_reader); - } - free_MIOBuffer(m_send_buffer); - } - if (m_abort_buffer) { - free_MIOBuffer(m_abort_buffer); - } - if (m_buffer_send_list) { - delete m_buffer_send_list; - } - - return EVENT_DONE; - - default: - ink_assert(!"unexpected event"); - return EVENT_DONE; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_fail -// next: client_fail || client_open -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_fail(int event, void * /* data ATS_UNUSED */) -{ - ip_port_text_buffer ipb; - - Debug("log-coll", "[%d]client::client_fail", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]client::client_fail - SWITCH", m_id); - m_client_state = LOG_COLL_CLIENT_FAIL; - - // avoid flooding log when host is down - if (m_host_is_up) { - Note("[log-coll] host down [%s:%u]", m_log_host->ip_addr().toString(ipb, sizeof ipb), m_log_host->m_port); - char msg_buf[128]; - snprintf(msg_buf, sizeof(msg_buf), "Collation host %s:%u down", m_log_host->ip_addr().toString(ipb, sizeof ipb), - m_log_host->m_port); - RecSignalManager(MGMT_SIGNAL_SAC_SERVER_DOWN, msg_buf); - m_host_is_up = false; - } - - // close our NetVConnection (do I need to delete this) - if (m_host_vc) { - m_host_vc->do_io_close(0); - m_host_vc = nullptr; - } - // flush unsent logs to orphan - flush_to_orphan(); - - // call back in collation_retry_sec seconds - ink_assert(m_pending_event == nullptr); - m_pending_event = eventProcessor.schedule_in(this, HRTIME_SECONDS(Log::config->collation_retry_sec)); - - return EVENT_CONT; - - case EVENT_INTERVAL: - Debug("log-coll", "[%d]client::client_fail - INTERVAL", m_id); - m_pending_event = nullptr; - return client_open(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected event"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_idle -// next: client_send -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_idle(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]client::client_idle", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_client_state = LOG_COLL_CLIENT_IDLE; - return EVENT_CONT; - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: - Debug("log-coll", "[%d]client::client_idle - TIMEOUT|EOS|ERROR", m_id); - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpcted state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_init -// next: client_dns -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_init(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]client::client_init", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_client_state = LOG_COLL_CLIENT_INIT; - ink_assert(m_pending_event == nullptr); - ink_mutex_acquire(&(mutex->the_mutex)); - m_pending_event = eventProcessor.schedule_imm(this); - ink_mutex_release(&(mutex->the_mutex)); - return EVENT_CONT; - - case EVENT_IMMEDIATE: - // callback complete, reset m_pending_event - m_pending_event = nullptr; - - // allocate buffers - m_auth_buffer = new_MIOBuffer(); - ink_assert(m_auth_buffer != nullptr); - m_auth_reader = m_auth_buffer->alloc_reader(); - ink_assert(m_auth_reader != nullptr); - m_send_buffer = new_MIOBuffer(); - ink_assert(m_send_buffer != nullptr); - m_send_reader = m_send_buffer->alloc_reader(); - ink_assert(m_send_reader != nullptr); - m_abort_buffer = new_MIOBuffer(); - ink_assert(m_abort_buffer != nullptr); - - // if we don't have an ip already, switch to client_dns - if (!m_log_host->ip_addr().isValid()) { - return client_dns(LOG_COLL_EVENT_SWITCH, nullptr); - } else { - return client_open(LOG_COLL_EVENT_SWITCH, nullptr); - } - - default: - ink_assert(!"unexpected state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_open -// next: client_auth || client_fail -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_open(int event, NetVConnection *net_vc) -{ - RecInt rec_timeout; - int timeout = 86400; - ip_port_text_buffer ipb; - Debug("log-coll", "[%d]client::client_open", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]client::client_open - SWITCH", m_id); - m_client_state = LOG_COLL_CLIENT_OPEN; - - { - IpEndpoint target; - target.assign(m_log_host->ip_addr(), htons(m_log_host->port())); - ink_assert(target.isValid()); - Action *connect_action_handle = netProcessor.connect_re(this, &target.sa); - - if (connect_action_handle != ACTION_RESULT_DONE) { - ink_assert(!m_pending_action); - m_pending_action = connect_action_handle; - } - } - - return EVENT_CONT; - - case NET_EVENT_OPEN: - Debug("log-coll", "[%d]client::client_open - %s:%u", m_id, m_log_host->ip_addr().toString(ipb, sizeof ipb), m_log_host->port()); - - // callback complete, reset m_pending_action - m_pending_action = nullptr; - - ink_assert(net_vc != nullptr); - m_host_vc = net_vc; - - // assign an explicit inactivity timeout so that it will not get the default value later - if (RecGetRecordInt("proxy.config.log.collation_client_timeout", &rec_timeout) == REC_ERR_OKAY) { - timeout = rec_timeout; - } - m_host_vc->set_inactivity_timeout(HRTIME_SECONDS(timeout)); - - // setup a client reader just for detecting a host disconnnect - // (iocore should call back this function with and EOS/ERROR) - m_abort_vio = m_host_vc->do_io_read(this, 1, m_abort_buffer); - - // change states - return client_auth(LOG_COLL_EVENT_SWITCH, nullptr); - - case NET_EVENT_OPEN_FAILED: - Debug("log-coll", "[%d]client::client_open - OPEN_FAILED", m_id); - // callback complete, reset m_pending_pending action - m_pending_action = nullptr; - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected event"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_send -// next: client_fail || client_idle || client_send -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_send(int event, VIO * /* vio ATS_UNUSED */) -{ - ip_port_text_buffer ipb; - - Debug("log-coll", "[%d]client::client_send", m_id); - - switch (event) { - case EVENT_IMMEDIATE: - Debug("log-coll", "[%d]client::client_send - EVENT_IMMEDIATE", m_id); - // callback complete, reset m_pending_event - m_pending_event = nullptr; - // fallthrough - - case LOG_COLL_EVENT_SWITCH: { - Debug("log-coll", "[%d]client::client_send - SWITCH", m_id); - m_client_state = LOG_COLL_CLIENT_SEND; - - // get a buffer off our queue - ink_assert(m_buffer_send_list != nullptr); - ink_assert(m_buffer_in_iocore == nullptr); - if ((m_buffer_in_iocore = m_buffer_send_list->get()) == nullptr) { - return client_idle(LOG_COLL_EVENT_SWITCH, nullptr); - } - Debug("log-coll", "[%d]client::client_send - send_list to m_buffer_in_iocore", m_id); - Debug("log-coll", "[%d]client::client_send - send_list_size(%d)", m_id, m_buffer_send_list->get_size()); - - // enable m_flow if we're out of work to do - if (m_flow == LOG_COLL_FLOW_DENY && m_buffer_send_list->get_size() == 0) { - Debug("log-coll", "[%d]client::client_send - m_flow = ALLOW", m_id); - Note("[log-coll] send-queue clear; resuming collation [%s:%u]", m_log_host->ip_addr().toString(ipb, sizeof ipb), - m_log_host->port()); - m_flow = LOG_COLL_FLOW_ALLOW; - } - // future work: - // Wrap the buffer in a io_buffer_block and send directly to - // do_io_write to save a memory copy. But for now, just - // write the lame way. - -#if defined(LOG_BUFFER_TRACKING) - Debug("log-buftrak", "[%d]client::client_send - network write begin", m_buffer_in_iocore->header()->id); -#endif // defined(LOG_BUFFER_TRACKING) - - // prepare to send data - ink_assert(m_buffer_in_iocore != nullptr); - LogBufferHeader *log_buffer_header = m_buffer_in_iocore->header(); - ink_assert(log_buffer_header != nullptr); - NetMsgHeader nmh; - int bytes_to_send = log_buffer_header->byte_count; - nmh.msg_bytes = bytes_to_send; - // TODO: We currently don't try to make the log buffers handle little vs big endian. TS-1156. - // m_buffer_in_iocore->convert_to_network_order(); - - RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_num_sent_to_network_stat, log_buffer_header->entry_count); - - RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_bytes_sent_to_network_stat, log_buffer_header->byte_count); - - // copy into m_send_buffer - ink_assert(m_send_buffer != nullptr); - m_send_buffer->write((char *)&nmh, sizeof(NetMsgHeader)); - m_send_buffer->write((char *)log_buffer_header, bytes_to_send); - bytes_to_send += sizeof(NetMsgHeader); - - // send m_send_buffer to iocore - Debug("log-coll", "[%d]client::client_send - do_io_write(%d)", m_id, bytes_to_send); - ink_assert(m_host_vc != nullptr); - m_host_vio = m_host_vc->do_io_write(this, bytes_to_send, m_send_reader); - ink_assert(m_host_vio != nullptr); - } - return EVENT_CONT; - - case VC_EVENT_WRITE_READY: - Debug("log-coll", "[%d]client::client_send - WRITE_READY", m_id); - return EVENT_CONT; - - case VC_EVENT_WRITE_COMPLETE: - Debug("log-coll", "[%d]client::client_send - WRITE_COMPLETE", m_id); - - ink_assert(m_buffer_in_iocore != nullptr); -#if defined(LOG_BUFFER_TRACKING) - Debug("log-buftrak", "[%d]client::client_send - network write complete", m_buffer_in_iocore->header()->id); -#endif // defined(LOG_BUFFER_TRACKING) - - // done with the buffer, delete it - Debug("log-coll", "[%d]client::client_send - m_buffer_in_iocore[%p] to delete_list", m_id, m_buffer_in_iocore); - LogBuffer::destroy(m_buffer_in_iocore); - m_buffer_in_iocore = nullptr; - - // switch back to client_send - return client_send(LOG_COLL_EVENT_SWITCH, nullptr); - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: { - Debug("log-coll", "[%d]client::client_send - TIMEOUT|EOS|ERROR", m_id); - int64_t read_avail = m_send_reader->read_avail(); - - if (read_avail > 0) { - Debug("log-coll", "[%d]client::client_send - consuming unsent data", m_id); - m_send_reader->consume(read_avail); - } - - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - } - - default: - Debug("log-coll", "[%d]client::client_send - default", m_id); - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// support functions -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationClientSM::flush_to_orphan -//------------------------------------------------------------------------- -void -LogCollationClientSM::flush_to_orphan() -{ - Debug("log-coll", "[%d]client::flush_to_orphan", m_id); - - // if in middle of a write, flush buffer_in_iocore to orphan - if (m_buffer_in_iocore != nullptr) { - Debug("log-coll", "[%d]client::flush_to_orphan - m_buffer_in_iocore to oprhan", m_id); - // TODO: We currently don't try to make the log buffers handle little vs big endian. TS-1156. - // m_buffer_in_iocore->convert_to_host_order(); - m_log_host->orphan_write_and_try_delete(m_buffer_in_iocore); - m_buffer_in_iocore = nullptr; - } - // flush buffers in send_list to orphan - LogBuffer *log_buffer; - ink_assert(m_buffer_send_list != nullptr); - while ((log_buffer = m_buffer_send_list->get()) != nullptr) { - Debug("log-coll", "[%d]client::flush_to_orphan - send_list to orphan", m_id); - m_log_host->orphan_write_and_try_delete(log_buffer); - } - - // Now send_list is empty, let's update m_flow to ALLOW status - Debug("log-coll", "[%d]client::client_send - m_flow = ALLOW", m_id); - m_flow = LOG_COLL_FLOW_ALLOW; -} diff --git a/proxy/logging/LogCollationClientSM.h b/proxy/logging/LogCollationClientSM.h deleted file mode 100644 index e10cc1fe5fe..00000000000 --- a/proxy/logging/LogCollationClientSM.h +++ /dev/null @@ -1,116 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -//------------------------------------------------------------------------- -// includes -//------------------------------------------------------------------------- -#include "P_HostDB.h" -#include "P_Net.h" -#include "LogCollationBase.h" - -//------------------------------------------------------------------------- -// pre-declarations -//------------------------------------------------------------------------- - -class LogBuffer; -class LogHost; - -//------------------------------------------------------------------------- -// LogCollationClientSM -//------------------------------------------------------------------------- - -class LogCollationClientSM : public LogCollationBase, public Continuation -{ -public: - LogCollationClientSM(LogHost *log_host); - ~LogCollationClientSM() override; - - int client_handler(int event, void *data); - - // public interface (for LogFile) - int send(LogBuffer *log_buffer); - -private: - enum ClientState { - LOG_COLL_CLIENT_START, - LOG_COLL_CLIENT_AUTH, - LOG_COLL_CLIENT_DNS, - LOG_COLL_CLIENT_DONE, - LOG_COLL_CLIENT_FAIL, - LOG_COLL_CLIENT_IDLE, - LOG_COLL_CLIENT_INIT, - LOG_COLL_CLIENT_OPEN, - LOG_COLL_CLIENT_SEND - }; - - enum ClientFlowControl { - LOG_COLL_FLOW_ALLOW, - LOG_COLL_FLOW_DENY, - }; - -private: - // client states - int client_auth(int event, VIO *vio); - int client_dns(int event, HostDBInfo *hostdb_info); - int client_done(int event, void *data); - int client_fail(int event, void *data); - int client_idle(int event, void *data); - int client_init(int event, void *data); - int client_open(int event, NetVConnection *net_vc); - int client_send(int event, VIO *vio); - ClientState m_client_state = LOG_COLL_CLIENT_START; - - // support functions - void flush_to_orphan(); - - // iocore stuff (two buffers to avoid races) - NetVConnection *m_host_vc = nullptr; - VIO *m_host_vio = nullptr; - MIOBuffer *m_auth_buffer = nullptr; - IOBufferReader *m_auth_reader = nullptr; - MIOBuffer *m_send_buffer = nullptr; - IOBufferReader *m_send_reader = nullptr; - Action *m_pending_action = nullptr; - Event *m_pending_event = nullptr; - - // to detect server closes (there's got to be a better way to do this) - VIO *m_abort_vio = nullptr; - MIOBuffer *m_abort_buffer = nullptr; - bool m_host_is_up = false; - - // send stuff - LogBufferList *m_buffer_send_list = nullptr; - LogBuffer *m_buffer_in_iocore = nullptr; - ClientFlowControl m_flow = LOG_COLL_FLOW_ALLOW; - - // back pointer to LogHost container - LogHost *m_log_host; - - // debugging - static int ID; - int m_id = 0; -}; - -typedef int (LogCollationClientSM::*LogCollationClientSMHandler)(int, void *); diff --git a/proxy/logging/LogCollationHostSM.cc b/proxy/logging/LogCollationHostSM.cc deleted file mode 100644 index 2962271a93d..00000000000 --- a/proxy/logging/LogCollationHostSM.cc +++ /dev/null @@ -1,527 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -//------------------------------------------------------------------------- -// include files -//------------------------------------------------------------------------- - -#include "tscore/ink_config.h" - -#include -#include -#include -#include -#include - -#include "P_EventSystem.h" -#include "P_Net.h" - -#include "LogUtils.h" -#include "LogSock.h" -#include "LogField.h" -#include "LogFile.h" -#include "LogFormat.h" -#include "LogBuffer.h" -#include "LogHost.h" -#include "LogObject.h" -#include "LogConfig.h" -#include "Log.h" - -#include "LogCollationHostSM.h" - -//------------------------------------------------------------------------- -// statics -//------------------------------------------------------------------------- - -int LogCollationHostSM::ID = 0; - -//------------------------------------------------------------------------- -// LogCollationHostSM::LogCollationHostSM -//------------------------------------------------------------------------- - -LogCollationHostSM::LogCollationHostSM(NetVConnection *client_vc) - : Continuation(new_ProxyMutex()), - m_client_vc(client_vc), - m_client_vio(nullptr), - m_client_buffer(nullptr), - m_client_reader(nullptr), - m_pending_event(nullptr), - m_read_buffer(nullptr), - m_read_bytes_wanted(0), - m_read_bytes_received(0), - m_read_buffer_fast_allocator_size(-1), - m_client_ip(0), - m_client_port(0), - m_id(ID++) -{ - RecInt rec_timeout; - int timeout = 86390; - Debug("log-coll", "[%d]host::constructor", m_id); - - ink_assert(m_client_vc != nullptr); - - // assign an explicit inactivity timeout so that it will not get the default value later - if (RecGetRecordInt("proxy.config.log.collation_host_timeout", &rec_timeout) == REC_ERR_OKAY) { - timeout = rec_timeout; - } - m_client_vc->set_inactivity_timeout(HRTIME_SECONDS(timeout)); - - // get client info - m_client_ip = m_client_vc->get_remote_ip(); - m_client_port = m_client_vc->get_remote_port(); - Note("[log-coll] client connected [%d.%d.%d.%d:%d]", ((unsigned char *)(&m_client_ip))[0], ((unsigned char *)(&m_client_ip))[1], - ((unsigned char *)(&m_client_ip))[2], ((unsigned char *)(&m_client_ip))[3], m_client_port); - - SET_HANDLER((LogCollationHostSMHandler)&LogCollationHostSM::host_handler); - host_init(LOG_COLL_EVENT_SWITCH, nullptr); -} - -void -LogCollationHostSM::freeReadBuffer() -{ - if (m_read_buffer) { - if (m_read_buffer_fast_allocator_size >= 0) { - ioBufAllocator[m_read_buffer_fast_allocator_size].free_void(m_read_buffer); - } else { - ats_free(m_read_buffer); - } - m_read_buffer = nullptr; - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// handlers -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_handler -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_handler(int event, void *data) -{ - switch (m_host_state) { - case LOG_COLL_HOST_AUTH: - return host_auth(event, data); - case LOG_COLL_HOST_DONE: - return host_done(event, data); - case LOG_COLL_HOST_INIT: - return host_init(event, data); - case LOG_COLL_HOST_RECV: - return host_recv(event, data); - default: - ink_assert(!"unexpected state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_handler -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_handler(int event, void *data) -{ - switch (m_read_state) { - case LOG_COLL_READ_BODY: - return read_body(event, (VIO *)data); - case LOG_COLL_READ_HDR: - return read_hdr(event, (VIO *)data); - default: - ink_assert(!"unexpected state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// host states -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_auth -// next: host_done || host_recv -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_auth(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]host::host_auth", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]host::host_auth - SWITCH", m_id); - m_host_state = LOG_COLL_HOST_AUTH; - return read_start(); - - case LOG_COLL_EVENT_READ_COMPLETE: - Debug("log-coll", "[%d]host::host_auth - READ_COMPLETE", m_id); - { - // compare authorization secrets - ink_assert(m_read_buffer != nullptr); - int diff = strncmp(m_read_buffer, Log::config->collation_secret, m_read_bytes_received); - freeReadBuffer(); - if (!diff) { - Debug("log-coll", "[%d]host::host_auth - authenticated!", m_id); - return host_recv(LOG_COLL_EVENT_SWITCH, nullptr); - } else { - Debug("log-coll", "[%d]host::host_auth - authenticated failed!", m_id); - Note("[log-coll] authentication failed [%d.%d.%d.%d:%d]", ((unsigned char *)(&m_client_ip))[0], - ((unsigned char *)(&m_client_ip))[1], ((unsigned char *)(&m_client_ip))[2], ((unsigned char *)(&m_client_ip))[3], - m_client_port); - return host_done(LOG_COLL_EVENT_SWITCH, nullptr); - } - } - - case LOG_COLL_EVENT_ERROR: - Debug("log-coll", "[%d]host::host_auth - ERROR", m_id); - return host_done(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_done -// next: none -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_done(int /* event ATS_UNUSED */, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]host::host_done", m_id); - - // close connections - if (m_client_vc) { - Debug("log-coll", "[%d]host::host_done - disconnecting!", m_id); - m_client_vc->do_io_close(); - m_client_vc = nullptr; - Note("[log-coll] client disconnected [%d.%d.%d.%d:%d]", ((unsigned char *)(&m_client_ip))[0], - ((unsigned char *)(&m_client_ip))[1], ((unsigned char *)(&m_client_ip))[2], ((unsigned char *)(&m_client_ip))[3], - m_client_port); - } - // free memory - if (m_client_buffer) { - if (m_client_reader) { - m_client_buffer->dealloc_reader(m_client_reader); - } - free_MIOBuffer(m_client_buffer); - } - // delete this state machine and return - delete this; - return EVENT_DONE; -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_init -// next: host_auth || host_done -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_init(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]host::host_init", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_host_state = LOG_COLL_HOST_INIT; - m_pending_event = eventProcessor.schedule_imm(this); - return EVENT_CONT; - - case EVENT_IMMEDIATE: - // allocate memory - m_client_buffer = new_MIOBuffer(); - ink_assert(m_client_buffer != nullptr); - m_client_reader = m_client_buffer->alloc_reader(); - ink_assert(m_client_reader != nullptr); - return host_auth(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected state"); - return EVENT_DONE; - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_recv -// next: host_done || host_recv -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_recv(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]host::host_recv", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]host::host_recv - SWITCH", m_id); - m_host_state = LOG_COLL_HOST_RECV; - return read_start(); - - case LOG_COLL_EVENT_READ_COMPLETE: - Debug("log-coll", "[%d]host::host_recv - READ_COMPLETE", m_id); - { - // grab the log_buffer - LogBufferHeader *log_buffer_header; - LogBuffer *log_buffer; - LogFormat *log_format; - LogObject *log_object; - unsigned version; - - ink_assert(m_read_buffer != nullptr); - ink_assert(m_read_bytes_received >= (int64_t)sizeof(LogBufferHeader)); - log_buffer_header = (LogBufferHeader *)m_read_buffer; - - // convert the buffer we just received to host order - // TODO: We currently don't try to make the log buffers handle little vs big endian. TS-1156. - // LogBuffer::convert_to_host_order(log_buffer_header); - - version = log_buffer_header->version; - if (version != LOG_SEGMENT_VERSION) { - Note("[log-coll] invalid LogBuffer received; invalid version - " - "buffer = %u, current = %u", - version, LOG_SEGMENT_VERSION); - freeReadBuffer(); - - } else { - log_object = Log::match_logobject(log_buffer_header); - if (!log_object) { - Note("[log-coll] LogObject not found with fieldlist id; " - "writing LogBuffer to scrap file"); - log_object = Log::global_scrap_object; - } - log_format = log_object->m_format; - Debug("log-coll", "[%d]host::host_recv - using format '%s'", m_id, log_format->name()); - - // make a new LogBuffer (log_buffer_header plus subsequent - // buffer already converted to host order) and add it to the - // object's flush queue - // - log_buffer = new LogBuffer(log_object, log_buffer_header); - - RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_num_received_from_network_stat, log_buffer_header->entry_count); - - RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_bytes_received_from_network_stat, log_buffer_header->byte_count); - - int idx = log_object->add_to_flush_queue(log_buffer); - Log::preproc_notify[idx].signal(); - } - -#if defined(LOG_BUFFER_TRACKING) - Debug("log-buftrak", "[%d]host::host_recv - network read complete", log_buffer_header->id); -#endif // defined(LOG_BUFFER_TRACKING) - - // get ready for next read (memory may not be freed!!!) - m_read_buffer = nullptr; - - return host_recv(LOG_COLL_EVENT_SWITCH, nullptr); - } - - case LOG_COLL_EVENT_ERROR: - Debug("log-coll", "[%d]host::host_recv - ERROR", m_id); - return host_done(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected state"); - return EVENT_DONE; - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// read states -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_start -// next: read_hdr -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_start() -{ - Debug("log-coll", "[%d]host::read_start", m_id); - - SET_HANDLER((LogCollationHostSMHandler)&LogCollationHostSM::read_handler); - if (m_read_buffer) { - ink_assert(!"m_read_buffer still points to something, doh!"); - } - return read_hdr(LOG_COLL_EVENT_SWITCH, nullptr); -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_hdr -// next: read_body || read_done -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_hdr(int event, VIO *vio) -{ - Debug("log-coll", "[%d]host::read_hdr", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]host:read_hdr - SWITCH", m_id); - m_read_state = LOG_COLL_READ_HDR; - - m_read_bytes_wanted = sizeof(NetMsgHeader); - m_read_bytes_received = 0; - m_read_buffer = (char *)&m_net_msg_header; - ink_assert(m_client_vc != nullptr); - Debug("log-coll", "[%d]host:read_hdr - do_io_read(%" PRId64 ")", m_id, m_read_bytes_wanted); - m_client_vio = m_client_vc->do_io_read(this, m_read_bytes_wanted, m_client_buffer); - ink_assert(m_client_vio != nullptr); - return EVENT_CONT; - - case VC_EVENT_IMMEDIATE: - Debug("log-coll", "[%d]host::read_hdr - IMMEDIATE", m_id); - return EVENT_CONT; - - case VC_EVENT_READ_READY: - Debug("log-coll", "[%d]host::read_hdr - READ_READY", m_id); - read_partial(vio); - return EVENT_CONT; - - case VC_EVENT_READ_COMPLETE: - Debug("log-coll", "[%d]host::read_hdr - READ_COMPLETE", m_id); - read_partial(vio); - ink_assert(m_read_bytes_wanted == m_read_bytes_received); - return read_body(LOG_COLL_EVENT_SWITCH, nullptr); - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: - Debug("log-coll", "[%d]host::read_hdr - TIMEOUT|EOS|ERROR", m_id); - return read_done(LOG_COLL_EVENT_ERROR, nullptr); - - default: - Debug("log-coll", "[%d]host::read_hdr - default %d", m_id, event); - return read_done(LOG_COLL_EVENT_ERROR, nullptr); - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_body -// next: read_body || read_done -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_body(int event, VIO *vio) -{ - Debug("log-coll", "[%d]host::read_body", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]host:read_body - SWITCH", m_id); - m_read_state = LOG_COLL_READ_BODY; - - m_read_bytes_wanted = m_net_msg_header.msg_bytes; - ink_assert(m_read_bytes_wanted > 0); - m_read_bytes_received = 0; - if (m_read_bytes_wanted <= max_iobuffer_size) { - m_read_buffer_fast_allocator_size = buffer_size_to_index(m_read_bytes_wanted); - m_read_buffer = (char *)ioBufAllocator[m_read_buffer_fast_allocator_size].alloc_void(); - } else { - m_read_buffer_fast_allocator_size = -1; - m_read_buffer = (char *)ats_malloc(m_read_bytes_wanted); - } - ink_assert(m_read_buffer != nullptr); - ink_assert(m_client_vc != nullptr); - Debug("log-coll", "[%d]host:read_body - do_io_read(%" PRId64 ")", m_id, m_read_bytes_wanted); - m_client_vio = m_client_vc->do_io_read(this, m_read_bytes_wanted, m_client_buffer); - ink_assert(m_client_vio != nullptr); - return EVENT_CONT; - - case VC_EVENT_IMMEDIATE: - Debug("log-coll", "[%d]host::read_body - IMMEDIATE", m_id); - return EVENT_CONT; - - case VC_EVENT_READ_READY: - Debug("log-coll", "[%d]host::read_body - READ_READY", m_id); - read_partial(vio); - return EVENT_CONT; - - case VC_EVENT_READ_COMPLETE: - Debug("log-coll", "[%d]host::read_body - READ_COMPLETE", m_id); - read_partial(vio); - ink_assert(m_read_bytes_wanted == m_read_bytes_received); - return read_done(LOG_COLL_EVENT_READ_COMPLETE, nullptr); - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: - Debug("log-coll", "[%d]host::read_body - TIMEOUT|EOS|ERROR", m_id); - return read_done(LOG_COLL_EVENT_ERROR, nullptr); - - default: - Debug("log-coll", "[%d]host::read_body - default %d", m_id, event); - return read_done(LOG_COLL_EVENT_ERROR, nullptr); - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_done -// next: give control back to host state-machine -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_done(int event, void * /* data ATS_UNUSED */) -{ - SET_HANDLER((LogCollationHostSMHandler)&LogCollationHostSM::host_handler); - return host_handler(event, nullptr); -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_partial -//------------------------------------------------------------------------- - -void -LogCollationHostSM::read_partial(VIO *vio) -{ - // checks - ink_assert(vio != nullptr); - ink_assert(vio->vc_server == m_client_vc); - ink_assert(m_client_buffer != nullptr); - ink_assert(m_client_reader != nullptr); - - // careful not to read more than we have memory for - char *p = &(m_read_buffer[m_read_bytes_received]); - int64_t bytes_wanted_now = m_read_bytes_wanted - m_read_bytes_received; - int64_t bytes_received_now = m_client_reader->read(p, bytes_wanted_now); - - m_read_bytes_received += bytes_received_now; -} diff --git a/proxy/logging/LogCollationHostSM.h b/proxy/logging/LogCollationHostSM.h deleted file mode 100644 index d7038fca0b6..00000000000 --- a/proxy/logging/LogCollationHostSM.h +++ /dev/null @@ -1,109 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -//------------------------------------------------------------------------- -// includes -//------------------------------------------------------------------------- - -#include "P_EventSystem.h" -#include "LogCollationBase.h" - -//------------------------------------------------------------------------- -// pre-declarations -//------------------------------------------------------------------------- - -struct LogBufferHeader; - -//------------------------------------------------------------------------- -// LogCollationHostSM -//------------------------------------------------------------------------- - -class LogCollationHostSM : public LogCollationBase, public Continuation -{ -public: - LogCollationHostSM(NetVConnection *client_vc); - - // handlers - int host_handler(int event, void *data); - int read_handler(int event, void *data); - -private: - enum HostState { - LOG_COLL_HOST_NULL, - LOG_COLL_HOST_AUTH, - LOG_COLL_HOST_DONE, - LOG_COLL_HOST_INIT, - LOG_COLL_HOST_RECV, - }; - - enum ReadState { - LOG_COLL_READ_NULL, - LOG_COLL_READ_BODY, - LOG_COLL_READ_HDR, - }; - -private: - // host states - int host_init(int event, void *data); - int host_auth(int event, void *data); - int host_recv(int event, void *data); - int host_done(int event, void *data); - HostState m_host_state; - - // read states - int read_hdr(int event, VIO *vio); - int read_body(int event, VIO *vio); - int read_done(int event, void *data); - int read_start(); - void freeReadBuffer(); - ReadState m_read_state; - - // helper for read states - void read_partial(VIO *vio); - - // iocore stuff - NetVConnection *m_client_vc; - VIO *m_client_vio; - MIOBuffer *m_client_buffer; - IOBufferReader *m_client_reader; - Event *m_pending_event; - - // read_state stuff - NetMsgHeader m_net_msg_header; - char *m_read_buffer; - int64_t m_read_bytes_wanted; - int64_t m_read_bytes_received; - int64_t m_read_buffer_fast_allocator_size; - - // client info - int m_client_ip; - int m_client_port; - - // debugging - static int ID; - int m_id; -}; - -typedef int (LogCollationHostSM::*LogCollationHostSMHandler)(int, void *); diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc index e952605b4e8..2644011f3d5 100644 --- a/proxy/logging/LogConfig.cc +++ b/proxy/logging/LogConfig.cc @@ -41,14 +41,11 @@ #include "LogFormat.h" #include "LogFile.h" #include "LogBuffer.h" -#include "LogHost.h" #include "LogObject.h" #include "LogConfig.h" #include "LogUtils.h" #include "tscore/SimpleTokenizer.h" -#include "LogCollationAccept.h" - #include "YamlLogConfig.h" #define DISK_IS_CONFIG_FULL_MESSAGE \ @@ -75,22 +72,14 @@ LogConfig::setup_default_values() } hostname = ats_strdup(name); - log_buffer_size = (int)(10 * LOG_KILOBYTE); - max_secs_per_buffer = 5; - max_space_mb_for_logs = 100; - max_space_mb_for_orphan_logs = 25; - max_space_mb_headroom = 10; - logfile_perm = 0644; - logfile_dir = ats_strdup("."); - - collation_mode = Log::NO_COLLATION; - collation_host = ats_strdup("none"); - collation_port = 0; - collation_host_tagged = false; - collation_preproc_threads = 1; - collation_secret = ats_strdup("foobar"); - collation_retry_sec = 0; - collation_max_send_buffers = 0; + log_buffer_size = (int)(10 * LOG_KILOBYTE); + max_secs_per_buffer = 5; + max_space_mb_for_logs = 100; + max_space_mb_headroom = 10; + logfile_perm = 0644; + logfile_dir = ats_strdup("."); + + preproc_threads = 1; rolling_enabled = Log::NO_ROLLING; rolling_interval_sec = 86400; // 24 hours @@ -103,8 +92,6 @@ LogConfig::setup_default_values() file_stat_frequency = 16; space_used_frequency = 900; - use_orphan_log_space_value = false; - ascii_buffer_size = 4 * 9216; max_line_size = 9216; // size of pipe buffer for SunOS 5.6 } @@ -136,11 +123,6 @@ LogConfig::read_configuration_variables() max_space_mb_for_logs = val; } - val = (int)REC_ConfigReadInteger("proxy.config.log.max_space_mb_for_orphan_logs"); - if (val > 0) { - max_space_mb_for_orphan_logs = val; - } - val = (int)REC_ConfigReadInteger("proxy.config.log.max_space_mb_headroom"); if (val > 0) { max_space_mb_headroom = val; @@ -169,45 +151,9 @@ LogConfig::read_configuration_variables() ::exit(1); } - // COLLATION - val = (int)REC_ConfigReadInteger("proxy.local.log.collation_mode"); - // do not restrict value so that error message is logged if - // collation_mode is out of range - collation_mode = val; - - ptr = REC_ConfigReadString("proxy.config.log.collation_host"); - if (ptr != nullptr) { - ats_free(collation_host); - collation_host = ptr; - } - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_port"); - if (val >= 0) { - collation_port = val; - } - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_host_tagged"); - collation_host_tagged = (val > 0); - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_preproc_threads"); + val = (int)REC_ConfigReadInteger("proxy.config.log.preproc_threads"); if (val > 0 && val <= 128) { - collation_preproc_threads = val; - } - - ptr = REC_ConfigReadString("proxy.config.log.collation_secret"); - if (ptr != nullptr) { - ats_free(collation_secret); - collation_secret = ptr; - } - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_retry_sec"); - if (val >= 0) { - collation_retry_sec = val; - } - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_max_send_buffers"); - if (val >= 0) { - collation_max_send_buffers = val; + preproc_threads = val; } // ROLLING @@ -302,77 +248,7 @@ LogConfig::LogConfig() : m_partition_space_left((int64_t)UINT_MAX) LogConfig::~LogConfig() { - // we don't delete the log collation accept because it may be transferred - // to another LogConfig object - // - // delete m_log_collation_accept; - - ats_free(hostname); ats_free(logfile_dir); - ats_free(collation_host); - ats_free(collation_secret); -} - -/*------------------------------------------------------------------------- - LogConfig::setup_collation - -------------------------------------------------------------------------*/ - -void -LogConfig::setup_collation(LogConfig *prev_config) -{ - // Set-up the collation status, but only if collation is enabled and - // there are valid entries for the collation host and port. - // - if (collation_mode < Log::NO_COLLATION || collation_mode >= Log::N_COLLATION_MODES) { - Note("Invalid value %d for proxy.local.log.collation_mode" - " configuration variable (valid range is from %d to %d)\n" - "Log collation disabled", - collation_mode, Log::NO_COLLATION, Log::N_COLLATION_MODES - 1); - } else if (collation_mode == Log::NO_COLLATION) { - // if the previous configuration had a collation accept, delete it - // - if (prev_config && prev_config->m_log_collation_accept) { - delete prev_config->m_log_collation_accept; - prev_config->m_log_collation_accept = nullptr; - } - } else { - Warning("Log collation is deprecated as of ATS v8.0.0!"); - if (!collation_port) { - Note("Cannot activate log collation, %d is an invalid collation port", collation_port); - } else if (collation_mode > Log::COLLATION_HOST && strcmp(collation_host, "none") == 0) { - Note("Cannot activate log collation, \"%s\" is an invalid collation host", collation_host); - } else { - if (collation_mode == Log::COLLATION_HOST) { - ink_assert(m_log_collation_accept == nullptr); - - if (prev_config && prev_config->m_log_collation_accept) { - if (prev_config->collation_port == collation_port) { - m_log_collation_accept = prev_config->m_log_collation_accept; - } else { - delete prev_config->m_log_collation_accept; - } - } - - if (!m_log_collation_accept) { - Log::collation_port = collation_port; - m_log_collation_accept = new LogCollationAccept(collation_port); - } - Debug("log", "I am a collation host listening on port %d.", collation_port); - } else { - Debug("log", - "I am a collation client (%d)." - " My collation host is %s:%d", - collation_mode, collation_host, collation_port); - } - - Debug("log", "using iocore log collation"); - if (collation_host_tagged) { - LogFormat::turn_tagging_on(); - } else { - LogFormat::turn_tagging_off(); - } - } - } } /*------------------------------------------------------------------------- @@ -386,8 +262,6 @@ LogConfig::init(LogConfig *prev_config) ink_assert(!initialized); - setup_collation(prev_config); - update_space_used(); // create log objects @@ -404,7 +278,7 @@ LogConfig::init(LogConfig *prev_config) Debug("log", "creating predefined error log object"); errlog = new LogObject(fmt.get(), logfile_dir, "error.log", LOG_FILE_ASCII, nullptr, (Log::RollingEnabledValues)rolling_enabled, - collation_preproc_threads, rolling_interval_sec, rolling_offset_hr, rolling_size_mb); + preproc_threads, rolling_interval_sec, rolling_offset_hr, rolling_size_mb); log_object_manager.manage_object(errlog); errlog->set_fmt_timestamps(); @@ -425,16 +299,6 @@ LogConfig::init(LogConfig *prev_config) ink_atomic_swap(&Log::error_log, errlog); - // determine if we should use the orphan log space value or not - // we use it if all objects are collation clients, or if some are and - // the specified space for collation is larger than that for local files - // - size_t num_collation_clients = log_object_manager.get_num_collation_clients(); - use_orphan_log_space_value = (num_collation_clients == 0 ? false : - (log_object_manager.get_num_objects() == num_collation_clients ? - true : - max_space_mb_for_orphan_logs > max_space_mb_for_logs)); - initialized = true; } @@ -454,18 +318,12 @@ LogConfig::display(FILE *fd) fprintf(fd, " log_buffer_size = %d\n", log_buffer_size); fprintf(fd, " max_secs_per_buffer = %d\n", max_secs_per_buffer); fprintf(fd, " max_space_mb_for_logs = %d\n", max_space_mb_for_logs); - fprintf(fd, " max_space_mb_for_orphan_logs = %d\n", max_space_mb_for_orphan_logs); - fprintf(fd, " use_orphan_log_space_value = %d\n", use_orphan_log_space_value); fprintf(fd, " max_space_mb_headroom = %d\n", max_space_mb_headroom); fprintf(fd, " hostname = %s\n", hostname); fprintf(fd, " logfile_dir = %s\n", logfile_dir); fprintf(fd, " logfile_perm = 0%o\n", logfile_perm); - fprintf(fd, " collation_mode = %d\n", collation_mode); - fprintf(fd, " collation_host = %s\n", collation_host); - fprintf(fd, " collation_port = %d\n", collation_port); - fprintf(fd, " collation_host_tagged = %d\n", collation_host_tagged); - fprintf(fd, " collation_preproc_threads = %d\n", collation_preproc_threads); - fprintf(fd, " collation_secret = %s\n", collation_secret); + + fprintf(fd, " preproc_threads = %d\n", preproc_threads); fprintf(fd, " rolling_enabled = %d\n", rolling_enabled); fprintf(fd, " rolling_interval_sec = %d\n", rolling_interval_sec); fprintf(fd, " rolling_offset_hr = %d\n", rolling_offset_hr); @@ -543,29 +401,11 @@ void LogConfig::register_config_callbacks() { static const char *names[] = { - "proxy.config.log.log_buffer_size", - "proxy.config.log.max_secs_per_buffer", - "proxy.config.log.max_space_mb_for_logs", - "proxy.config.log.max_space_mb_for_orphan_logs", - "proxy.config.log.max_space_mb_headroom", - "proxy.config.log.logfile_perm", - "proxy.config.log.hostname", - "proxy.config.log.logfile_dir", - "proxy.local.log.collation_mode", - "proxy.config.log.collation_host", - "proxy.config.log.collation_port", - "proxy.config.log.collation_host_tagged", - "proxy.config.log.collation_secret", - "proxy.config.log.collation_retry_sec", - "proxy.config.log.collation_max_send_buffers", - "proxy.config.log.rolling_enabled", - "proxy.config.log.rolling_interval_sec", - "proxy.config.log.rolling_offset_hr", - "proxy.config.log.rolling_size_mb", - "proxy.config.log.auto_delete_rolled_files", - "proxy.config.log.config.filename", - "proxy.config.log.sampling_frequency", - "proxy.config.log.file_stat_frequency", + "proxy.config.log.log_buffer_size", "proxy.config.log.max_secs_per_buffer", "proxy.config.log.max_space_mb_for_logs", + "proxy.config.log.max_space_mb_headroom", "proxy.config.log.logfile_perm", "proxy.config.log.hostname", + "proxy.config.log.logfile_dir", "proxy.config.log.rolling_enabled", "proxy.config.log.rolling_interval_sec", + "proxy.config.log.rolling_offset_hr", "proxy.config.log.rolling_size_mb", "proxy.config.log.auto_delete_rolled_files", + "proxy.config.log.config.filename", "proxy.config.log.sampling_frequency", "proxy.config.log.file_stat_frequency", "proxy.config.log.space_used_frequency", }; diff --git a/proxy/logging/LogConfig.h b/proxy/logging/LogConfig.h index fdf3def9799..c36d38b0681 100644 --- a/proxy/logging/LogConfig.h +++ b/proxy/logging/LogConfig.h @@ -77,7 +77,6 @@ enum { extern RecRawStatBlock *log_rsb; struct dirent; -struct LogCollationAccept; /*------------------------------------------------------------------------- LogDeleteCandidate, LogDeletingInfo&Descriptor @@ -188,12 +187,6 @@ class LogConfig : public ConfigInfo bool space_to_write(int64_t bytes_to_write) const; - bool - am_collation_host() const - { - return collation_mode == Log::COLLATION_HOST; - } - bool space_is_short() const { @@ -216,7 +209,7 @@ class LogConfig : public ConfigInfo int get_max_space_mb() const { - return (use_orphan_log_space_value ? max_space_mb_for_orphan_logs : max_space_mb_for_logs); + return max_space_mb_for_logs; } void @@ -247,15 +240,11 @@ class LogConfig : public ConfigInfo int log_buffer_size; int max_secs_per_buffer; int max_space_mb_for_logs; - int max_space_mb_for_orphan_logs; int max_space_mb_headroom; int logfile_perm; - int collation_mode; - int collation_port; - bool collation_host_tagged; - int collation_preproc_threads; - int collation_retry_sec; - int collation_max_send_buffers; + + int preproc_threads; + Log::RollingEnabledValues rolling_enabled; int rolling_interval_sec; int rolling_offset_hr; @@ -274,23 +263,13 @@ class LogConfig : public ConfigInfo char *hostname; char *logfile_dir; - char *collation_host; - char *collation_secret; private: bool evaluate_config(); void setup_default_values(); - void setup_collation(LogConfig *prev_config); private: - // if true, use max_space_mb_for_orphan_logs to determine the amount - // of space that logging can use, otherwise use max_space_mb_for_logs - // - bool use_orphan_log_space_value; - - LogCollationAccept *m_log_collation_accept = nullptr; - bool m_disk_full = false; bool m_disk_low = false; bool m_partition_full = false; diff --git a/proxy/logging/LogFile.cc b/proxy/logging/LogFile.cc index 7d70bbdde4a..3f9a5819b63 100644 --- a/proxy/logging/LogFile.cc +++ b/proxy/logging/LogFile.cc @@ -38,7 +38,6 @@ #include "P_EventSystem.h" #include "I_Machine.h" -#include "LogSock.h" #include "tscore/BaseLogFile.h" #include "LogField.h" @@ -46,7 +45,6 @@ #include "LogFormat.h" #include "LogBuffer.h" #include "LogFile.h" -#include "LogHost.h" #include "LogObject.h" #include "LogUtils.h" #include "LogConfig.h" diff --git a/proxy/logging/LogFile.h b/proxy/logging/LogFile.h index f439cb23ef1..fe3b1102e5f 100644 --- a/proxy/logging/LogFile.h +++ b/proxy/logging/LogFile.h @@ -29,7 +29,6 @@ #include "tscore/ink_platform.h" #include "LogBufferSink.h" -class LogSock; class LogBuffer; struct LogBufferHeader; class LogObject; diff --git a/proxy/logging/LogFilter.cc b/proxy/logging/LogFilter.cc index 900268f06bf..8b85394e883 100644 --- a/proxy/logging/LogFilter.cc +++ b/proxy/logging/LogFilter.cc @@ -34,7 +34,6 @@ #include "LogFormat.h" #include "LogFile.h" #include "LogBuffer.h" -#include "LogHost.h" #include "LogObject.h" #include "LogConfig.h" #include "Log.h" diff --git a/proxy/logging/LogFormat.cc b/proxy/logging/LogFormat.cc index e4571d5b644..e87e1542122 100644 --- a/proxy/logging/LogFormat.cc +++ b/proxy/logging/LogFormat.cc @@ -40,7 +40,6 @@ #include "LogField.h" #include "LogFilter.h" #include "LogFormat.h" -#include "LogHost.h" #include "LogBuffer.h" #include "LogObject.h" #include "LogConfig.h" @@ -201,32 +200,6 @@ LogFormat::LogFormat(const char *name, const char *format_str, unsigned interval m_format_type = format_str ? LOG_FORMAT_CUSTOM : LOG_FORMAT_TEXT; } -//----------------------------------------------------------------------------- -// This constructor is used only in Log::match_logobject -// -// It is awkward because it does not take a format_str but a fieldlist_str -// and a printf_str. These should be derived from a format_str but a -// LogBufferHeader does not store the format_str, if it did, we could probably -// delete this. -// -LogFormat::LogFormat(const char *name, const char *fieldlist_str, const char *printf_str, unsigned interval_sec) - : m_interval_sec(0), - m_interval_next(0), - m_agg_marshal_space(nullptr), - m_valid(false), - m_name_str(nullptr), - m_name_id(0), - m_fieldlist_str(nullptr), - m_fieldlist_id(0), - m_field_count(0), - m_printf_str(nullptr), - m_aggregate(false), - m_format_str(nullptr) -{ - init_variables(name, fieldlist_str, printf_str, interval_sec); - m_format_type = LOG_FORMAT_CUSTOM; -} - /*------------------------------------------------------------------------- LogFormat::LogFormat diff --git a/proxy/logging/LogHost.cc b/proxy/logging/LogHost.cc deleted file mode 100644 index 649682b1cbb..00000000000 --- a/proxy/logging/LogHost.cc +++ /dev/null @@ -1,466 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/*************************************************************************** - LogHost.cc - - - ***************************************************************************/ -#include "tscore/ink_platform.h" - -#include "LogUtils.h" -#include "LogSock.h" -#include "LogField.h" -#include "LogFile.h" -#include "LogFormat.h" -#include "LogBuffer.h" -#include "LogHost.h" -#include "LogObject.h" -#include "LogConfig.h" -#include "Log.h" - -#include "LogCollationClientSM.h" - -#define PING true -#define NOPING false - -static Ptr -make_orphan_logfile(LogHost *lh, const char *filename) -{ - const char *ext = "orphan"; - unsigned name_len = (unsigned)(strlen(filename) + strlen(lh->name()) + strlen(ext) + 16); - char *name_buf = (char *)ats_malloc(name_len); - - // NT: replace ':'s with '-'s. This change is necessary because - // NT doesn't like filenames with ':'s in them. ^_^ - snprintf(name_buf, name_len, "%s%s%s-%u.%s", filename, LOGFILE_SEPARATOR_STRING, lh->name(), lh->port(), ext); - - // XXX should check for conflicts with orphan filename - - Ptr orphan(new LogFile(name_buf, nullptr, LOG_FILE_ASCII, lh->signature())); - - ats_free(name_buf); - return orphan; -} - -/*------------------------------------------------------------------------- - LogHost - -------------------------------------------------------------------------*/ - -LogHost::LogHost(const char *object_filename, uint64_t object_signature) - : m_object_filename(ats_strdup(object_filename)), - m_object_signature(object_signature), - m_port(0), - m_name(nullptr), - m_sock(nullptr), - m_sock_fd(-1), - m_connected(false), - m_orphan_file(nullptr), - m_log_collation_client_sm(nullptr) -{ - ink_zero(m_ip); - ink_zero(m_ipstr); -} - -LogHost::LogHost(const LogHost &rhs) - : m_object_filename(ats_strdup(rhs.m_object_filename)), - m_object_signature(rhs.m_object_signature), - m_ip(rhs.m_ip), - m_port(0), - m_name(ats_strdup(rhs.m_name)), - m_sock(nullptr), - m_sock_fd(-1), - m_connected(false), - m_orphan_file(nullptr), - m_log_collation_client_sm(nullptr) -{ - memcpy(m_ipstr, rhs.m_ipstr, sizeof(m_ipstr)); - m_orphan_file = make_orphan_logfile(this, m_object_filename); -} - -LogHost::~LogHost() -{ - clear(); - ats_free(m_object_filename); -} - -// -// There are 3 ways to establish a LogHost: -// - by "hostname:port" or IP:port", where IP is a string of the -// form "xxx.xxx.xxx.xxx". -// - by specifying a hostname and a port (as separate arguments). -// - by specifying an ip and a port (as separate arguments). -// -bool -LogHost::set_name_port(const char *hostname, unsigned int pt) -{ - if (!hostname || hostname[0] == 0) { - Note("Cannot establish LogHost with NULL hostname"); - return false; - } - - clear(); // remove all previous state for this LogHost - - m_name = ats_strdup(hostname); - m_port = pt; - - Debug("log-host", "LogHost established as %s:%u", this->name(), this->port()); - - m_orphan_file = make_orphan_logfile(this, m_object_filename); - return true; -} - -bool -LogHost::set_ipstr_port(const char *ipstr, unsigned int pt) -{ - if (!ipstr || ipstr[0] == 0) { - Note("Cannot establish LogHost with NULL ipstr"); - return false; - } - - clear(); // remove all previous state for this LogHost - - if (0 != m_ip.load(ipstr)) { - Note("Log host failed to parse IP address %s", ipstr); - } - - m_port = pt; - ink_strlcpy(m_ipstr, ipstr, sizeof(m_ipstr)); - m_name = ats_strdup(ipstr); - - Debug("log-host", "LogHost established as %s:%u", name(), pt); - - m_orphan_file = make_orphan_logfile(this, m_object_filename); - return true; -} - -bool -LogHost::set_name_or_ipstr(const char *name_or_ip) -{ - if (name_or_ip && name_or_ip[0] != '\0') { - std::string_view addr, port; - if (ats_ip_parse(std::string_view(name_or_ip), &addr, &port) == 0) { - uint16_t p = port.empty() ? Log::config->collation_port : atoi(port.data()); - char *n = const_cast(addr.data()); - // Force termination. We know we can do this because the address - // string is followed by either a nul or a colon. - n[addr.size()] = 0; - if (AF_UNSPEC == ats_ip_check_characters(addr)) { - return set_name_port(n, p); - } else { - return set_ipstr_port(n, p); - } - } - } - - return false; -} - -bool -LogHost::connected(bool ping) -{ - if (m_connected && m_sock && m_sock_fd >= 0) { - if (m_sock->is_connected(m_sock_fd, ping)) { - return true; - } - } - return false; -} - -bool -LogHost::connect() -{ - if (!m_ip.isValid()) { - Note("Cannot connect to LogHost; host IP has not been established"); - return false; - } - - if (connected(PING)) { - return true; - } - - IpEndpoint target; - ip_port_text_buffer ipb; - target.assign(m_ip, htons(m_port)); - - if (is_debug_tag_set("log-host")) { - Debug("log-host", "Connecting to LogHost %s", ats_ip_nptop(&target, ipb, sizeof ipb)); - } - - disconnect(); // make sure connection members are initialized - - if (m_sock == nullptr) { - m_sock = new LogSock(); - ink_assert(m_sock != nullptr); - } - m_sock_fd = m_sock->connect(&target.sa); - if (m_sock_fd < 0) { - Note("Connection to LogHost %s failed", ats_ip_nptop(&target, ipb, sizeof ipb)); - return false; - } - m_connected = true; - - if (!authenticated()) { - Note("Authentication to LogHost %s failed", ats_ip_nptop(&target, ipb, sizeof ipb)); - disconnect(); - return false; - } - - return true; -} - -void -LogHost::disconnect() -{ - if (m_sock && m_sock_fd >= 0) { - m_sock->close(m_sock_fd); - m_sock_fd = -1; - } - if (m_log_collation_client_sm) { - delete m_log_collation_client_sm; - m_log_collation_client_sm = nullptr; - } - m_connected = false; -} - -// -// preprocess the given buffer data before sent to target host -// and try to delete it when its reference become zero. -// -bool -LogHost::preproc_and_try_delete(LogBuffer *&lb) -{ - if (lb == nullptr) { - Note("Cannot write LogBuffer to LogHost %s; LogBuffer is NULL", name()); - return false; - } - - LogBufferHeader *buffer_header = lb->header(); - if (buffer_header == nullptr) { - Note("Cannot write LogBuffer to LogHost %s; LogBufferHeader is NULL", name()); - goto done; - } - - if (buffer_header->entry_count == 0) { - // no bytes to write - goto done; - } - - // create a new collation client if necessary - if (m_log_collation_client_sm == nullptr) { - m_log_collation_client_sm = new LogCollationClientSM(this); - ink_assert(m_log_collation_client_sm != nullptr); - } - - // send log_buffer - if (m_log_collation_client_sm->send(lb) <= 0) { - goto done; - } - - return true; - -done: - LogBuffer::destroy(lb); - return false; -} - -// -// write the given buffer data to orphan file and -// try to delete it when its reference become zero. -// -void -LogHost::orphan_write_and_try_delete(LogBuffer *&lb) -{ - RecIncrRawStat(log_rsb, this_thread()->mutex->thread_holding, log_stat_num_lost_before_sent_to_network_stat, - lb->header()->entry_count); - - RecIncrRawStat(log_rsb, this_thread()->mutex->thread_holding, log_stat_bytes_lost_before_sent_to_network_stat, - lb->header()->byte_count); - - if (!Log::config->logging_space_exhausted) { - Debug("log-host", "Sending LogBuffer to orphan file %s", m_orphan_file->get_name()); - m_orphan_file->preproc_and_try_delete(lb); - } else { - Debug("log-host", "logging space exhausted, failed to write orphan file, drop(%" PRIu32 ") bytes", lb->header()->byte_count); - } - LogBuffer::destroy(lb); -} - -void -LogHost::display(FILE *fd) -{ - fprintf(fd, "LogHost: %s:%u, %s\n", name(), port(), (connected(NOPING)) ? "connected" : "not connected"); - - LogHost *host = this; - while (host->failover_link.next != nullptr) { - fprintf(fd, "Failover: %s:%u, %s\n", host->name(), host->port(), (host->connected(NOPING)) ? "connected" : "not connected"); - host = host->failover_link.next; - } -} - -void -LogHost::clear() -{ - // close an established connection and clear the state of this host - - disconnect(); - - ats_free(m_name); - delete m_sock; - m_orphan_file.clear(); - - ink_zero(m_ip); - m_port = 0; - ink_zero(m_ipstr); - m_name = nullptr; - m_sock = nullptr; - m_sock_fd = -1; - m_connected = false; -} - -bool -LogHost::authenticated() -{ - if (!connected(NOPING)) { - Note("Cannot authenticate LogHost %s; not connected", name()); - return false; - } - - Debug("log-host", "Authenticating LogHost %s ...", name()); - char *auth_key = Log::config->collation_secret; - unsigned auth_key_len = (unsigned)::strlen(auth_key) + 1; // incl null - int bytes = m_sock->write(m_sock_fd, auth_key, auth_key_len); - if ((unsigned)bytes != auth_key_len) { - Debug("log-host", "... bad write on authenticate"); - return false; - } - - Debug("log-host", "... authenticated"); - return true; -} - -/*------------------------------------------------------------------------- - LogHostList - -------------------------------------------------------------------------*/ - -LogHostList::LogHostList() {} - -LogHostList::~LogHostList() -{ - clear(); -} - -void -LogHostList::add(LogHost *object, bool copy) -{ - ink_assert(object != nullptr); - if (copy) { - m_host_list.enqueue(new LogHost(*object)); - } else { - m_host_list.enqueue(object); - } -} - -unsigned -LogHostList::count() -{ - unsigned cnt = 0; - for (LogHost *host = first(); host; host = next(host)) { - cnt++; - } - return cnt; -} - -void -LogHostList::clear() -{ - LogHost *host; - while ((host = m_host_list.dequeue())) { - delete host; - } -} - -int -LogHostList::preproc_and_try_delete(LogBuffer *lb) -{ - int success = false; - unsigned nr; - bool need_orphan = true; - LogHost *available_host = nullptr; - - ink_release_assert(lb->m_references == 0); - - nr = count(); - ink_atomic_increment(&lb->m_references, 1); - - for (LogHost *host = first(); host && nr; host = next(host)) { - LogHost *lh = host; - available_host = lh; - - do { - ink_atomic_increment(&lb->m_references, 1); - success = lh->preproc_and_try_delete(lb); - need_orphan = need_orphan && (success == false); - } while (lb && (success == false) && (lh = lh->failover_link.next)); - - nr--; - } - - if (lb != nullptr && need_orphan && available_host) { - ink_atomic_increment(&lb->m_references, 1); - available_host->orphan_write_and_try_delete(lb); - } - - if (lb != nullptr) { - LogBuffer::destroy(lb); - } - return 0; -} - -void -LogHostList::display(FILE *fd) -{ - for (LogHost *host = first(); host; host = next(host)) { - host->display(fd); - } -} - -bool -LogHostList::operator==(LogHostList &rhs) -{ - LogHost *host; - for (host = first(); host; host = next(host)) { - LogHost *rhs_host; - for (rhs_host = rhs.first(); rhs_host; rhs_host = next(host)) { - if ((host->port() == rhs_host->port() && host->ip_addr().isValid() && host->ip_addr() == rhs_host->ip_addr()) || - (host->name() && rhs_host->name() && (strcmp(host->name(), rhs_host->name()) == 0)) || - (*(host->ipstr()) && *(rhs_host->ipstr()) && (strcmp(host->ipstr(), rhs_host->ipstr()) == 0))) { - break; - } - } - if (rhs_host == nullptr) { - return false; - } - } - return true; -} diff --git a/proxy/logging/LogHost.h b/proxy/logging/LogHost.h deleted file mode 100644 index cb62e107fab..00000000000 --- a/proxy/logging/LogHost.h +++ /dev/null @@ -1,165 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -#pragma once - -class LogSock; -class LogBuffer; -class LogCollationClientSM; - -#include "LogBufferSink.h" - -/*------------------------------------------------------------------------- - LogHost - This object corresponds to a named log collation host. - -------------------------------------------------------------------------*/ -class LogHost -{ - friend class LogCollationClientSM; - -public: - LogHost(const char *object_filename, uint64_t object_signature); - LogHost(const LogHost &); - ~LogHost(); - - bool set_name_or_ipstr(const char *name_or_ipstr); - bool set_ipstr_port(const char *ipstr, unsigned int port); - bool set_name_port(const char *hostname, unsigned int port); - - bool connected(bool ping); - bool connect(); - void disconnect(); - // - // preprocess the given buffer data before sent to target host - // and try to delete it when its reference become zero. - // - bool preproc_and_try_delete(LogBuffer *&lb); - - // - // write the given buffer data to orphan file and - // try to delete it when its reference become zero. - // - void orphan_write_and_try_delete(LogBuffer *&lb); - - const char * - name() const - { - return m_name ? m_name : "UNKNOWN"; - } - - uint64_t - signature() const - { - return m_object_signature; - } - - IpAddr const & - ip_addr() const - { - return m_ip; - } - - in_port_t - port() const - { - return m_port; - } - - const char * - ipstr() const - { - return m_ipstr; - } - - void display(FILE *fd = stdout); - - LogFile * - get_orphan_logfile() const - { - return m_orphan_file.get(); - } - -private: - void clear(); - bool authenticated(); - -private: - char *m_object_filename; - uint64_t m_object_signature; - IpAddr m_ip; // IP address, network order. - in_port_t m_port; // IP port, host order. - ip_text_buffer m_ipstr; - char *m_name; - LogSock *m_sock; - int m_sock_fd; - bool m_connected; - Ptr m_orphan_file; - LogCollationClientSM *m_log_collation_client_sm; - -public: - LINK(LogHost, link); - SLINK(LogHost, failover_link); - - // noncopyable - LogHost &operator=(const LogHost &) = delete; - -private: - // -- member functions not allowed -- - LogHost(); -}; - -/*------------------------------------------------------------------------- - LogHostList - -------------------------------------------------------------------------*/ -class LogHostList : public LogBufferSink -{ -public: - LogHostList(); - ~LogHostList() override; - - void add(LogHost *host, bool copy = true); - unsigned count(); - void clear(); - int preproc_and_try_delete(LogBuffer *lb) override; - - LogHost * - first() - { - return m_host_list.head; - } - - LogHost * - next(LogHost *here) - { - return (here->link).next; - } - - void display(FILE *fd = stdout); - bool operator==(LogHostList &rhs); - - // -- member functions not allowed -- - LogHostList(const LogHostList &) = delete; - LogHostList &operator=(const LogHostList &) = delete; - -private: - Queue m_host_list; -}; diff --git a/proxy/logging/LogObject.cc b/proxy/logging/LogObject.cc index 336815cc507..cb91f5b6610 100644 --- a/proxy/logging/LogObject.cc +++ b/proxy/logging/LogObject.cc @@ -91,8 +91,7 @@ LogBufferManager::preproc_buffers(LogBufferSink *sink) LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *basename, LogFileFormat file_format, const char *header, Log::RollingEnabledValues rolling_enabled, int flush_threads, int rolling_interval_sec, int rolling_offset_hr, int rolling_size_mb, bool auto_created) - : m_auto_created(auto_created), - m_alt_filename(nullptr), + : m_alt_filename(nullptr), m_flags(0), m_signature(0), m_flush_threads(flush_threads), @@ -117,9 +116,6 @@ LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *b // compute_signature is a static function m_signature = compute_signature(m_format, m_basename, m_flags); - // by default, create a LogFile for this object, if a loghost is - // later specified, then we will delete the LogFile object - // m_logFile = new LogFile(m_filename, header, file_format, m_signature, Log::config->ascii_buffer_size, Log::config->max_line_size); LogBuffer *b = new LogBuffer(this, Log::config->log_buffer_size); @@ -133,7 +129,6 @@ LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *b LogObject::LogObject(LogObject &rhs) : RefCountObj(rhs), - m_auto_created(rhs.m_auto_created), m_basename(ats_strdup(rhs.m_basename)), m_filename(ats_strdup(rhs.m_filename)), m_alt_filename(ats_strdup(rhs.m_alt_filename)), @@ -162,11 +157,6 @@ LogObject::LogObject(LogObject &rhs) add_filter(filter); } - LogHost *host; - for (host = rhs.m_host_list.first(); host; host = rhs.m_host_list.next(host)) { - add_loghost(host); - } - // copy gets a fresh log buffer // LogBuffer *b = new LogBuffer(this, Log::config->log_buffer_size); @@ -184,13 +174,6 @@ LogObject::~LogObject() Debug("log-config", "entering LogObject destructor, this=%p", this); preproc_buffers(); - - // here we need to free LogHost if it is remote logging. - if (is_collation_client()) { - if (m_host_list.count()) { - m_host_list.clear(); - } - } ats_free(m_basename); ats_free(m_filename); ats_free(m_alt_filename); @@ -302,23 +285,6 @@ LogObject::set_filter_list(const LogFilterList &list, bool copy) m_filter_list.set_conjunction(list.does_conjunction()); } -void -LogObject::add_loghost(LogHost *host, bool copy) -{ - if (!host) { - return; - } - m_host_list.add(host, copy); - - // A LogObject either writes to a file, or sends to a collation host, but - // not both. By default, it writes to a file. If a LogHost is specified, - // then clear the intelligent Ptr containing LogFile. - // - m_logFile.clear(); - - Debug("log", "added log host %p to object %p for target %s:%d", host, this, host->name(), host->port()); -} - // we conpute the object signature from the fieldlist_str and the printf_str // of the LogFormat rather than from the format_str because the format_str // is not part of a LogBuffer header @@ -355,11 +321,8 @@ LogObject::display(FILE *fd) "flags = %u\n" "signature = %" PRIu64 "\n", this, m_format->name(), m_format, m_basename, m_flags, m_signature); - if (is_collation_client()) { - m_host_list.display(fd); - } else { - fprintf(fd, "full path = %s\n", get_full_filename()); - } + + fprintf(fd, "full path = %s\n", get_full_filename()); m_filter_list.display(fd); fprintf(fd, "++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"); } @@ -785,15 +748,8 @@ LogObject::_roll_files(long last_roll_time, long time_now) if (!writes_to_pipe()) { num_rolled += m_logFile->roll(last_roll_time, time_now); } - } else { - LogHost *host; - for (host = m_host_list.first(); host; host = m_host_list.next(host)) { - LogFile *orphan_logfile = host->get_orphan_logfile(); - if (orphan_logfile) { - num_rolled += orphan_logfile->roll(last_roll_time, time_now); - } - } } + m_last_roll_time = time_now; return num_rolled; } @@ -895,14 +851,10 @@ LogObjectManager::_manage_object(LogObject *log_object, bool is_api_object, int ACQUIRE_API_MUTEX("A LogObjectManager::_manage_object"); } - bool col_client = log_object->is_collation_client(); - int retVal = _solve_internal_filename_conflicts(log_object, maxConflicts); + int retVal = _solve_internal_filename_conflicts(log_object, maxConflicts); if (retVal == NO_FILENAME_CONFLICTS) { - // check for external conflicts only if the object is not a collation - // client - // - if (col_client || (retVal = _solve_filename_conflicts(log_object, maxConflicts), retVal == NO_FILENAME_CONFLICTS)) { + if (retVal = _solve_filename_conflicts(log_object, maxConflicts), retVal == NO_FILENAME_CONFLICTS) { // do filesystem checks // { @@ -920,16 +872,15 @@ LogObjectManager::_manage_object(LogObject *log_object, bool is_api_object, int Debug("log", "LogObjectManager managing object %s (%s) " "[signature = %" PRIu64 ", address = %p]", - log_object->get_base_filename(), col_client ? "collation client" : log_object->get_full_filename(), - log_object->get_signature(), log_object); + log_object->get_base_filename(), log_object->get_full_filename(), log_object->get_signature(), log_object); if (log_object->has_alternate_name()) { - Warning("The full path for the (%s) LogObject %s " + Warning("The full path for the (%s) LogObject " "with signature %" PRIu64 " " "has been set to %s rather than %s because the latter " "is being used by another LogObject", - log_object->receives_remote_data() ? "remote" : "local", log_object->get_base_filename(), - log_object->get_signature(), log_object->get_full_filename(), log_object->get_original_filename()); + log_object->get_base_filename(), log_object->get_signature(), log_object->get_full_filename(), + log_object->get_original_filename()); } } } @@ -1063,14 +1014,12 @@ bool LogObjectManager::_has_internal_filename_conflict(const char *filename, LogObjectList &objects) { for (auto &object : objects) { - if (!object->is_collation_client()) { - // an internal conflict exists if two objects request the - // same filename, regardless of the object signatures, since - // two objects writing to the same file would produce a - // log with duplicate entries and non monotonic timestamps - if (strcmp(object->get_full_filename(), filename) == 0) { - return true; - } + // an internal conflict exists if two objects request the + // same filename, regardless of the object signatures, since + // two objects writing to the same file would produce a + // log with duplicate entries and non monotonic timestamps + if (strcmp(object->get_full_filename(), filename) == 0) { + return true; } } return false; @@ -1189,7 +1138,7 @@ LogObjectManager::open_local_pipes() // for (unsigned i = 0; i < this->_objects.size(); i++) { LogObject *obj = _objects[i]; - if (obj->writes_to_pipe() && !obj->is_collation_client()) { + if (obj->writes_to_pipe()) { obj->m_logFile->open_file(); } } @@ -1299,19 +1248,6 @@ LogObjectManager::find_by_format_name(const char *name) const return nullptr; } -unsigned -LogObjectManager::get_num_collation_clients() const -{ - unsigned coll_clients = 0; - - for (auto _object : this->_objects) { - if (_object && _object->is_collation_client()) { - ++coll_clients; - } - } - return coll_clients; -} - int LogObjectManager::log(LogAccess *lad) { @@ -1319,15 +1255,6 @@ LogObjectManager::log(LogAccess *lad) ProxyMutex *mutex = this_thread()->mutex.get(); for (unsigned i = 0; i < this->_objects.size(); i++) { - // - // Auto created LogObject is only applied to LogBuffer - // data received from network in collation host. It should - // be ignored here. - // - if (_objects[i]->m_auto_created) { - continue; - } - ret |= _objects[i]->log(lad); } diff --git a/proxy/logging/LogObject.h b/proxy/logging/LogObject.h index dff58531f54..73bc3298078 100644 --- a/proxy/logging/LogObject.h +++ b/proxy/logging/LogObject.h @@ -28,7 +28,6 @@ #include "LogFile.h" #include "LogFormat.h" #include "LogFilter.h" -#include "LogHost.h" #include "LogBuffer.h" #include "LogAccess.h" #include "LogFilter.h" @@ -87,14 +86,11 @@ class LogObject : public RefCountObj public: enum LogObjectFlags { BINARY = 1, - REMOTE_DATA = 2, WRITES_TO_PIPE = 4, LOG_OBJECT_FMT_TIMESTAMP = 8, // always format a timestamp into each log line (for raw text logs) }; // BINARY: log is written in binary format (rather than ascii) - // REMOTE_DATA: object receives data from remote collation clients, so - // it should not be destroyed during a reconfiguration // WRITES_TO_PIPE: object writes to a named pipe rather than to a file LogObject(const LogFormat *format, const char *log_dir, const char *basename, LogFileFormat file_format, const char *header, @@ -105,13 +101,7 @@ class LogObject : public RefCountObj void add_filter(LogFilter *filter, bool copy = true); void set_filter_list(const LogFilterList &list, bool copy = true); - void add_loghost(LogHost *host, bool copy = true); - inline void - set_remote_flag() - { - m_flags |= REMOTE_DATA; - }; inline void set_fmt_timestamps() { @@ -153,11 +143,8 @@ class LogObject : public RefCountObj idx = m_buffer_manager_idx++ % m_flush_threads; } - if (m_logFile) { - nfb = m_buffer_manager[idx].preproc_buffers(m_logFile.get()); - } else { - nfb = m_buffer_manager[idx].preproc_buffers(&m_host_list); - } + nfb = m_buffer_manager[idx].preproc_buffers(m_logFile.get()); + return nfb; } @@ -226,16 +213,6 @@ class LogObject : public RefCountObj _setup_rolling(m_rolling_enabled, m_rolling_interval_sec, m_rolling_offset_hr, rolling_size_mb); } - inline bool - is_collation_client() const - { - return (m_logFile ? false : true); - } - inline bool - receives_remote_data() const - { - return (m_flags & REMOTE_DATA) ? true : false; - } inline bool writes_to_pipe() const { @@ -276,11 +253,9 @@ class LogObject : public RefCountObj bool operator==(LogObject &rhs); public: - bool m_auto_created; LogFormat *m_format; Ptr m_logFile; LogFilterList m_filter_list; - LogHostList m_host_list; private: char *m_basename; // the name of the file associated @@ -426,41 +401,19 @@ class LogObjectManager { return _objects.size(); } - unsigned get_num_collation_clients() const; }; inline bool LogObject::operator==(LogObject &old) { - if (!receives_remote_data() && !old.receives_remote_data()) { - return (get_signature() == old.get_signature() && - (is_collation_client() && old.is_collation_client() ? - m_host_list == old.m_host_list : - m_logFile && old.m_logFile && strcmp(m_logFile->get_name(), old.m_logFile->get_name()) == 0) && - (m_filter_list == old.m_filter_list) && - (m_rolling_interval_sec == old.m_rolling_interval_sec && m_rolling_offset_hr == old.m_rolling_offset_hr && - m_rolling_size_mb == old.m_rolling_size_mb)); - } - return false; + return (get_signature() == old.get_signature() && m_logFile && old.m_logFile && + strcmp(m_logFile->get_name(), old.m_logFile->get_name()) == 0 && (m_filter_list == old.m_filter_list) && + (m_rolling_interval_sec == old.m_rolling_interval_sec && m_rolling_offset_hr == old.m_rolling_offset_hr && + m_rolling_size_mb == old.m_rolling_size_mb)); } inline off_t LogObject::get_file_size_bytes() { - if (m_logFile) { - return m_logFile->get_size_bytes(); - } else { - off_t max_size = 0; - LogHost *host; - for (host = m_host_list.first(); host; host = m_host_list.next(host)) { - LogFile *orphan_logfile = host->get_orphan_logfile(); - if (orphan_logfile) { - off_t s = orphan_logfile->get_size_bytes(); - if (s > max_size) { - max_size = s; - } - } - } - return max_size; - } + return m_logFile->get_size_bytes(); } diff --git a/proxy/logging/LogSock.cc b/proxy/logging/LogSock.cc deleted file mode 100644 index bf9a9cdf9b3..00000000000 --- a/proxy/logging/LogSock.cc +++ /dev/null @@ -1,734 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "tscore/ink_inet.h" -#include "tscore/ink_string.h" -#include "P_EventSystem.h" - -#include "LogSock.h" -#include "LogUtils.h" - -static const int LS_SOCKTYPE = SOCK_STREAM; -static const int LS_PROTOCOL = 0; - -/** - LogSock - - The constructor establishes the connection table (ct) and initializes the - first entry of the table (index 0) to be the port on which new - connections are accepted. -*/ -LogSock::LogSock(int max_connects) : ct((ConnectTable *)nullptr), m_max_connections(max_connects + 1) -{ - ink_assert(m_max_connections > 0); - - // - // allocate space for the connection table. - // - ct = new ConnectTable[m_max_connections]; - ink_assert(ct != nullptr); - for (int i = 0; i < m_max_connections; ++i) { - init_cid(i, nullptr, 0, -1, LogSock::LS_STATE_UNUSED); - } - - Debug("log-sock", "LogSocket established"); -} - -/** - LogSock::~LogSock - - Shut down all connections and delete memory for the tables. -*/ -LogSock::~LogSock() -{ - Debug("log-sock", "shutting down LogSocket on [%s:%d]", ct[0].host, ct[0].port); - - this->close(); // close all connections - this->close(0); // close accept socket - delete[] ct; // delete the connection table -} - -/** - LogSock::listen - - This routine sets up the LogSock to begin accepting connections on the - given @param accept port. A maximum number of connections is also specified, - which is used to establish the size of the listen queue. - - @Return zero if all goes well, -1 otherwise. -*/ -int -LogSock::listen(int accept_port, int family) -{ - IpEndpoint bind_addr; - char this_host[MAXDNAME]; - int ret; - ats_scoped_fd accept_sd; - - Debug("log-sock", "Listening ..."); - - // Set up local address for bind. - bind_addr.setToAnyAddr(family); - if (!bind_addr.isValid()) { - Warning("Could not set up socket - invalid address family %d", family); - return -1; - } - bind_addr.port() = htons(accept_port); - - int size = ats_ip_size(&bind_addr.sa); - - // - // create the socket for accepting new connections - // - accept_sd = ::socket(family, LS_SOCKTYPE, LS_PROTOCOL); - if (accept_sd < 0) { - Warning("Could not create a socket for family %d: %s", family, strerror(errno)); - return -1; - } - // - // Set socket options (NO_LINGER, TCP_NODELAY, SO_REUSEADDR) - // - // CLOSE ON EXEC - if ((ret = safe_fcntl(accept_sd, F_SETFD, FD_CLOEXEC)) < 0) { - Warning("Could not set option CLOSE ON EXEC on socket (%d): %s", ret, strerror(errno)); - return -1; - } - // NO_LINGER - struct linger l; - l.l_onoff = 0; - l.l_linger = 0; - if ((ret = safe_setsockopt(accept_sd, SOL_SOCKET, SO_LINGER, (char *)&l, sizeof(l))) < 0) { - Warning("Could not set option NO_LINGER on socket (%d): %s", ret, strerror(errno)); - return -1; - } - // REUSEADDR - if ((ret = safe_setsockopt(accept_sd, SOL_SOCKET, SO_REUSEADDR, SOCKOPT_ON, sizeof(int))) < 0) { - Warning("Could not set option REUSEADDR on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - // Bind to local address. - if ((ret = safe_bind(accept_sd, &bind_addr.sa, size)) < 0) { - Warning("Could not bind port: %s", strerror(errno)); - return -1; - } - - if ((ret = safe_setsockopt(accept_sd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int))) < 0) { - Warning("Could not set option TCP_NODELAY on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - if ((ret = safe_setsockopt(accept_sd, SOL_SOCKET, SO_KEEPALIVE, SOCKOPT_ON, sizeof(int))) < 0) { - Warning("Could not set option SO_KEEPALIVE on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - // - // if the accept_port argument was zero, then the system just picked - // one for us, so we need to find out what it was and record it in the - // connection table correctly. - // - if (accept_port == 0) { - ret = safe_getsockname(accept_sd, &bind_addr.sa, &size); - if (ret == 0) { - accept_port = ntohs(bind_addr.port()); - } - } - // - // establish the listen queue for incomming connections - // - if ((ret = safe_listen(accept_sd, m_max_connections)) < 0) { - Warning("Could not establish listen queue: %s", strerror(errno)); - return -1; - } - // - // initialize the first entry of the table for accepting incoming - // connection requests. - // - if (gethostname(&this_host[0], MAXDNAME) != 0) { - snprintf(this_host, sizeof(this_host), "unknown-host"); - } - init_cid(0, this_host, accept_port, accept_sd, LogSock::LS_STATE_INCOMING); - - m_accept_connections = true; - Debug("log-sock", "LogSocket established on [%s:%d]", this_host, accept_port); - - accept_sd.release(); - return 0; -} - -/** - LogSock::accept - - Accept a new connection. This is a blocking operation, so you may want - to use one of the non-blocking pending_XXX calls to see if there is a - connection first. - @return This returns the table index for the new connection. -*/ -int -LogSock::accept() -{ - int cid, connect_sd; - IpEndpoint connect_addr; - socklen_t size = sizeof(connect_addr); - in_port_t connect_port; - - if (!m_accept_connections || ct[0].sd < 0) { - return LogSock::LS_ERROR_NO_CONNECTION; - } - - cid = new_cid(); - if (cid < 0) { - return LogSock::LS_ERROR_CONNECT_TABLE_FULL; - } - - Debug("log-sock", "waiting to accept a new connection"); - - connect_sd = ::accept(ct[0].sd, &connect_addr.sa, &size); - if (connect_sd < 0) { - return LogSock::LS_ERROR_ACCEPT; - } - connect_port = ntohs(connect_addr.port()); - - init_cid(cid, nullptr, connect_port, connect_sd, LogSock::LS_STATE_INCOMING); - - Debug("log-sock", "new connection accepted, cid = %d, port = %d", cid, connect_port); - - return cid; -} - -/** - LogSock::connect - - Establish a new connection to another machine [host:port], and place this - information into the connection and poll tables. -*/ -int -LogSock::connect(sockaddr const *ip) -{ - int cid, ret; - ats_scoped_fd connect_sd; - uint16_t port; - - if (!ats_is_ip(ip)) { - Note("Invalid host IP or port number for connection"); - return LogSock::LS_ERROR_NO_SUCH_HOST; - } - port = ntohs(ats_ip_port_cast(ip)); - - ip_port_text_buffer ipstr; - Debug("log-sock", "connecting to [%s:%d]", ats_ip_nptop(ip, ipstr, sizeof(ipstr)), port); - - // get an index into the connection table - cid = new_cid(); - if (cid < 0) { - Note("No more connections allowed for this socket"); - return LogSock::LS_ERROR_CONNECT_TABLE_FULL; - } - // initialize a new socket descriptor - connect_sd = ::socket(ip->sa_family, LS_SOCKTYPE, LS_PROTOCOL); - if (connect_sd < 0) { - Note("Error initializing socket for connection: %d", static_cast(connect_sd)); - return LogSock::LS_ERROR_SOCKET; - } - - if ((ret = safe_setsockopt(connect_sd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int))) < 0) { - Note("Could not set option TCP_NODELAY on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - if ((ret = safe_setsockopt(connect_sd, SOL_SOCKET, SO_KEEPALIVE, SOCKOPT_ON, sizeof(int))) < 0) { - Note("Could not set option SO_KEEPALIVE on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - // attempt to connect - if (::connect(connect_sd, ip, ats_ip_size(ip)) != 0) { - Note("Failure to connect"); - return LogSock::LS_ERROR_CONNECT; - } - - init_cid(cid, ipstr, port, connect_sd, LogSock::LS_STATE_OUTGOING); - - Debug("log-sock", "outgoing connection to [%s:%d] established, fd = %d", ipstr, port, cid); - - connect_sd.release(); - return cid; -} - -/** - LogSock::pending_data - - This private routine checks for incoming data on some of the socket - descriptors. - @return Returns true if there is something incoming, with *cid - set to the index corresponding to the incoming socket. -*/ -bool -LogSock::pending_data(int *cid, int timeout_msec, bool include_connects) -{ - int ret, n_poll_fds, i; - static struct pollfd fds[LS_CONST_MAX_CONNS]; - int fd_to_cid[LS_CONST_MAX_CONNS]; - - ink_assert(m_max_connections <= (LS_CONST_MAX_CONNS + 1)); - ink_assert(cid != nullptr); - ink_assert(timeout_msec >= 0); - - // - // we'll use the poll() routine, which replaces the select routine - // to support a larger number of socket descriptors. to use poll, - // we need to set-up a pollfd array for the socket descriptors - // that will be polled. - // - - if (*cid >= 0) { // look for data on this specific socket - - ink_assert(*cid < m_max_connections); - fds[0].fd = ct[*cid].sd; - fds[0].events = POLLIN; - fds[0].revents = 0; - fd_to_cid[0] = *cid; - n_poll_fds = 1; - - } else { // look for data on any INCOMING socket - int start_index; - if (include_connects) { - start_index = 0; - } else { - start_index = 1; - } - n_poll_fds = 0; - for (i = start_index; i < m_max_connections; i++) { - if (ct[i].state == LogSock::LS_STATE_INCOMING) { - fds[n_poll_fds].fd = ct[i].sd; - fds[n_poll_fds].events = POLLIN; - fds[n_poll_fds].revents = 0; - fd_to_cid[n_poll_fds] = i; - n_poll_fds++; - } - } - } - - if (n_poll_fds == 0) { - return false; - } - - ret = ::poll(fds, n_poll_fds, timeout_msec); - - if (ret == 0) { - return false; // timeout - } else if (ret < 0) { - Debug("log-sock", "error on poll"); - return false; // error - } - // - // a positive return value indicates how many descriptors had something - // waiting on them. We only care about finding one of them, so we'll - // look for the first one with an revents flag set to POLLIN. - // - - for (i = 0; i < n_poll_fds; i++) { - if (fds[i].revents & POLLIN) { - *cid = fd_to_cid[i]; - Debug("log-sock", "poll successful on index %d", *cid); - return true; - } - } - - Debug("log-sock", "invalid revents in the poll table"); - return false; -} - -/** - LogSock::pending_any - - Check for incomming data on any of the INCOMING sockets. -*/ -bool -LogSock::pending_any(int *cid, int timeout_msec) -{ - ink_assert(cid != nullptr); - *cid = -1; - if (m_accept_connections) { - return pending_data(cid, timeout_msec, true); - } else { - return pending_data(cid, timeout_msec, false); - } -} - -/*------------------------------------------------------------------------- - LogSock::pending_message_any - - Check for an incomming message on any of the INCOMING sockets, aside from - the socket reserved for accepting new connections. - -------------------------------------------------------------------------*/ - -bool -LogSock::pending_message_any(int *cid, int timeout_msec) -{ - ink_assert(cid != nullptr); - *cid = -1; - return pending_data(cid, timeout_msec, false); -} - -/** - LogSock::pending_message_on - - Check for incomming data on the specified socket. -*/ -bool -LogSock::pending_message_on(int cid, int timeout_msec) -{ - return pending_data(&cid, timeout_msec, false); -} - -/** - LogSock::pending_connect - - Check for an incoming connection request on the socket reserved for that - (cid = 0). -*/ -bool -LogSock::pending_connect(int timeout_msec) -{ - int cid = 0; - if (m_accept_connections) { - return pending_data(&cid, timeout_msec, true); - } else { - return false; - } -} - -/** - LogSock::close - - Close one (cid specified) or all (no argument) sockets, except for the - incomming connection socket. -*/ -void -LogSock::close(int cid) -{ - ink_assert(cid >= 0 && cid < m_max_connections); - - Debug("log-sock", "closing connection for cid %d", cid); - - if (ct[cid].state != LogSock::LS_STATE_UNUSED) { - ::close(ct[cid].sd); - delete ct[cid].host; - ct[cid].state = LogSock::LS_STATE_UNUSED; - } -} - -void -LogSock::close() -{ - for (int i = 1; i < m_max_connections; i++) { - this->close(i); - } -} - -/** - LogSock::write - - Write data onto the socket corresponding to the given cid. Return the - number of bytes actually written. -*/ -int -LogSock::write(int cid, void *buf, int bytes) -{ - LogSock::MsgHeader header = {0}; - header.msg_bytes = 0; - int ret; - - ink_assert(cid >= 0 && cid < m_max_connections); - - if (buf == nullptr || bytes == 0) { - return 0; - } - - if (ct[cid].state != LogSock::LS_STATE_OUTGOING) { - return LogSock::LS_ERROR_STATE; - } - - Debug("log-sock", "Sending %d bytes to cid %d", bytes, cid); - - // - // send the message header - // - Debug("log-sock", " sending header (%zu bytes)", sizeof(LogSock::MsgHeader)); - header.msg_bytes = bytes; - ret = ::send(ct[cid].sd, (char *)&header, sizeof(LogSock::MsgHeader), 0); - if (ret != sizeof(LogSock::MsgHeader)) { - return LogSock::LS_ERROR_WRITE; - } - // - // send the actual data - // - Debug("log-sock", " sending data (%d bytes)", bytes); - return ::send(ct[cid].sd, (char *)buf, bytes, 0); -} - -/** - LogSock::read - - Read data from the specified connection. This is a blocking call, so you - may want to use one of the pending_XXX calls to see if there is anything - to read first. Returns number of bytes read. -*/ -int -LogSock::read(int cid, void *buf, unsigned maxsize) -{ - LogSock::MsgHeader header; - unsigned size; - - ink_assert(cid >= 0 && cid < m_max_connections); - ink_assert(buf != nullptr); - - if (ct[cid].state != LogSock::LS_STATE_INCOMING) { - return LogSock::LS_ERROR_STATE; - } - - Debug("log-sock", "reading data from cid %d", cid); - - if (read_header(ct[cid].sd, &header) < 0) { - return LogSock::LS_ERROR_READ; - } - - size = ((unsigned)header.msg_bytes < maxsize) ? (unsigned)header.msg_bytes : maxsize; - return read_body(ct[cid].sd, buf, size); -} - -/** - LogSock::read_alloc - - This routine reads data from the specified connection, and returns a - pointer to newly allocated space (allocated with new) containing the - data. The number of bytes read is set in the argument size, which is - expected to be a pointer to an int. -*/ -void * -LogSock::read_alloc(int cid, int *size) -{ - LogSock::MsgHeader header; - char *data; - - ink_assert(cid >= 0 && cid < m_max_connections); - - if (ct[cid].state != LogSock::LS_STATE_INCOMING) { - return nullptr; - } - - Debug("log-sock", "reading data from cid %d", cid); - - if (read_header(ct[cid].sd, &header) < 0) { - return nullptr; - } - - data = new char[header.msg_bytes]; - ink_assert(data != nullptr); - - if ((*size = read_body(ct[cid].sd, data, header.msg_bytes)) < 0) { - delete[] data; - data = nullptr; - } - - return data; -} - -/** - */ -bool -LogSock::is_connected(int cid, bool ping) const -{ - ink_assert(cid >= 0 && cid < m_max_connections); - - if (ct[cid].state == LogSock::LS_STATE_UNUSED) { - return false; - } - - if (ping) { - int flags = fcntl(ct[cid].sd, F_GETFL); - ::fcntl(ct[cid].sd, F_SETFL, O_NONBLOCK); - int i; - int j = ::recv(ct[cid].sd, (char *)&i, sizeof(int), MSG_PEEK); - ::fcntl(ct[cid].sd, F_SETFL, flags); - if (j != 0) { - return true; - } else { - return false; - } - } else { - return (ct[cid].sd >= 0); - } -} - -/** - */ -void -LogSock::check_connections() -{ - for (int i = 1; i < m_max_connections; i++) { - if (ct[i].state == LogSock::LS_STATE_INCOMING) { - if (!is_connected(i, true)) { - Debug("log-sock", "Connection %d is no longer connected", i); - close(i); - } - } - } -} - -/** - This routine will check to ensure that the client connecting is - authorized to use the log collation port. To authorize, the client is - expected to send the logging secret string. -*/ -bool -LogSock::authorized_client(int cid, char *key) -{ - // - // Wait for up to 5 seconds for the client to authenticate - // - if (!pending_message_on(cid, 5000)) { - return false; - } - // - // Ok, the client has a pending message, so check to see if it matches - // the given key. - // - char buf[1024]; - int size = this->read(cid, buf, 1024); - ink_assert(size >= 0 && size <= 1024); - - if (strncmp(buf, key, size) == 0) { - return true; - } - - return false; -} - -/** - */ -char * -LogSock::connected_host(int cid) -{ - ink_assert(cid >= 0 && cid < m_max_connections); - return ct[cid].host; -} - -/** - */ -int -LogSock::connected_port(int cid) -{ - ink_assert(cid >= 0 && cid < m_max_connections); - return ct[cid].port; -} - -/*------------------------------------------------------------------------- - LOCAL ROUTINES - -------------------------------------------------------------------------*/ - -/** - LogSock::new_cid -*/ -int -LogSock::new_cid() -{ - int cid = -1; - - for (int i = 1; i < m_max_connections; i++) { - if (ct[i].state == LogSock::LS_STATE_UNUSED) { - cid = i; - break; - } - } - - return cid; -} - -/** - LogSock::init_cid -*/ -void -LogSock::init_cid(int cid, char *host, int port, int sd, LogSock::State state) -{ - ink_assert(cid >= 0 && cid < m_max_connections); - // host can be NULL if it's not known - ink_assert(port >= 0); - // sd can be -1 to indicate no connection yet - ink_assert(state >= 0 && state < LogSock::LS_N_STATES); - - if (host != nullptr) { - const size_t host_size = strlen(host) + 1; - ct[cid].host = new char[host_size]; - ink_strlcpy(ct[cid].host, host, host_size); - } else { - ct[cid].host = nullptr; - } - - ct[cid].port = port; - ct[cid].sd = sd; - ct[cid].state = state; -} - -/** - */ -int -LogSock::read_header(int sd, LogSock::MsgHeader *header) -{ - ink_assert(sd >= 0); - ink_assert(header != nullptr); - - int bytes = ::recv(sd, (char *)header, sizeof(LogSock::MsgHeader), 0); - if (bytes != sizeof(LogSock::MsgHeader)) { - return -1; - } - - return bytes; -} - -/** - */ -int -LogSock::read_body(int sd, void *buf, int bytes) -{ - ink_assert(sd >= 0); - ink_assert(buf != nullptr); - ink_assert(bytes >= 0); - - if (bytes == 0) { - return 0; - } - - unsigned bytes_left = bytes; - char *to = (char *)buf; - - while (bytes_left) { - unsigned bytes_read = ::recv(sd, to, bytes_left, 0); - to += bytes_read; - bytes_left -= bytes_read; - } - - return bytes; -} diff --git a/proxy/logging/LogSock.h b/proxy/logging/LogSock.h deleted file mode 100644 index f2eb70e8324..00000000000 --- a/proxy/logging/LogSock.h +++ /dev/null @@ -1,128 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -#include "tscore/ink_platform.h" - -/*------------------------------------------------------------------------- - LogSock - - This class implements a multiplexed socket class that supports both - client and server functionality. - -------------------------------------------------------------------------*/ - -class LogSock -{ -public: - enum Constant { - LS_CONST_PACKETSIZE = 1024, - LS_CONST_MAX_CONNS = 256, - }; - - enum Err { - LS_ERROR_UNKNOWN = -1, - LS_ERROR_CONNECT_TABLE_FULL = -3, - LS_ERROR_SOCKET = -4, - LS_ERROR_BIND = -5, - LS_ERROR_CONNECT = -6, - LS_ERROR_ACCEPT = -7, - LS_ERROR_NO_SUCH_HOST = -8, - LS_ERROR_NO_CONNECTION = -9, - LS_ERROR_STATE = -10, - LS_ERROR_WRITE = -11, - LS_ERROR_READ = -12 - }; - - enum State { - LS_STATE_UNUSED = 0, - LS_STATE_INCOMING, - LS_STATE_OUTGOING, - LS_N_STATES, - }; - - explicit LogSock(int max_connects = 1); - ~LogSock(); - - bool pending_any(int *cid, int timeout_msec = 0); - bool pending_message_any(int *cid, int timeout_msec = 0); - bool pending_message_on(int cid, int timeout_msec = 0); - bool pending_connect(int timeout_msec = 0); - - int listen(int accept_port, int family = AF_INET); - int accept(); - int connect(sockaddr const *ip); - - void close(int cid); // this connection - void close(); // all connections - - int write(int cid, void *buf, int bytes); - - int read(int cid, void *buf, unsigned maxsize); - void *read_alloc(int cid, int *size); - - char * - on_host() - { - return ct[0].host; - } - - int - on_port() - { - return ct[0].port; - } - - bool is_connected(int cid, bool ping = false) const; - void check_connections(); - bool authorized_client(int cid, char *key); - char *connected_host(int cid); - int connected_port(int cid); - - // noncopyable - LogSock(const LogSock &) = delete; - LogSock &operator=(const LogSock &) = delete; - -private: - struct ConnectTable { - char *host; // hostname for this connection - int port; // port number for this connection - int sd; // socket descriptor for this connection - State state; // state of this entry - }; - - struct MsgHeader { - int msg_bytes; // length of the following message - }; - - bool pending_data(int *cid, int timeout_msec, bool include_connects); - int new_cid(); - void init_cid(int cid, char *host, int port, int sd, State state); - int read_header(int sd, MsgHeader *header); - int read_body(int sd, void *buf, int bytes); - - ConnectTable *ct; // list of all connections; index 0 is - // the accept port. - bool m_accept_connections = false; // do we accept new connections? - int m_max_connections; // max size of all tables -}; diff --git a/proxy/logging/Makefile.am b/proxy/logging/Makefile.am index 9f438edb1a4..abf8e866cfc 100644 --- a/proxy/logging/Makefile.am +++ b/proxy/logging/Makefile.am @@ -34,7 +34,7 @@ AM_CPPFLAGS += \ EXTRA_DIST = LogStandalone.cc -noinst_LIBRARIES = liblogging.a liblogcollation.a +noinst_LIBRARIES = liblogging.a liblogging_a_SOURCES = \ Log.cc \ @@ -56,28 +56,15 @@ liblogging_a_SOURCES = \ LogFilter.h \ LogFormat.cc \ LogFormat.h \ - LogHost.cc \ - LogHost.h \ LogLimits.h \ LogObject.cc \ LogObject.h \ - LogSock.cc \ - LogSock.h \ LogUtils.cc \ LogUtils.h \ YamlLogConfig.cc \ YamlLogConfigDecoders.cc \ YamlLogConfig.h -liblogcollation_a_SOURCES = \ - LogCollationAccept.cc \ - LogCollationAccept.h \ - LogCollationBase.h \ - LogCollationClientSM.cc \ - LogCollationClientSM.h \ - LogCollationHostSM.cc \ - LogCollationHostSM.h - check_PROGRAMS = \ test_LogUtils \ test_LogUtils2 @@ -111,5 +98,5 @@ test_LogUtils2_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/iocore/eventsystem/libinkevent.a -clang-tidy-local: $(liblogging_a_SOURCES) $(liblogcollation_a_SOURCES) $(EXTRA_DIST) +clang-tidy-local: $(liblogging_a_SOURCES) $(EXTRA_DIST) $(CXX_Clang_Tidy) diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc index 8988458f805..3e8d408e587 100644 --- a/proxy/logging/YamlLogConfig.cc +++ b/proxy/logging/YamlLogConfig.cc @@ -104,8 +104,8 @@ TsEnumDescriptor ROLLING_MODE_LUA = { {{"log.roll.none", 0}, {"log.roll.time", 1}, {"log.roll.size", 2}, {"log.roll.both", 3}, {"log.roll.any", 4}}}; std::set valid_log_object_keys = { - "filename", "format", "mode", "header", "rolling_enabled", "rolling_interval_sec", - "rolling_offset_hr", "rolling_size_mb", "filters", "collation_hosts", "min_count"}; + "filename", "format", "mode", "header", "rolling_enabled", "rolling_interval_sec", + "rolling_offset_hr", "rolling_size_mb", "filters", "min_count"}; LogObject * YamlLogConfig::decodeLogObject(const YAML::Node &node) @@ -183,8 +183,8 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } auto logObject = new LogObject(fmt, Log::config->logfile_dir, filename.c_str(), file_type, header.c_str(), - (Log::RollingEnabledValues)obj_rolling_enabled, Log::config->collation_preproc_threads, - obj_rolling_interval_sec, obj_rolling_offset_hr, obj_rolling_size_mb); + (Log::RollingEnabledValues)obj_rolling_enabled, obj_rolling_interval_sec, obj_rolling_offset_hr, + obj_rolling_size_mb); // Generate LogDeletingInfo entry for later use std::string ext; @@ -223,53 +223,5 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } } - auto collation_host_list = node["collation_hosts"]; - if (!collation_host_list) { - return logObject; - } - - if (!collation_host_list.IsSequence()) { - throw YAML::ParserException(collation_host_list.Mark(), "'collation_hosts' should be a list of collation_host objects"); - } - - for (auto const &collation_host : collation_host_list) { - if (!collation_host["host"]) { - Warning("no collation 'host' name; cannot add this Collation host"); - continue; - } - - auto collation_host_name = collation_host["host"].as(); - - LogHost *lh = new LogHost(logObject->get_full_filename(), logObject->get_signature()); - if (!lh->set_name_or_ipstr(collation_host_name.c_str())) { - Warning("Could not set \"%s\" as collation host", collation_host_name.c_str()); - delete lh; - continue; - } - - logObject->add_loghost(lh, false); - if (!collation_host["failover"]) { - continue; - } - - if (!collation_host["failover"].IsSequence()) { - delete lh; - throw YAML::ParserException(collation_host["failover"].Mark(), "'failover' should be a list of host names"); - } - - LogHost *prev = lh; - for (auto const &failover_host : collation_host["failover"]) { - auto failover_host_name = failover_host.as(); - LogHost *flh = new LogHost(logObject->get_full_filename(), logObject->get_signature()); - if (!flh->set_name_or_ipstr(failover_host_name.c_str())) { - Warning("Could not set \"%s\" as a failover host", failover_host_name.c_str()); - delete flh; - continue; - } - prev->failover_link.next = flh; - prev = flh; - } - } - return logObject; } diff --git a/proxy/shared/UglyLogStubs.cc b/proxy/shared/UglyLogStubs.cc index ce4d97be2b4..15ec22e53af 100644 --- a/proxy/shared/UglyLogStubs.cc +++ b/proxy/shared/UglyLogStubs.cc @@ -75,22 +75,6 @@ Machine::instance() return nullptr; } -#include "LogCollationAccept.h" -LogCollationAccept::LogCollationAccept(int port) : Continuation(new_ProxyMutex()), m_port(port) {} -LogCollationAccept::~LogCollationAccept() {} - -#include "LogCollationClientSM.h" -LogCollationClientSM::LogCollationClientSM(LogHost *log_host) : Continuation(new_ProxyMutex()), m_log_host(log_host) {} - -LogCollationClientSM::~LogCollationClientSM() {} - -int -LogCollationClientSM::send(LogBuffer * /* log_buffer ATS_UNUSED */) -{ - ink_release_assert(false); - return 0; -} - NetAccept * UnixNetProcessor::createNetAccept(const NetProcessor::AcceptOptions &opt) { diff --git a/src/traffic_logcat/logcat.cc b/src/traffic_logcat/logcat.cc index b1cd96491fd..fef226b3dc2 100644 --- a/src/traffic_logcat/logcat.cc +++ b/src/traffic_logcat/logcat.cc @@ -38,12 +38,10 @@ #include "LogFilter.h" #include "LogFormat.h" #include "LogFile.h" -#include "LogHost.h" #include "LogObject.h" #include "LogConfig.h" #include "LogBuffer.h" #include "LogUtils.h" -#include "LogSock.h" #include "Log.h" // logcat-specific command-line flags diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 6945a96a969..94b829fdda3 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -7257,10 +7257,9 @@ TSTextLogObjectCreate(const char *filename, int mode, TSTextLogObject *new_objec return TS_ERROR; } - TextLogObject *tlog = - new TextLogObject(filename, Log::config->logfile_dir, (bool)mode & TS_LOG_MODE_ADD_TIMESTAMP, nullptr, - Log::config->rolling_enabled, Log::config->collation_preproc_threads, Log::config->rolling_interval_sec, - Log::config->rolling_offset_hr, Log::config->rolling_size_mb); + TextLogObject *tlog = new TextLogObject( + filename, Log::config->logfile_dir, (bool)mode & TS_LOG_MODE_ADD_TIMESTAMP, nullptr, Log::config->rolling_enabled, + Log::config->preproc_threads, Log::config->rolling_interval_sec, Log::config->rolling_offset_hr, Log::config->rolling_size_mb); if (tlog == nullptr) { *new_object = nullptr; return TS_ERROR; diff --git a/src/traffic_server/Makefile.inc b/src/traffic_server/Makefile.inc index e828109b508..a18d6cec51d 100644 --- a/src/traffic_server/Makefile.inc +++ b/src/traffic_server/Makefile.inc @@ -65,7 +65,6 @@ traffic_server_traffic_server_LDADD = \ $(top_builddir)/proxy/http2/libhttp2.a \ $(top_builddir)/proxy/logging/liblogging.a \ $(top_builddir)/proxy/hdrs/libhdrs.a \ - $(top_builddir)/proxy/logging/liblogcollation.a \ $(top_builddir)/proxy/shared/libdiagsconfig.a \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/iocore/utils/libinkutils.a \ From 33097a8d385b933811107db8775c6856fb57f56d Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Thu, 25 Apr 2019 17:50:48 +0800 Subject: [PATCH 494/526] cppcheck: Fix various issues in proxy/http2/ --- proxy/http2/HPACK.cc | 4 +-- proxy/http2/HPACK.h | 13 ++++++++-- proxy/http2/Http2Stream.cc | 7 +++-- proxy/http2/Http2Stream.h | 3 ++- proxy/http2/HuffmanCodec.cc | 9 ++++--- proxy/http2/test_HPACK.cc | 26 ++++++++++--------- .../unit_tests/test_Http2DependencyTree.cc | 6 ++--- 7 files changed, 40 insertions(+), 28 deletions(-) diff --git a/proxy/http2/HPACK.cc b/proxy/http2/HPACK.cc index 1c45d1e178e..5abe3de420c 100644 --- a/proxy/http2/HPACK.cc +++ b/proxy/http2/HPACK.cc @@ -385,7 +385,7 @@ bool HpackDynamicTable::update_maximum_size(uint32_t new_size) { while (_current_size > new_size) { - if (_headers.size() <= 0) { + if (_headers.size() == 0) { return false; } int last_name_len, last_value_len; @@ -450,7 +450,7 @@ encode_string(uint8_t *buf_start, const uint8_t *buf_end, const char *value, siz int64_t data_len = 0; // TODO Choose whether to use Huffman encoding wisely - + // cppcheck-suppress knownConditionTrueFalse; leaving "use_huffman" for wise huffman usage in the future if (use_huffman && value_len) { data = static_cast(ats_malloc(value_len * 4)); if (data == nullptr) { diff --git a/proxy/http2/HPACK.h b/proxy/http2/HPACK.h index 9f93d4e4fba..78efb296b26 100644 --- a/proxy/http2/HPACK.h +++ b/proxy/http2/HPACK.h @@ -105,7 +105,7 @@ class MIMEFieldWrapper class HpackDynamicTable { public: - HpackDynamicTable(uint32_t size) : _current_size(0), _maximum_size(size) + explicit HpackDynamicTable(uint32_t size) : _current_size(0), _maximum_size(size) { _mhdr = new MIMEHdr(); _mhdr->create(); @@ -119,6 +119,10 @@ class HpackDynamicTable delete _mhdr; } + // noncopyable + HpackDynamicTable(HpackDynamicTable &) = delete; + HpackDynamicTable &operator=(const HpackDynamicTable &) = delete; + const MIMEField *get_header_field(uint32_t index) const; void add_header_field(const MIMEField *field); @@ -140,8 +144,13 @@ class HpackDynamicTable class HpackIndexingTable { public: - HpackIndexingTable(uint32_t size) { _dynamic_table = new HpackDynamicTable(size); } + explicit HpackIndexingTable(uint32_t size) { _dynamic_table = new HpackDynamicTable(size); } ~HpackIndexingTable() { delete _dynamic_table; } + + // noncopyable + HpackIndexingTable(HpackIndexingTable &) = delete; + HpackIndexingTable &operator=(const HpackIndexingTable &) = delete; + HpackLookupResult lookup(const MIMEFieldWrapper &field) const; HpackLookupResult lookup(const char *name, int name_len, const char *value, int value_len) const; int get_header_field(uint32_t index, MIMEFieldWrapper &header_field) const; diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index c6a83ed7c74..46614a63847 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -154,11 +154,10 @@ Http2Stream::send_request(Http2ConnectionState &cstate) int bufindex; int dumpoffset = 0; int done, tmp; - IOBufferBlock *block; do { - bufindex = 0; - tmp = dumpoffset; - block = request_buffer.get_current_block(); + bufindex = 0; + tmp = dumpoffset; + IOBufferBlock *block = request_buffer.get_current_block(); if (!block) { request_buffer.add_block(); block = request_buffer.get_current_block(); diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index a753d0db9dc..71fc00d7c2e 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -41,6 +41,8 @@ class Http2Stream : public ProxyClientTransaction typedef ProxyClientTransaction super; ///< Parent type. Http2Stream(Http2StreamId sid = 0, ssize_t initial_rwnd = Http2::initial_window_size) : client_rwnd(initial_rwnd), _id(sid) { + http_parser_init(&http_parser); + SET_HANDLER(&Http2Stream::main_event_handler); } @@ -52,7 +54,6 @@ class Http2Stream : public ProxyClientTransaction _thread = this_ethread(); this->client_rwnd = initial_rwnd; sm_reader = request_reader = request_buffer.alloc_reader(); - http_parser_init(&http_parser); // FIXME: Are you sure? every "stream" needs request_header? _req_header.create(HTTP_TYPE_REQUEST); response_header.create(HTTP_TYPE_RESPONSE); diff --git a/proxy/http2/HuffmanCodec.cc b/proxy/http2/HuffmanCodec.cc index 0c103731c14..562fa36b88a 100644 --- a/proxy/http2/HuffmanCodec.cc +++ b/proxy/http2/HuffmanCodec.cc @@ -93,12 +93,12 @@ static Node * make_huffman_tree() { Node *root = make_huffman_tree_node(); - Node *current; - uint32_t bit_len; + // insert leafs for each ascii code for (unsigned i = 0; i < countof(huffman_table); i++) { - bit_len = huffman_table[i].bit_len; - current = root; + uint32_t bit_len = huffman_table[i].bit_len; + Node *current = root; + while (bit_len > 0) { if (huffman_table[i].code_as_hex & (1 << (bit_len - 1))) { if (!current->right) { @@ -116,6 +116,7 @@ make_huffman_tree() current->ascii_code = i; current->leaf_node = true; } + return root; } diff --git a/proxy/http2/test_HPACK.cc b/proxy/http2/test_HPACK.cc index b1d4ed459d1..be29e8fdd6a 100644 --- a/proxy/http2/test_HPACK.cc +++ b/proxy/http2/test_HPACK.cc @@ -67,10 +67,9 @@ int unpack(string &packed, uint8_t *unpacked) { int n = packed.length() / 2; - int u, l; for (int i = 0; i < n; ++i) { - u = packed[i * 2]; - l = packed[i * 2 + 1]; + int u = packed[i * 2]; + int l = packed[i * 2 + 1]; unpacked[i] = (((u >= 'a') ? u - 'a' + 10 : u - '0') << 4) + ((l >= 'a') ? l - 'a' + 10 : l - '0'); } return n; @@ -129,21 +128,21 @@ print_difference(const char *a_str, const int a_str_len, const char *b_str, cons int compare_header_fields(HTTPHdr *a, HTTPHdr *b) { - const char *a_str, *b_str; - int a_str_len, b_str_len; + // compare fields count + if (a->fields_count() != b->fields_count()) { + return -1; + } + MIMEFieldIter a_iter, b_iter; const MIMEField *a_field = a->iter_get_first(&a_iter); const MIMEField *b_field = b->iter_get_first(&b_iter); - // compare fields count - if (a->fields_count() != b->fields_count()) { - return -1; - } - for (; a_field != nullptr; a_field = a->iter_get_next(&a_iter), b_field = b->iter_get_next(&b_iter)) { + while (a_field != nullptr && b_field != nullptr) { + int a_str_len, b_str_len; // compare header name - a_str = a_field->name_get(&a_str_len); - b_str = b_field->name_get(&b_str_len); + const char *a_str = a_field->name_get(&a_str_len); + const char *b_str = b_field->name_get(&b_str_len); if (a_str_len != b_str_len) { if (memcmp(a_str, b_str, a_str_len) != 0) { print_difference(a_str, a_str_len, b_str, b_str_len); @@ -159,6 +158,9 @@ compare_header_fields(HTTPHdr *a, HTTPHdr *b) return -1; } } + + a_field = a->iter_get_next(&a_iter); + b_field = b->iter_get_next(&b_iter); } return 0; diff --git a/proxy/http2/unit_tests/test_Http2DependencyTree.cc b/proxy/http2/unit_tests/test_Http2DependencyTree.cc index b72bbce4dcd..6a00b1ae4c3 100644 --- a/proxy/http2/unit_tests/test_Http2DependencyTree.cc +++ b/proxy/http2/unit_tests/test_Http2DependencyTree.cc @@ -282,7 +282,7 @@ TEST_CASE("Http2DependencyTree_Chrome_50", "[http2][Http2DependencyTree]") ostringstream oss; - for (int i = 0; i < 108; ++i) { + for (int index = 0; index < 108; ++index) { Node *node = tree->top(); oss << static_cast(node->t)->c_str(); @@ -336,7 +336,7 @@ TEST_CASE("Http2DependencyTree_Chrome_51", "[http2][Http2DependencyTree]") ostringstream oss; - for (int i = 0; i < 9; ++i) { + for (int index = 0; index < 9; ++index) { Node *node = tree->top(); if (node != nullptr) { oss << static_cast(node->t)->c_str(); @@ -352,7 +352,7 @@ TEST_CASE("Http2DependencyTree_Chrome_51", "[http2][Http2DependencyTree]") tree->activate(node_f); tree->activate(node_h); - for (int i = 0; i < 9; ++i) { + for (int index = 0; index < 9; ++index) { Node *node = tree->top(); if (node != nullptr) { oss << static_cast(node->t)->c_str(); From 14143b7c4c77817eddd313f7c46c8924cff3c270 Mon Sep 17 00:00:00 2001 From: yangjian Date: Thu, 28 Mar 2019 16:00:43 +0800 Subject: [PATCH 495/526] Cache:ttl-in-cache should always override never-cache --- doc/admin-guide/files/cache.config.en.rst | 13 ++++++++++++- proxy/CacheControl.cc | 9 ++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/doc/admin-guide/files/cache.config.en.rst b/doc/admin-guide/files/cache.config.en.rst index 40d3a056176..3971b9a70c1 100644 --- a/doc/admin-guide/files/cache.config.en.rst +++ b/doc/admin-guide/files/cache.config.en.rst @@ -157,7 +157,8 @@ specifiers of the rule in question. =========================== ================================================ Value Effect =========================== ================================================ - ``never-cache`` Never cache specified objects. + ``never-cache`` Never cache specified objects, it will be + overwrited by ``ttl-in-cache``. ``ignore-no-cache`` Ignore all ``Cache-Control: no-cache`` headers. ``ignore-client-no-cache`` Ignore ``Cache-Control: no-cache`` headers from client requests. @@ -250,6 +251,16 @@ prefix. The former fails at this goal, because the second rule will match all Javascript files and will override any previous ``revalidate`` values that may have been set by prior rules. +ttl-in-cache and never-cache +---------------------------- + +When multiple rules are matched in the same request, ``never-cache`` will always +be overwrited by ``ttl-in-cache``. For example:: + + # ttl-in-cache=1d never-cache=false + dest_domain=example.com action=never-cache + dest_domain=example.com ttl-in-cache=1d + Examples ======== diff --git a/proxy/CacheControl.cc b/proxy/CacheControl.cc index 6e78ed0ebbf..423584e96b5 100644 --- a/proxy/CacheControl.cc +++ b/proxy/CacheControl.cc @@ -372,9 +372,12 @@ CacheControlRecord::UpdateMatch(CacheControlResult *result, RequestData *rdata) break; case CC_NEVER_CACHE: if (this->CheckForMatch(h_rdata, result->never_line) == true) { - result->never_cache = true; - result->never_line = this->line_num; - match = true; + // ttl-in-cache overrides never-cache + if (result->ttl_line == -1) { + result->never_cache = true; + result->never_line = this->line_num; + match = true; + } } break; case CC_STANDARD_CACHE: From 93961f82c6718ec7190b533be0900b27e6045211 Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Tue, 23 Apr 2019 10:15:41 +0800 Subject: [PATCH 496/526] Doc: Add links to Slicer plugin --- doc/admin-guide/plugins/index.en.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index 1df7cadf26b..91f9c1e0843 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -161,6 +161,7 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Multiplexer MySQL Remap Signed URLs + Slicer SSL Headers SSL Session Reuse System Statistics @@ -227,6 +228,11 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`Signed URLs ` Adds support for verifying URL signatures for incoming requests to either deny or redirect access. +:doc:`Slicer ` + Slice full file or range based requests into deterministic chunks, allowing large files to be + spread across multiple cache stripes. Allows range requests to be satisfied by stitching these + chunks together. + :doc:`SSL Session Reuse ` Coordinates Session ID and ticket based TLS session resumption between a group of ATS machines. From 5fda043ce0c29c8253e43a6d670a287b84554d9d Mon Sep 17 00:00:00 2001 From: Miles Libbey Date: Tue, 23 Apr 2019 09:57:34 +0800 Subject: [PATCH 497/526] Doc: add prefetch to plugin index page --- doc/admin-guide/plugins/index.en.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index 91f9c1e0843..da38b3c288e 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -221,6 +221,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`MySQL Remap ` Allows dynamic remaps from a MySQL database. +:doc:`Prefetch ` + Pre-fetch objects based on the requested URL path pattern. + :doc:`Remap Purge ` This remap plugin allows the administrator to easily setup remotely controlled ``PURGE`` for the content of an entire remap rule. From 67d98bbfcb619f39c516f3116fa7da9ee34bfb6e Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 29 Apr 2019 08:40:26 -0700 Subject: [PATCH 498/526] De-tabifies REVIEWERS --- REVIEWERS | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/REVIEWERS b/REVIEWERS index db992a5490e..628abc22baa 100644 --- a/REVIEWERS +++ b/REVIEWERS @@ -18,12 +18,12 @@ Committers: add modules as needed and any qualifications after your e-mail addre all/general interest jplevyak@apache.org bcall@apache.org - briang@apache.org + briang@apache.org mturk@apache.org zwoop@apache.org jim@apache.org ericb@apache.org - manjesh@apache.org + manjesh@apache.org lib/ts jplevyak@apache.org zwoop@apache.org @@ -33,19 +33,19 @@ lib/ts lib/records bcall@apache.org georgep@apache.org - zwoop@apache.org + zwoop@apache.org Event System/Buffering/VIO/VConnection jplevyak@apache.org bcall@apache.org - briang@apache.org + briang@apache.org georgep@apache.org Network I/O jplevyak@apache.org - not including SSL or UDP except where it intersects the other code bcall@apache.org georgep@apache.org - briang@apache.org + briang@apache.org jim@apache.org - zwoop@apache.org + zwoop@apache.org Raw Cache and AIO jplevyak@apache.org bcall@apache.org @@ -53,13 +53,12 @@ Raw Cache and AIO HTTP Caching bcall@apache.org jim@apache.org - zwoop@apache.org + zwoop@apache.org Block-Cache -Clustering jplevyak@apache.org georgep@apache.org jim@apache.org - zym@apache.org + zym@apache.org DNS/HostDB jplevyak@apache.org zwoop@apache.org @@ -68,11 +67,11 @@ DNS/HostDB ericb@apache.org FastIO Docs - dianes@apache.org + dianes@apache.org Examples/Plugins zwoop@apache.org bcall@apache.org - briang@apache.org + briang@apache.org ericb@apache.org HDRs bcall@apache.org @@ -81,7 +80,7 @@ HDRs HTTP bcall@apache.org zwoop@apache.org - briang@apache.org + briang@apache.org jim@apache.org ericb@apache.org Remap @@ -91,7 +90,7 @@ Remap ericb@apache.org API zwoop@apache.org - briang@apache.org + briang@apache.org bcall@apache.org georgep@apache.org ericb@apache.org @@ -106,8 +105,8 @@ MGMT Stats bcall@apache.org georgep@apache.org - briang@apache.org - zwoop@apache.org + briang@apache.org + zwoop@apache.org Build System zwoop@apache.org bcall@apache.org From 2050aae3cf5efe938d796ae5422a6522de99a89d Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 29 Apr 2019 13:36:03 -0700 Subject: [PATCH 499/526] Fixes logging after collation removal A followup to 82d564486e058ead160bfb8ad15e40b0507ff8aa --- proxy/logging/YamlLogConfig.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc index 3e8d408e587..db3580f7868 100644 --- a/proxy/logging/YamlLogConfig.cc +++ b/proxy/logging/YamlLogConfig.cc @@ -183,8 +183,8 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } auto logObject = new LogObject(fmt, Log::config->logfile_dir, filename.c_str(), file_type, header.c_str(), - (Log::RollingEnabledValues)obj_rolling_enabled, obj_rolling_interval_sec, obj_rolling_offset_hr, - obj_rolling_size_mb); + (Log::RollingEnabledValues)obj_rolling_enabled, Log::config->preproc_threads, + obj_rolling_interval_sec, obj_rolling_offset_hr, obj_rolling_size_mb); // Generate LogDeletingInfo entry for later use std::string ext; From 4a43f53e2ec6fedcb6b60d7787f1508eb8872c7d Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 7 Aug 2018 09:56:10 -0700 Subject: [PATCH 500/526] Fixes spelling in src --- include/tscore/ParseRules.h | 2 +- src/traffic_cache_tool/CacheDefs.cc | 2 +- src/traffic_cache_tool/CacheDefs.h | 2 +- src/traffic_cache_tool/CacheScan.cc | 4 ++-- src/traffic_cache_tool/CacheTool.cc | 2 +- src/traffic_ctl/config.cc | 8 +++---- src/traffic_ctl/traffic_ctl.h | 2 +- src/traffic_layout/engine.cc | 4 ++-- src/traffic_layout/engine.h | 4 ++-- src/traffic_layout/file_system.cc | 4 ++-- src/traffic_layout/file_system.h | 2 +- src/traffic_logstats/logstats.cc | 4 ++-- src/traffic_manager/AddConfigFilesHere.cc | 2 +- src/traffic_manager/traffic_manager.cc | 2 +- src/traffic_server/FetchSM.cc | 2 +- src/traffic_server/FetchSM.h | 4 ++-- src/traffic_server/HostStatus.cc | 2 +- src/traffic_server/InkAPI.cc | 4 ++-- src/traffic_server/InkAPITest.cc | 18 +++++++------- src/traffic_server/SocksProxy.cc | 6 ++--- src/traffic_server/traffic_server.cc | 2 +- src/traffic_top/traffic_top.cc | 2 +- src/traffic_wccp/wccp_client.cc | 2 +- src/tscore/ArgParser.cc | 10 ++++---- src/tscore/BaseLogFile.cc | 12 +++++----- src/tscore/BufferWriterFormat.cc | 8 +++---- src/tscore/CryptoHash.cc | 2 +- src/tscore/Diags.cc | 2 +- src/tscore/HostLookup.cc | 12 +++++----- src/tscore/IntrusivePtrTest.cc | 2 +- src/tscore/IpMap.cc | 10 ++++---- src/tscore/JeAllocator.cc | 4 ++-- src/tscore/MatcherUtils.cc | 4 ++-- src/tscore/ParseRules.cc | 6 ++--- src/tscore/TextBuffer.cc | 14 +++++------ src/tscore/Tokenizer.cc | 2 +- src/tscore/ink_base64.cc | 2 +- src/tscore/ink_file.cc | 4 ++-- src/tscore/ink_memory.cc | 6 ++--- src/tscore/ink_queue_utils.cc | 6 ++--- src/tscore/ink_res_init.cc | 8 +++---- src/tscore/ink_res_mkquery.cc | 4 ++-- src/tscore/ink_sock.cc | 2 +- src/tscore/ink_string++.cc | 2 +- src/tscore/ink_time.cc | 2 +- src/tscore/load_http_hdr.cc | 2 +- src/tscore/lockfile.cc | 4 ++-- src/tscore/unit_tests/test_AcidPtr.cc | 2 +- src/tscore/unit_tests/test_IntrusivePtr.cc | 2 +- src/tscore/unit_tests/test_arena.cc | 2 +- src/tscpp/api/AsyncHttpFetch.cc | 2 +- src/tscpp/api/GzipDeflateTransformation.cc | 4 ++-- src/tscpp/api/Transaction.cc | 2 +- src/tscpp/api/TransformationPlugin.cc | 6 ++--- src/tscpp/api/utils_internal.cc | 2 +- src/tscpp/util/TextView.cc | 4 ++-- src/wccp/WccpEndPoint.cc | 2 +- src/wccp/WccpLocal.h | 28 +++++++++++----------- src/wccp/WccpMsg.cc | 2 +- src/wccp/WccpUtil.h | 2 +- 60 files changed, 138 insertions(+), 138 deletions(-) diff --git a/include/tscore/ParseRules.h b/include/tscore/ParseRules.h index d8fe31e51a7..847522e7d7e 100644 --- a/include/tscore/ParseRules.h +++ b/include/tscore/ParseRules.h @@ -33,7 +33,7 @@ typedef unsigned int CTypeResult; // Set this to 0 to disable SI // decimal multipliers -#define USE_SI_MULTILIERS 1 +#define USE_SI_MULTIPLIERS 1 #define is_char_BIT (1 << 0) #define is_upalpha_BIT (1 << 1) diff --git a/src/traffic_cache_tool/CacheDefs.cc b/src/traffic_cache_tool/CacheDefs.cc index a87e3c5aad3..e84fd23db8b 100644 --- a/src/traffic_cache_tool/CacheDefs.cc +++ b/src/traffic_cache_tool/CacheDefs.cc @@ -874,7 +874,7 @@ Errata Stripe::loadMeta() { // Read from disk in chunks of this size. This needs to be a multiple of both the - // store block size and the directory entry size so neither goes acrss read boundaries. + // store block size and the directory entry size so neither goes across read boundaries. // Beyond that the value should be in the ~10MB range for what I guess is best performance // vs. blocking production disk I/O on a live system. constexpr static int64_t N = (1 << 8) * CacheStoreBlocks::SCALE * sizeof(CacheDirEntry); diff --git a/src/traffic_cache_tool/CacheDefs.h b/src/traffic_cache_tool/CacheDefs.h index 1d42d481695..8b42a92ba40 100644 --- a/src/traffic_cache_tool/CacheDefs.h +++ b/src/traffic_cache_tool/CacheDefs.h @@ -467,7 +467,7 @@ struct Span { /// Local copy of serialized header data stored on in the span. std::unique_ptr _header; /// Live information about stripes. - /// Seeded from @a _header and potentially agumented with direct probing. + /// Seeded from @a _header and potentially augmented with direct probing. std::list _stripes; }; /* --------------------------------------------------------------------------------------- */ diff --git a/src/traffic_cache_tool/CacheScan.cc b/src/traffic_cache_tool/CacheScan.cc index e52b256db7d..d46572b6cb9 100644 --- a/src/traffic_cache_tool/CacheScan.cc +++ b/src/traffic_cache_tool/CacheScan.cc @@ -208,7 +208,7 @@ CacheScan::unmarshal(HdrHeap *hh, int buf_length, int obj_type, HdrHeapObjImpl * while (obj_data < hh->m_free_start) { HdrHeapObjImpl *obj = (HdrHeapObjImpl *)obj_data; if (!obj_is_aligned(obj)) { - std::cout << "Invalid alignmgnt of object of type HdrHeapObjImpl" << std::endl; + std::cout << "Invalid alignment of object of type HdrHeapObjImpl" << std::endl; return zret; } @@ -259,7 +259,7 @@ CacheScan::unmarshal(char *buf, int len, RefCountObj *block_ref) int orig_len = len; if (alt->m_magic == CACHE_ALT_MAGIC_ALIVE) { - // Already unmarshaled, must be a ram cache + // Already unmarshalled, must be a ram cache // it ink_assert(alt->m_unmarshal_len > 0); ink_assert(alt->m_unmarshal_len <= len); diff --git a/src/traffic_cache_tool/CacheTool.cc b/src/traffic_cache_tool/CacheTool.cc index 094538228f9..4c60dd9730f 100644 --- a/src/traffic_cache_tool/CacheTool.cc +++ b/src/traffic_cache_tool/CacheTool.cc @@ -980,7 +980,7 @@ Cache::build_stripe_hash_table() ttable[i] = VOL_HASH_EMPTY; } - // generate random numbers proportaion to allocation + // generate random numbers proportional to allocation rtable_pair *rtable = (rtable_pair *)ats_malloc(sizeof(rtable_pair) * rtable_size); int rindex = 0; for (int i = 0; i < num_stripes; i++) { diff --git a/src/traffic_ctl/config.cc b/src/traffic_ctl/config.cc index 22198d23a29..61af1998bc5 100644 --- a/src/traffic_ctl/config.cc +++ b/src/traffic_ctl/config.cc @@ -63,7 +63,7 @@ rec_typeof(int rec_type) return "FLOAT"; case TS_REC_STRING: return "STRING"; - case TS_REC_UNDEFINED: /* fallthru */ + case TS_REC_UNDEFINED: /* fallthrough */ default: return "UNDEFINED"; } @@ -98,7 +98,7 @@ rec_accessof(int rec_access) return "no access"; case RECA_READ_ONLY: return "read only"; - case RECA_NULL: /* fallthru */ + case RECA_NULL: /* fallthrough */ default: return "default"; } @@ -115,7 +115,7 @@ rec_updateof(int rec_updatetype) return "static, restart traffic_server"; case RECU_RESTART_TM: return "static, restart traffic_manager"; - case RECU_NULL: /* fallthru */ + case RECU_NULL: /* fallthrough */ default: return "none"; } @@ -132,7 +132,7 @@ rec_checkof(int rec_checktype) return "integer with a specified range"; case RECC_IP: return "IP address"; - case RECC_NULL: /* fallthru */ + case RECC_NULL: /* fallthrough */ default: return "none"; } diff --git a/src/traffic_ctl/traffic_ctl.h b/src/traffic_ctl/traffic_ctl.h index f02fe27b51b..94073f6323e 100644 --- a/src/traffic_ctl/traffic_ctl.h +++ b/src/traffic_ctl/traffic_ctl.h @@ -175,7 +175,7 @@ struct CtrlEngine { int status_code = CTRL_EX_OK; // All traffic_ctl methods: - // umimplemented command + // unimplemented command void CtrlUnimplementedCommand(std::string_view command); // alarm methods diff --git a/src/traffic_layout/engine.cc b/src/traffic_layout/engine.cc index 3838d63acd8..fc994b9c74e 100644 --- a/src/traffic_layout/engine.cc +++ b/src/traffic_layout/engine.cc @@ -96,7 +96,7 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) std::string cur_working_dir = ""; char cwd[PATH_MAX]; if (!getcwd(cwd, sizeof(cwd))) { - ink_warning("unexcepted failure from getcwd() - %s", strerror(errno)); + ink_warning("unexpected failure from getcwd() - %s", strerror(errno)); } else { cur_working_dir = cwd; } @@ -378,7 +378,7 @@ LayoutEngine::remove_runroot() std::string cur_working_dir = ""; char cwd[PATH_MAX]; if (getcwd(cwd, sizeof(cwd)) == nullptr) { - ink_warning("unexcepted failure from getcwd() - %s", strerror(errno)); + ink_warning("unexpected failure from getcwd() - %s", strerror(errno)); } else { cur_working_dir = cwd; } diff --git a/src/traffic_layout/engine.h b/src/traffic_layout/engine.h index c79a186d648..91098e7d42e 100644 --- a/src/traffic_layout/engine.h +++ b/src/traffic_layout/engine.h @@ -43,7 +43,7 @@ struct PermissionEntry { // PermissionEntry contains the read/write/execute mode and the result of output using PermissionMapType = std::unordered_map; -// structure for informaiton of the runroot passing around +// structure for information of the runroot passing around struct LayoutEngine { // default output of all layouts void info(); @@ -62,7 +62,7 @@ struct LayoutEngine { ts::ArgParser parser; // parsed arguments ts::Arguments arguments; - // mordern argv + // modern argv std::vector _argv; int status_code = 0; diff --git a/src/traffic_layout/file_system.cc b/src/traffic_layout/file_system.cc index cbfa14bcd7b..f75fcebc0ed 100644 --- a/src/traffic_layout/file_system.cc +++ b/src/traffic_layout/file_system.cc @@ -247,7 +247,7 @@ ts_copy_function(const char *src_path, const struct stat *sb, int flag) // if the file already exist, overwrite it if (exists(dst_path)) { if (remove(dst_path.c_str())) { - ink_error("overwrite file falied during copy"); + ink_error("overwrite file failed during copy"); } } // hardlink bin executable @@ -271,7 +271,7 @@ ts_copy_function(const char *src_path, const struct stat *sb, int flag) std::ofstream dst(dst_path, std::ios::binary); dst << src.rdbuf(); if (chmod(dst_path.c_str(), sb->st_mode) == -1) { - ink_warning("failed chomd the destination path: %s", strerror(errno)); + ink_warning("failed chmod the destination path: %s", strerror(errno)); } } return 0; diff --git a/src/traffic_layout/file_system.h b/src/traffic_layout/file_system.h index 2d1210c4ff6..1f174c200cb 100644 --- a/src/traffic_layout/file_system.h +++ b/src/traffic_layout/file_system.h @@ -32,7 +32,7 @@ // full copy, hardlink, softlink enum CopyStyle { FULL, HARD, SOFT }; -// append slash & remove slash of path for convinient use +// append slash & remove slash of path for convenient use void append_slash(std::string &path); // for file system diff --git a/src/traffic_logstats/logstats.cc b/src/traffic_logstats/logstats.cc index 2cb72233b61..d396c4ed3c1 100644 --- a/src/traffic_logstats/logstats.cc +++ b/src/traffic_logstats/logstats.cc @@ -1107,7 +1107,7 @@ update_codes(OriginStats *stat, int code, int size) inline void update_methods(OriginStats *stat, int method, int size) { - // We're so loppsided on GETs, so makes most sense to test 'out of order'. + // We're so lopsided on GETs, so makes most sense to test 'out of order'. switch (method) { case METHOD_GET: update_counter(stat->methods.get, size); @@ -2167,7 +2167,7 @@ print_detail_stats(const OriginStats *stat, bool json, bool concise) if (!json) { std::cout << std::endl << std::endl; - // Protocol familes + // Protocol families format_detail_header("Protocols"); } diff --git a/src/traffic_manager/AddConfigFilesHere.cc b/src/traffic_manager/AddConfigFilesHere.cc index dc816b00903..0ace074e4ea 100644 --- a/src/traffic_manager/AddConfigFilesHere.cc +++ b/src/traffic_manager/AddConfigFilesHere.cc @@ -54,7 +54,7 @@ registerFile(const char *configName, const char *defaultName) // // initializeRegistry() // -// Code to initialze of registry of objects that represent +// Code to initialize of registry of objects that represent // Web Editable configuration files // // thread-safe: NO! - Should only be executed once from the main diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc index b4b7aa96777..f7d3835afdc 100644 --- a/src/traffic_manager/traffic_manager.cc +++ b/src/traffic_manager/traffic_manager.cc @@ -309,7 +309,7 @@ initSignalHandlers() // Block the delivery of any signals we are not catching // - // except for SIGALARM since we use it + // except for SIGALRM since we use it // to break out of deadlock on semaphore // we share with the proxy // diff --git a/src/traffic_server/FetchSM.cc b/src/traffic_server/FetchSM.cc index 436405fa34e..fd953c01fab 100644 --- a/src/traffic_server/FetchSM.cc +++ b/src/traffic_server/FetchSM.cc @@ -376,7 +376,7 @@ FetchSM::get_info_from_buffer(IOBufferReader *reader) info = (char *)ats_malloc(sizeof(char) * (read_avail + 1)); client_response = info; - // To maintain backwards compatability we don't allow chunking when it's not streaming. + // To maintain backwards compatibility we don't allow chunking when it's not streaming. if (!(fetch_flags & TS_FETCH_FLAGS_STREAM) || !check_chunked()) { /* Read the data out of the reader */ while (read_avail > 0) { diff --git a/src/traffic_server/FetchSM.h b/src/traffic_server/FetchSM.h index dc7f89bfc8c..0e57e4ce8b4 100644 --- a/src/traffic_server/FetchSM.h +++ b/src/traffic_server/FetchSM.h @@ -66,7 +66,7 @@ class FetchSM : public Continuation mutex = new_ProxyMutex(); // - // We had dropped response_buffer/respone_reader to avoid unnecessary + // We had dropped response_buffer/response_reader to avoid unnecessary // memory copying. But for the original TSFetchURL() API, PluginVC may // stop adding data to resp_buffer when the pending data in resp_buffer // reach its water_mark. @@ -130,7 +130,7 @@ class FetchSM : public Continuation { return req_reader->read_avail(); } - /// Check if the comma supproting MIME field @a name has @a value in it. + /// Check if the comma supporting MIME field @a name has @a value in it. bool check_for_field_value(const char *name, size_t name_len, char const *value, size_t value_len); bool has_body(); diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index fdbd6b9b08d..0facb4cc222 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -115,7 +115,7 @@ handle_record_read(const RecRecord *rec, void *edata) // if the data loaded from stats indicates that the host was down, // then update the state so that the host remains down until - // specificcaly marked up using traffic_ctl. + // specifically marked up using traffic_ctl. if (rec->data.rec_int == 0 && Reasons::validReason(reason.c_str())) { hs.setHostStatus(hostname.c_str(), HOST_STATUS_DOWN, 0, reason.c_str()); } diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 94b829fdda3..2111fa0c265 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -3510,7 +3510,7 @@ TSMimeHdrFieldValueDateInsert(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, time_t v char tmp[33]; int len = mime_format_date(tmp, value); - // idx ignored, overwrite all exisiting values + // idx ignored, overwrite all existing values // (void)TSMimeFieldValueInsert(bufp, field_obj, tmp, len, idx); (void)TSMimeFieldValueSet(bufp, field, -1, tmp, len); return TS_SUCCESS; @@ -7507,7 +7507,7 @@ TSMgmtConfigIntSet(const char *var_name, TSMgmtInt value) // tell manager to set the configuration; note that this is not // transactional (e.g. we return control to the plugin before the - // value is commited to disk by the manager) + // value is committed to disk by the manager) RecSignalManager(MGMT_SIGNAL_PLUGIN_SET_CONFIG, buffer); return TS_SUCCESS; diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 5e574991284..50e0c99364c 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -438,7 +438,7 @@ generate_response(const char *request) break; } } else { - /* Didin't recognize a testcase request. send the default response */ + /* Didn't recognize a testcase request. send the default response */ snprintf(response, RESPONSE_MAX_SIZE + 1, HTTP_RESPONSE_DEFAULT_FORMAT, test_case); } @@ -1202,7 +1202,7 @@ SDK_RPRINT(RegressionTest *t, const char *api_name, const char *testcase_name, i REGRESSION_TEST_INPROGRESS REGRESSION_TEST_FAILED REGRESSION_TEST_NOT_RUN - Note: pstatus is polled and can be used for asynchroneous tests. + Note: pstatus is polled and can be used for asynchronous tests. */ @@ -1595,7 +1595,7 @@ REGRESSION_TEST(SDK_API_TSPortDescriptor)(RegressionTest *test, int /* atype ATS // (OBJECT_SIZE/2, then OBJECT_SIZE-100 and finally OBJECT_SIZE) // - read object from the cache // - remove it from the cache -// - try to read it (should faild) +// - try to read it (should fail) #define OBJECT_SIZE 100000 // size of the object we'll write/read/remove in cache @@ -2621,7 +2621,7 @@ REGRESSION_TEST(SDK_API_TSContMutexGet)(RegressionTest *test, int /* atype ATS_U SDK_RPRINT(test, "TSContMutexGet", "TestCase1", TC_PASS, "ok"); test_passed = true; } else { - SDK_RPRINT(test, "TSContMutexGet", "TestCase1", TC_FAIL, "Continutation's mutex corrupted"); + SDK_RPRINT(test, "TSContMutexGet", "TestCase1", TC_FAIL, "Continuation's mutex corrupted"); } // Status of the whole test @@ -3005,7 +3005,7 @@ REGRESSION_TEST(SDK_API_TSIOBufferBlockNext)(RegressionTest *test, int /* atype TSIOBufferReader readerp = TSIOBufferReaderAlloc(bufp); TSIOBufferBlock blockp = TSIOBufferReaderStart(readerp); - // TODO: This is probaby not the best of regression tests right now ... + // TODO: This is probably not the best of regression tests right now ... // Note that this assumes block size is > sizeof(int) bytes. if (TSIOBufferBlockNext(blockp) == nullptr) { SDK_RPRINT(test, "TSIOBufferBlockNext", "TestCase1", TC_PASS, "ok"); @@ -6361,7 +6361,7 @@ log_test_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) *(data->pstatus) = REGRESSION_TEST_PASSED; SDK_RPRINT(data->test, "TSTextLogObject", "TestCase1", TC_PASS, "ok"); - // figure out the matainfo file for cleanup. + // figure out the metainfo file for cleanup. // code from MetaInfo::_build_name(const char *filename) int i = -1, l = 0; char c; @@ -8363,7 +8363,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpAltInfo)(RegressionTest *test, int /* atyp ////////////////////////////////////////////// // Important: we create servers listening on different port than the default one -// to make sure our synthetix servers are called +// to make sure our synthetic servers are called #define TEST_CASE_CONNECT_ID1 9 // TSHttpTxnIntercept #define TEST_CASE_CONNECT_ID2 10 // TSHttpTxnServerIntercept @@ -8527,7 +8527,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_TSHttpConnectServerIntercept)(RegressionTest * { *pstatus = REGRESSION_TEST_INPROGRESS; - TSDebug(UTDBG_TAG, "Starting test TSHttpConnectServerintercept"); + TSDebug(UTDBG_TAG, "Starting test TSHttpConnectServerIntercept"); TSCont cont_test = TSContCreate(cont_test_handler, TSMutexCreate()); ConnectTestData *data = (ConnectTestData *)TSmalloc(sizeof(ConnectTestData)); @@ -9073,7 +9073,7 @@ REGRESSION_TEST(SDK_API_UUID)(RegressionTest *test, int /* atype ATS_UNUSED */, // Test TSUuidCreate if (!(uuid = TSUuidCreate())) { - SDK_RPRINT(test, "TSUuidCreate", "TestCase1", TC_FAIL, "Failed to crete a UUID object"); + SDK_RPRINT(test, "TSUuidCreate", "TestCase1", TC_FAIL, "Failed to create a UUID object"); *pstatus = REGRESSION_TEST_FAILED; return; } else { diff --git a/src/traffic_server/SocksProxy.cc b/src/traffic_server/SocksProxy.cc index 77208ee4669..824e395a75f 100644 --- a/src/traffic_server/SocksProxy.cc +++ b/src/traffic_server/SocksProxy.cc @@ -22,7 +22,7 @@ */ /* - This implements SOCKS server. We intecept the http traffic and send it + This implements SOCKS server. We intercept the http traffic and send it through HTTP. Others are tunneled through directly to the socks server. @@ -403,7 +403,7 @@ SocksProxy::sendResp(bool granted) // In SOCKS 4, IP addr and Dest Port fields are ignored. // In SOCKS 5, IP addr and Dest Port are the ones we use to connect to the // real host. In our case, it does not make sense, since we may not - // connect at all. Set these feilds to zeros. Any socks client which uses + // connect at all. Set these fields to zeros. Any socks client which uses // these breaks caching. buf->reset(); @@ -446,7 +446,7 @@ SocksProxy::setupHttpRequest(unsigned char *p) break; case SOCKS_ATYPE_FQHN: - // This is stored as a zero terminicated string + // This is stored as a zero terminated string a->addr.buf = (unsigned char *)ats_malloc(p[4] + 1); memcpy(a->addr.buf, &p[5], p[4]); a->addr.buf[p[4]] = 0; diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 9c3c4bcf452..40e23026287 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -640,7 +640,7 @@ initialize_process_manager() RECP_NON_PERSISTENT); } -#define CMD_ERROR -2 // serious error, exit maintaince mode +#define CMD_ERROR -2 // serious error, exit maintenance mode #define CMD_FAILED -1 // error, but recoverable #define CMD_OK 0 // ok, or minor (user) error #define CMD_HELP 1 // ok, print help diff --git a/src/traffic_top/traffic_top.cc b/src/traffic_top/traffic_top.cc index e093636f065..ffb4b3dd3aa 100644 --- a/src/traffic_top/traffic_top.cc +++ b/src/traffic_top/traffic_top.cc @@ -247,7 +247,7 @@ help(const string &host, const string &version) attron(A_BOLD); mvprintw(7, 0, "Definitions:"); attroff(A_BOLD); - mvprintw(8, 0, "Fresh => Requests that were servered by fresh entries in cache"); + mvprintw(8, 0, "Fresh => Requests that were served by fresh entries in cache"); mvprintw(9, 0, "Revalidate => Requests that contacted the origin to verify if still valid"); mvprintw(10, 0, "Cold => Requests that were not in cache at all"); mvprintw(11, 0, "Changed => Requests that required entries in cache to be updated"); diff --git a/src/traffic_wccp/wccp_client.cc b/src/traffic_wccp/wccp_client.cc index 95a16b4c172..6f2c691b816 100644 --- a/src/traffic_wccp/wccp_client.cc +++ b/src/traffic_wccp/wccp_client.cc @@ -49,7 +49,7 @@ bool do_daemon = false; static const char USAGE_TEXT[] = "%s\n" "--address IP address to bind.\n" - "--router Booststrap IP address for routers.\n" + "--router Bootstrap IP address for routers.\n" "--service Path to service group definitions.\n" "--debug Print debugging information.\n" "--daemon Run as daemon.\n" diff --git a/src/tscore/ArgParser.cc b/src/tscore/ArgParser.cc index b411f4c1e71..b7cb4af0219 100644 --- a/src/tscore/ArgParser.cc +++ b/src/tscore/ArgParser.cc @@ -162,7 +162,7 @@ ArgParser::parse(const char **argv) parser_program_name = _argv[0]; Arguments ret; // the parsed arg object to return AP_StrVec args = _argv; - // call the recrusive parse method in Command + // call the recursive parse method in Command if (!_top_level_command.parse(ret, args)) { // deal with default command if (!default_command.empty()) { @@ -310,9 +310,9 @@ void ArgParser::Command::output_command(std::ostream &out, std::string const &prefix) const { if (_name != parser_program_name) { - // a nicely formated way to output command usage + // a nicely formatted way to output command usage std::string msg = prefix + _name; - // nicely formated output + // nicely formatted output if (!_description.empty()) { if (INDENT_ONE - static_cast(msg.size()) < 0) { // if the command msg is too long @@ -433,7 +433,7 @@ ArgParser::Command::append_option_data(Arguments &ret, AP_StrVec &args, int inde // output help message if ((args[i] == "--help" || args[i] == "-h") && _option_list.find("--help") != _option_list.end()) { ArgParser::Command *command = this; - // find the correct level to output help messsage + // find the correct level to output help message for (unsigned i = 1; i < args.size(); i++) { auto it = command->_subcommand_list.find(args[i]); if (it == command->_subcommand_list.end()) { @@ -627,7 +627,7 @@ std::string const & ArgumentData::at(unsigned index) const { if (index >= _values.size()) { - throw std::out_of_range("argument not fonud at index: " + std::to_string(index)); + throw std::out_of_range("argument not found at index: " + std::to_string(index)); } return _values.at(index); } diff --git a/src/tscore/BaseLogFile.cc b/src/tscore/BaseLogFile.cc index 921a1608c84..682c32a1a6a 100644 --- a/src/tscore/BaseLogFile.cc +++ b/src/tscore/BaseLogFile.cc @@ -24,7 +24,7 @@ #include "tscore/BaseLogFile.h" /* - * This consturctor creates a BaseLogFile based on a given name. + * This constructor creates a BaseLogFile based on a given name. * This is the most common way BaseLogFiles are created. */ BaseLogFile::BaseLogFile(const char *name) : m_name(ats_strdup(name)) @@ -33,7 +33,7 @@ BaseLogFile::BaseLogFile(const char *name) : m_name(ats_strdup(name)) } /* - * This consturctor creates a BaseLogFile based on a given name. + * This constructor creates a BaseLogFile based on a given name. * Similar to above constructor, but is overloaded with the object signature */ BaseLogFile::BaseLogFile(const char *name, uint64_t sig) : m_name(ats_strdup(name)), m_signature(sig), m_has_signature(true) @@ -155,7 +155,7 @@ BaseLogFile::roll(long interval_start, long interval_end) // produce overlapping filenames (the problem is that we have // no easy way of keeping track of the timestamp of the first // transaction - log_log_trace("in BaseLogFile::roll(..), didn't use metadata starttime, used earlist available starttime\n"); + log_log_trace("in BaseLogFile::roll(..), didn't use metadata starttime, used earliest available starttime\n"); if (interval_start == 0) { start = m_start_time; } else { @@ -204,7 +204,7 @@ BaseLogFile::roll(long interval_start, long interval_end) } /* - * The more convienent rolling function. Intended use is for less + * The more convenient rolling function. Intended use is for less * critical logs such as diags.log or traffic.out, since _exact_ * timestamps may be less important * @@ -331,7 +331,7 @@ BaseLogFile::open_file(int perm) } } - // set m_bytes_written to force the rolling based on filesize. + // set m_bytes_written to force the rolling based on file size. m_bytes_written = fseek(m_fp, 0, SEEK_CUR); log_log_trace("BaseLogFile %s is now open (fd=%d)\n", m_name.get(), fileno(m_fp)); @@ -535,7 +535,7 @@ BaseMetaInfo::_write_to_file() // TODO modify this runtime check so that it is not an assertion ink_release_assert(n <= BUF_SIZE); if (write(fd, _buffer, n) == -1) { - log_log_error("Could not write object_signaure\n"); + log_log_error("Could not write object_signature\n"); } log_log_trace("BaseMetaInfo::_write_to_file\n" "\tfilename = %s\n" diff --git a/src/tscore/BufferWriterFormat.cc b/src/tscore/BufferWriterFormat.cc index 05d821a07d3..54eafdf7a29 100644 --- a/src/tscore/BufferWriterFormat.cc +++ b/src/tscore/BufferWriterFormat.cc @@ -239,7 +239,7 @@ namespace bw_fmt size_t delta = min - extent; char *base = w.auxBuffer(); // should be first byte of @a lw e.g. lw.data() - avoid const_cast. char *limit = base + lw.capacity(); // first invalid byte. - char *dst; // used to track memory operation targest; + char *dst; // used to track memory operation target; char *last; // track limit of memory operation. size_t d2; switch (spec._align) { @@ -452,7 +452,7 @@ namespace bw_fmt width -= static_cast(n); std::string_view digits{buff + sizeof(buff) - n, n}; - if (spec._align == BWFSpec::Align::SIGN) { // custom for signed case because prefix and digits are seperated. + if (spec._align == BWFSpec::Align::SIGN) { // custom for signed case because prefix and digits are separated. if (neg) { w.write(neg); } @@ -482,9 +482,9 @@ namespace bw_fmt return w; } - /// Format for floating point values. Seperates floating point into a whole number and a + /// Format for floating point values. Separates floating point into a whole number and a /// fraction. The fraction is converted into an unsigned integer based on the specified - /// precision, spec._prec. ie. 3.1415 with precision two is seperated into two unsigned + /// precision, spec._prec. ie. 3.1415 with precision two is separated into two unsigned /// integers 3 and 14. The different pieces are assembled and placed into the BufferWriter. /// The default is two decimal places. ie. X.XX. The value is always written in base 10. /// diff --git a/src/tscore/CryptoHash.cc b/src/tscore/CryptoHash.cc index 926e5f2452b..b422c85493a 100644 --- a/src/tscore/CryptoHash.cc +++ b/src/tscore/CryptoHash.cc @@ -69,7 +69,7 @@ CryptoContext::CryptoContext() @brief Converts a hash to a null-terminated string Externalizes an hash as a null-terminated string into the first argument. - Does so without intenal procedure calls. + Does so without internal procedure calls. Side Effects: none. Reentrancy: n/a. Thread Safety: safe. diff --git a/src/tscore/Diags.cc b/src/tscore/Diags.cc index b34db02566c..6c85e578469 100644 --- a/src/tscore/Diags.cc +++ b/src/tscore/Diags.cc @@ -204,7 +204,7 @@ Diags::~Diags() // // This routine outputs to all of the output targets enabled for this // debugging level in config.outputs[diags_level]. Many higher level -// diagnosting printing routines are built upon print_va, including: +// diagnostics printing routines are built upon print_va, including: // // void print(...) // void log_va(...) diff --git a/src/tscore/HostLookup.cc b/src/tscore/HostLookup.cc index 0a47f38a44c..b2c7166baf6 100644 --- a/src/tscore/HostLookup.cc +++ b/src/tscore/HostLookup.cc @@ -49,7 +49,7 @@ namespace bool domaincmp(string_view hostname, string_view domain) { - // Check to see if were passed emtpy strings for either + // Check to see if were passed empty strings for either // argument. Empty strings do not match anything // if (domain.empty() || hostname.empty()) { @@ -150,7 +150,7 @@ hostcmp(string_view lhs, string_view rhs) // '_' is also included although it is not in the spec (RFC 883) // // Uppercase and lowercase "a-z" both map to same indexes -// since hostnames are not case sensative +// since hostnames are not case sensitive // // Illegal characters map to 255 // @@ -177,7 +177,7 @@ static const unsigned char asciiToTable[256] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}; -// Number of legal characters in the acssiToTable array +// Number of legal characters in the asciiToTable array static const int numLegalChars = 38; // struct CharIndexBlock @@ -193,7 +193,7 @@ struct CharIndexBlock { }; // class CharIndex - A constant time string matcher intended for -// short strings in a sparsely populated DNS paritition +// short strings in a sparsely populated DNS partition // // Creates a look up table for character in data string // @@ -458,7 +458,7 @@ CharIndex::iterator::operator!=(const self_type &that) const // class HostArray // -// Is a fixed size array for holding HostBrach* +// Is a fixed size array for holding HostBranch* // Allows only sequential access to data // @@ -656,7 +656,7 @@ HostLookup::TableNewLevel(HostBranch *from, string_view level_data) // HostBranch* HostLookup::InsertBranch(HostBranch* insert_to, const char* level_data) // // -// Abstrction to place a new node for level_data below node +// Abstraction to place a new node for level_data below node // insert to. Inserts into any of the data types used by // by class HostMatcher // diff --git a/src/tscore/IntrusivePtrTest.cc b/src/tscore/IntrusivePtrTest.cc index c7cb58c3f1c..85bd7e3f063 100644 --- a/src/tscore/IntrusivePtrTest.cc +++ b/src/tscore/IntrusivePtrTest.cc @@ -26,7 +26,7 @@ #include "tscore/TestBox.h" namespace -{ // Hide our local defintions +{ // Hide our local definitions // Test class for pointers and lists. class A : public IntrusivePtrCounter diff --git a/src/tscore/IpMap.cc b/src/tscore/IpMap.cc index 4a51962ca68..2fd01f8c937 100644 --- a/src/tscore/IpMap.cc +++ b/src/tscore/IpMap.cc @@ -842,7 +842,7 @@ namespace detail { return this->setMin(min + 1); } - /** Decremement the maximum value in place. + /** decrement the maximum value in place. @return This object. */ self_type & @@ -921,7 +921,7 @@ namespace detail /** Construct from the argument type. * * @param min Minimum value in the range. - * @param max Maximum value in the range (inclusvie). + * @param max Maximum value in the range (inclusive). * @param data Data to attach to the range. */ @@ -936,7 +936,7 @@ namespace detail /** Construct from the underlying @c Metric type @a min to @a max * * @param min Minimum value in the range. - * @param max Maximum value in the range (inclusvie). + * @param max Maximum value in the range (inclusive). * @param data Data to attach to the range. */ Ip6Node(Metric const &min, Metric const &max, void *data) : Node(data), Ip6Span(min, max) {} @@ -1028,7 +1028,7 @@ namespace detail inc(_min); return *this; } - /** Decremement the maximum value in place. + /** Decrement the maximum value in place. @return This object. */ self_type & @@ -1037,7 +1037,7 @@ namespace detail dec(_max); return *this; } - /** Increment the mininimum value in place. + /** Increment the minimum value in place. @return This object. */ self_type & diff --git a/src/tscore/JeAllocator.cc b/src/tscore/JeAllocator.cc index 214ffc0223e..fd1b418a8bf 100644 --- a/src/tscore/JeAllocator.cc +++ b/src/tscore/JeAllocator.cc @@ -92,7 +92,7 @@ JemallocNodumpAllocator::extend_and_setup_arena() } /** - * This will retain the orignal functionality if + * This will retain the original functionality if * !defined(JEMALLOC_NODUMP_ALLOCATOR_SUPPORTED) */ void * @@ -121,7 +121,7 @@ JemallocNodumpAllocator::allocate(InkFreeList *f) } /** - * This will retain the orignal functionality if + * This will retain the original functionality if * !defined(JEMALLOC_NODUMP_ALLOCATOR_SUPPORTED) */ void diff --git a/src/tscore/MatcherUtils.cc b/src/tscore/MatcherUtils.cc index 7e7648c0275..cb9822a2d75 100644 --- a/src/tscore/MatcherUtils.cc +++ b/src/tscore/MatcherUtils.cc @@ -44,7 +44,7 @@ // off the heap (via ats_malloc() ) Returns a pointer to the buffer // is successful and nullptr otherwise. // -// CALLEE is responsibled for deallocating the buffer via ats_free() +// CALLEE is responsible for deallocating the buffer via ats_free() // char * readIntoBuffer(const char *file_path, const char *module_name, int *read_size_ptr) @@ -495,7 +495,7 @@ parseConfigLine(char *line, matcher_line *p_line, const matcher_tags *tags) case PARSE_VAL: if (inQuote == true) { if (*s == '\\') { - // The next character is esacped + // The next character is escaped // // To remove the escaped character // we need to copy diff --git a/src/tscore/ParseRules.cc b/src/tscore/ParseRules.cc index a3b0f78a568..8958d39443d 100644 --- a/src/tscore/ParseRules.cc +++ b/src/tscore/ParseRules.cc @@ -75,7 +75,7 @@ ink_atoi64(const char *str) while (*str && ParseRules::is_digit(*str)) { num = (num * 10) - (*str++ - '0'); } -#if USE_SI_MULTILIERS +#if USE_SI_MULTIPLIERS if (*str) { if (*str == 'K') { num = num * (1LL << 10); @@ -113,7 +113,7 @@ ink_atoui64(const char *str) while (*str && ParseRules::is_digit(*str)) { num = (num * 10) + (*str++ - '0'); } -#if USE_SI_MULTILIERS +#if USE_SI_MULTIPLIERS if (*str) { if (*str == 'K') { num = num * (1LL << 10); @@ -165,7 +165,7 @@ ink_atoi64(const char *str, int len) num = (num * 10) - (*str++ - '0'); len--; } -#if USE_SI_MULTILIERS +#if USE_SI_MULTIPLIERS if (len > 0 && *str) { if (*str == 'K') { num = num * (1 << 10); diff --git a/src/tscore/TextBuffer.cc b/src/tscore/TextBuffer.cc index ec62ca40c46..ef2b0b1750f 100644 --- a/src/tscore/TextBuffer.cc +++ b/src/tscore/TextBuffer.cc @@ -28,7 +28,7 @@ /**************************************************************************** * - * TextBuffer.cc - A self-expanding buffer, primarly meant for strings + * TextBuffer.cc - A self-expanding buffer, primarily meant for strings * * * @@ -40,7 +40,7 @@ TextBuffer::TextBuffer(int size) nextAdd = nullptr; currentSize = spaceLeft = 0; if (size > 0) { - // Insitute a minimum size + // Institute a minimum size if (size < 1024) { size = 1024; } @@ -162,8 +162,8 @@ TextBuffer::rawReadFromFile(int fd) { int readSize; - // Check to see if we have got a resonable amount of space left in our - // buffer, if not try to get somemore + // Check to see if we have got a reasonable amount of space left in our + // buffer, if not try to get some more if (spaceLeft < 4096) { if (enlargeBuffer(4096) == -1) { return -1; @@ -198,15 +198,15 @@ TextBuffer::slurp(int fd) // int TextBuffer::readFromFD(int fd) // // Issues a single read command on the file -// descritor passed in. Attempts to read a minimum of +// descriptor passed in. Attempts to read a minimum of // 512 bytes from file descriptor passed. int TextBuffer::readFromFD(int fd) { int readSize; - // Check to see if we have got a resonable amount of space left in our - // buffer, if not try to get somemore + // Check to see if we have got a reasonable amount of space left in our + // buffer, if not try to get some more if (spaceLeft < 512) { if (enlargeBuffer(512) == -1) { return -1; diff --git a/src/tscore/Tokenizer.cc b/src/tscore/Tokenizer.cc index b538facc9fd..b7c7f790f56 100644 --- a/src/tscore/Tokenizer.cc +++ b/src/tscore/Tokenizer.cc @@ -192,7 +192,7 @@ Tokenizer::Initialize(char *str, unsigned opt) quoteFound = false; - // Check to see if we stoped due to a maxToken limit + // Check to see if we stopped due to a maxToken limit if (max_limit_hit == true) { if (options & ALLOW_EMPTY_TOKS) { // Go till either we hit a delimiter or we've diff --git a/src/tscore/ink_base64.cc b/src/tscore/ink_base64.cc index 47f2e6a8745..92dfa132660 100644 --- a/src/tscore/ink_base64.cc +++ b/src/tscore/ink_base64.cc @@ -101,7 +101,7 @@ ats_base64_encode(const char *inBuffer, size_t inBufferSize, char *outBuffer, si } /*------------------------------------------------------------------------- - This is a reentrant, and malloc free implemetnation of ats_base64_decode. + This is a reentrant, and malloc free implementation of ats_base64_decode. -------------------------------------------------------------------------*/ #ifdef DECODE #undef DECODE diff --git a/src/tscore/ink_file.cc b/src/tscore/ink_file.cc index d737ad8b2d1..c0003c867eb 100644 --- a/src/tscore/ink_file.cc +++ b/src/tscore/ink_file.cc @@ -174,7 +174,7 @@ ink_filepath_merge(char *path, int pathsz, const char *rootpath, const char *add } else { // If INK_FILEPATH_NOTABSOLUTE is specified, the caller // requires a relative result. If the rootpath is - // ommitted, we do not retrieve the working path, + // omitted, we do not retrieve the working path, // if rootpath was supplied as absolute then fail. // if (flags & INK_FILEPATH_NOTABSOLUTE) { @@ -490,7 +490,7 @@ ink_file_get_geometry(int fd ATS_UNUSED, ink_device_geometry &geometry) #if defined(BLKALIGNOFF) // BLKALIGNOFF gets the number of bytes needed to align the I/Os to the block device with // and underlying block devices. This might be non-zero when you are using a logical volume - // backed by JBOD or RAID device(s). BLKALIGNOFF was addeed in 2.6.32, so it's not present in + // backed by JBOD or RAID device(s). BLKALIGNOFF was added in 2.6.32, so it's not present in // RHEL 5. if (ioctl(fd, BLKALIGNOFF, &arg.u32) == 0) { geometry.alignsz = arg.u32; diff --git a/src/tscore/ink_memory.cc b/src/tscore/ink_memory.cc index 8ffb964935d..aca92b80142 100644 --- a/src/tscore/ink_memory.cc +++ b/src/tscore/ink_memory.cc @@ -33,7 +33,7 @@ #include #if defined(linux) -// XXX: SHouldn't that be part of CPPFLAGS? +// XXX: Shouldn't that be part of CPPFLAGS? #ifndef _XOPEN_SOURCE #define _XOPEN_SOURCE 600 #endif @@ -168,7 +168,7 @@ ats_msync(caddr_t addr, size_t len, caddr_t end, int flags) // align start back to page boundary caddr_t a = (caddr_t)(((uintptr_t)addr) & ~(pagesize - 1)); - // align length to page boundry covering region + // align length to page boundary covering region size_t l = (len + (addr - a) + (pagesize - 1)) & ~(pagesize - 1); if ((a + l) > end) { l = end - a; // strict limit @@ -271,7 +271,7 @@ _xstrdup(const char *str, int length, const char * /* path ATS_UNUSED */) *newstr = '\0'; } else { strncpy(newstr, str, length); // we cannot do length + 1 because the string isn't - newstr[length] = '\0'; // guaranteeed to be null terminated! + newstr[length] = '\0'; // guaranteed to be null terminated! } return newstr; } diff --git a/src/tscore/ink_queue_utils.cc b/src/tscore/ink_queue_utils.cc index 4246a3b58df..36fcdae2c6f 100644 --- a/src/tscore/ink_queue_utils.cc +++ b/src/tscore/ink_queue_utils.cc @@ -34,10 +34,10 @@ * problem resulted in the discovery that gcc was spitting out the * "ldd" (load double) instruction for loading of the 64 bit field "data" * while CC was spitting out two "ld" (load) instructions. The old code - * was calling item.data = head.data on sparcs and not putting any restriction + * was calling item.data = head.data on Sparcs and not putting any restriction * on the order of loading of the fields. * - * This is a problem on the sparcs because the "pointer" field was being loaded + * This is a problem on the Sparcs because the "pointer" field was being loaded * before the "version" field. This can result in a very subtle race condition * which subverts the addition of the "version" field. * @@ -50,7 +50,7 @@ * next.version = item.version ++; * cas64(head, item, next) - * Note, that the cas64 call will be succesful and the next.ptr will probably + * Note, that the cas64 call will be successful and the next.ptr will probably * be a pointer into the vtable entry. The next alloc will result in a write into * the vtable area. * diff --git a/src/tscore/ink_res_init.cc b/src/tscore/ink_res_init.cc index 962fc6b6070..be4dccce24d 100644 --- a/src/tscore/ink_res_init.cc +++ b/src/tscore/ink_res_init.cc @@ -129,7 +129,7 @@ ink_res_setservers(ink_res_state statp, IpEndpoint const *set, int cnt) /* The goal here seems to be to compress the source list (@a set) by squeezing out invalid addresses. We handle the special case where - the destination and sourcea are the same. + the destination and source are the same. */ int nserv = 0; for (IpEndpoint const *limit = set + cnt; nserv < INK_MAXNS && set < limit; ++set) { @@ -275,7 +275,7 @@ ink_res_randomid() * there will have precedence. Otherwise, the server address is set to * INADDR_ANY and the default domain name comes from the gethostname(). * - * An interrim version of this code (BIND 4.9, pre-4.4BSD) used 127.0.0.1 + * An interim version of this code (BIND 4.9, pre-4.4BSD) used 127.0.0.1 * rather than INADDR_ANY ("0.0.0.0") as the default name server address * since it was noted that INADDR_ANY actually meant ``the first interface * you "ifconfig"'d at boot time'' and if this was a SLIP or PPP interface, @@ -382,7 +382,7 @@ ink_res_init(ink_res_state statp, ///< State object to update. } /* --------------------------------------------- - Default domain name and doamin Search list: + Default domain name and domain Search list: if we are supplied a default domain name, and/or search list we will use it. Otherwise, @@ -427,7 +427,7 @@ ink_res_init(ink_res_state statp, ///< State object to update. } /* ------------------------------------------- - we must be provided with atleast a named! + we must be provided with at least a named! ------------------------------------------- */ if (pHostList) { if (pHostListSize > INK_MAXNS) { diff --git a/src/tscore/ink_res_mkquery.cc b/src/tscore/ink_res_mkquery.cc index 4ae16427763..fdd0aa1375e 100644 --- a/src/tscore/ink_res_mkquery.cc +++ b/src/tscore/ink_res_mkquery.cc @@ -217,7 +217,7 @@ labellen(const u_char *lp) } return ((bitlen + 7) / 8 + 1); } - return (-1); /*%< unknwon ELT */ + return (-1); /*%< unknown ELT */ } return (l); } @@ -278,7 +278,7 @@ decode_bitstring(const unsigned char **cpp, char *dn, const char *eom) /*% * Thinking in noninternationalized USASCII (per the DNS spec), - * is this characted special ("in need of quoting") ? + * is this character special ("in need of quoting") ? * * return: *\li boolean. diff --git a/src/tscore/ink_sock.cc b/src/tscore/ink_sock.cc index c9406205f9f..ccb5d9995aa 100644 --- a/src/tscore/ink_sock.cc +++ b/src/tscore/ink_sock.cc @@ -50,7 +50,7 @@ check_valid_sockaddr(sockaddr *sa, char *file, int line) if (port > 20000) { cerr << "[byteordering] In " << file << ", line " << line << " the IP port "; cerr << "was found to be " << port << "(in host byte order).\n"; - cerr << "[byteordering] This seems inplausible, so check for byte order problems\n"; + cerr << "[byteordering] This seems implausible, so check for byte order problems\n"; } } #else diff --git a/src/tscore/ink_string++.cc b/src/tscore/ink_string++.cc index bc8db8ac0d9..eac0718bac6 100644 --- a/src/tscore/ink_string++.cc +++ b/src/tscore/ink_string++.cc @@ -177,7 +177,7 @@ StrListOverflow::create_heap(int user_size) // matter since we are talking about strings but since this is a // last minute emergency bug fix, I'm not take any changes. If // allocations are not of aligned values then subsequents allocations - // aren't aligned, again mirroring the previous implemnetation + // aren't aligned, again mirroring the previous implementation int total_size = overflow_head_hdr_size + user_size; StrListOverflow *o = (StrListOverflow *)ats_malloc(total_size); diff --git a/src/tscore/ink_time.cc b/src/tscore/ink_time.cc index c8145cd01e8..ba36a1238bc 100644 --- a/src/tscore/ink_time.cc +++ b/src/tscore/ink_time.cc @@ -107,7 +107,7 @@ struct dtconv { /* * The man page for cftime lies. It claims that it is thread safe. * Instead, it silently trashes the heap (by freeing things more than - * once) when used in a mulithreaded program. Gack! + * once) when used in a multithreaded program. Gack! */ int cftime_replacement(char *s, int maxsize, const char *format, const time_t *clock) diff --git a/src/tscore/load_http_hdr.cc b/src/tscore/load_http_hdr.cc index b565438acf8..c9d840f6cb1 100644 --- a/src/tscore/load_http_hdr.cc +++ b/src/tscore/load_http_hdr.cc @@ -128,7 +128,7 @@ load_buffer(int fd, hdr_type h_type) fprintf(stderr, "Failed to read data file : %s\n", strerror(errno)); exit(1); } else if (done == 0) { - fprintf(stderr, "EOF encounted\n", strerror(errno)); + fprintf(stderr, "EOF encountered\n", strerror(errno)); exit(1); } diff --git a/src/tscore/lockfile.cc b/src/tscore/lockfile.cc index b3782682af8..a0c87cecaf5 100644 --- a/src/tscore/lockfile.cc +++ b/src/tscore/lockfile.cc @@ -24,7 +24,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_lockfile.h" -#define LOCKFILE_BUF_LEN 16 // 16 bytes should be enought for a pid +#define LOCKFILE_BUF_LEN 16 // 16 bytes should be enough for a pid int Lockfile::Open(pid_t *holding_pid) @@ -99,7 +99,7 @@ Lockfile::Open(pid_t *holding_pid) FAIL(0); } // If we did get the lock, then set the close on exec flag so that - // we don't accidently pass the file descriptor to a child process + // we don't accidentally pass the file descriptor to a child process // when we do a fork/exec. do { err = fcntl(fd, F_GETFD, 0); diff --git a/src/tscore/unit_tests/test_AcidPtr.cc b/src/tscore/unit_tests/test_AcidPtr.cc index c8994cfb692..4f7e98a4e56 100644 --- a/src/tscore/unit_tests/test_AcidPtr.cc +++ b/src/tscore/unit_tests/test_AcidPtr.cc @@ -85,7 +85,7 @@ TEST_CASE("AcidPtr Isolation") { AcidCommitPtr w = p; *w += 1; - CHECK(*p.getPtr() == 40); // new value not commited until end of scope + CHECK(*p.getPtr() == 40); // new value not committed until end of scope } CHECK(*p.getPtr() == 41); { diff --git a/src/tscore/unit_tests/test_IntrusivePtr.cc b/src/tscore/unit_tests/test_IntrusivePtr.cc index ffacbc6a615..d4a130aaac7 100644 --- a/src/tscore/unit_tests/test_IntrusivePtr.cc +++ b/src/tscore/unit_tests/test_IntrusivePtr.cc @@ -96,7 +96,7 @@ TEST_CASE("IntrusivePtr", "[libts][IntrusivePtr]") // List test. TEST_CASE("IntrusivePtr List", "[libts][IntrusivePtr]") { - // The clang analyzer claims this type of list manipularion leads to use after free because of + // The clang analyzer claims this type of list manipulation leads to use after free because of // premature class destruction but these tests verify that is a false positive. using LP = ts::IntrusivePtr; diff --git a/src/tscore/unit_tests/test_arena.cc b/src/tscore/unit_tests/test_arena.cc index dbf97ad618b..01f4217f0ed 100644 --- a/src/tscore/unit_tests/test_arena.cc +++ b/src/tscore/unit_tests/test_arena.cc @@ -28,7 +28,7 @@ Description: A short test program intended to be used with Purify to detect problems - with the arnea code + with the arena code ****************************************************************************/ diff --git a/src/tscpp/api/AsyncHttpFetch.cc b/src/tscpp/api/AsyncHttpFetch.cc index 64239946912..add00c70b5a 100644 --- a/src/tscpp/api/AsyncHttpFetch.cc +++ b/src/tscpp/api/AsyncHttpFetch.cc @@ -222,7 +222,7 @@ AsyncHttpFetch::run() request_str += "\r\n"; request_str += state_->request_body_; - LOG_DEBUG("Issing (non-streaming) TSFetchUrl with request\n[%s]", request_str.c_str()); + LOG_DEBUG("Issuing (non-streaming) TSFetchUrl with request\n[%s]", request_str.c_str()); TSFetchUrl(request_str.c_str(), request_str.size(), reinterpret_cast(&addr), fetchCont, AFTER_BODY, event_ids); } else { diff --git a/src/tscpp/api/GzipDeflateTransformation.cc b/src/tscpp/api/GzipDeflateTransformation.cc index 9832ec1f8ca..d371e3076d1 100644 --- a/src/tscpp/api/GzipDeflateTransformation.cc +++ b/src/tscpp/api/GzipDeflateTransformation.cc @@ -134,7 +134,7 @@ GzipDeflateTransformation::handleInputComplete() // We will flush out anything that's remaining in the gzip buffer int status = Z_OK; int iteration = 0; - const int buffer_size = 1024; // 1024 bytes is usually more than enough for the epilouge + const int buffer_size = 1024; // 1024 bytes is usually more than enough for the epilogue unsigned char buffer[buffer_size]; /* Deflate remaining data */ @@ -154,7 +154,7 @@ GzipDeflateTransformation::handleInputComplete() bytes_to_write, status); produce(std::string_view(reinterpret_cast(buffer), static_cast(bytes_to_write))); } else if (status != Z_STREAM_END) { - LOG_ERROR("Iteration %d: Gzip deflinate finalize produced an error '%d'", iteration, status); + LOG_ERROR("Iteration %d: Gzip deflate finalize produced an error '%d'", iteration, status); } } while (status == Z_OK); diff --git a/src/tscpp/api/Transaction.cc b/src/tscpp/api/Transaction.cc index 97e00dd6878..5beb03d7c31 100644 --- a/src/tscpp/api/Transaction.cc +++ b/src/tscpp/api/Transaction.cc @@ -413,7 +413,7 @@ namespace * @param constructor takes a function pointer of type GetterFunction * @param txn a TSHttpTxn * @param hdr_buf the address where the hdr buf will be stored - * @param hdr_loc the address where the mem loc will be storeds + * @param hdr_loc the address where the mem loc will be stored * @param name name of the entity - used for logging */ class initializeHandles diff --git a/src/tscpp/api/TransformationPlugin.cc b/src/tscpp/api/TransformationPlugin.cc index 524b07bae27..e578657fc81 100644 --- a/src/tscpp/api/TransformationPlugin.cc +++ b/src/tscpp/api/TransformationPlugin.cc @@ -69,7 +69,7 @@ struct TransformationPluginState : noncopyable, public detail::ResumeAfterPauseC // We can only send a single WRITE_COMPLETE even though // we may receive an immediate event after we've sent a // write complete, so we'll keep track of whether or not we've - // sent the input end our write complte. + // sent the input end our write complete. bool input_complete_dispatched_; std::string request_xform_output_; // in case of request xform, data produced is buffered here @@ -263,7 +263,7 @@ handleTransformationPluginEvents(TSCont contp, TSEvent event, void *edata) return 0; } - // All other events includign WRITE_READY will just attempt to transform more data. + // All other events including WRITE_READY will just attempt to transform more data. return handleTransformationPluginRead(state->vconn_, state); } @@ -426,7 +426,7 @@ TransformationPlugin::setOutputComplete() state_->txn_); // We're done without ever outputting anything, to correctly - // clean up we'll initiate a write then immeidately set it to 0 bytes done. + // clean up we'll initiate a write then immediately set it to 0 bytes done. state_->output_vio_ = TSVConnWrite(TSTransformOutputVConnGet(state_->vconn_), state_->vconn_, state_->output_buffer_reader_, 0); if (state_->output_vio_) { diff --git a/src/tscpp/api/utils_internal.cc b/src/tscpp/api/utils_internal.cc index f70f2c9855a..923c48d961a 100644 --- a/src/tscpp/api/utils_internal.cc +++ b/src/tscpp/api/utils_internal.cc @@ -79,7 +79,7 @@ handleTransactionEvents(TSCont cont, TSEvent event, void *edata) const std::list &plugins = utils::internal::getTransactionPlugins(transaction); for (auto plugin : plugins) { std::shared_ptr trans_mutex = utils::internal::getTransactionPluginMutex(*plugin); - LOG_DEBUG("Locking TransacitonPlugin mutex to delete transaction plugin at %p", plugin); + LOG_DEBUG("Locking TransactionPlugin mutex to delete transaction plugin at %p", plugin); trans_mutex->lock(); LOG_DEBUG("Locked Mutex...Deleting transaction plugin at %p", plugin); delete plugin; diff --git a/src/tscpp/util/TextView.cc b/src/tscpp/util/TextView.cc index 4532b027fa2..78d668db90d 100644 --- a/src/tscpp/util/TextView.cc +++ b/src/tscpp/util/TextView.cc @@ -65,7 +65,7 @@ strcasecmp(const std::string_view &lhs, const std::string_view &rhs) } const int8_t ts::svtoi_convert[256] = { - /* [can't do this nicely because clang format won't allow exdented comments] + /* [can't do this nicely because clang format won't allow extended comments] 0 1 2 3 4 5 6 7 8 9 A B C D E F */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 00 @@ -153,7 +153,7 @@ ts::svtoi(TextView src, TextView *out, int base) return zret; } -// Do the template instantions. +// Do the template instantiations. template std::ostream &ts::TextView::stream_write(std::ostream &, const ts::TextView &) const; namespace std diff --git a/src/wccp/WccpEndPoint.cc b/src/wccp/WccpEndPoint.cc index bbd45f36295..16e6377b054 100644 --- a/src/wccp/WccpEndPoint.cc +++ b/src/wccp/WccpEndPoint.cc @@ -721,7 +721,7 @@ CacheImpl::handleISeeYou(IpHeader const & /* ip_hdr ATS_UNUSED */, ts::Buffer co logf(LVL_DEBUG, "Received WCCP2_I_SEE_YOU for group %d.", group.m_svc.getSvcId()); - // Prefered address for router. + // Preferred address for router. uint32_t router_addr = msg.m_router_id.idElt().getAddr(); // Where we sent our packet. uint32_t to_addr = msg.m_router_id.getToAddr(); diff --git a/src/wccp/WccpLocal.h b/src/wccp/WccpLocal.h index dff8f6d9856..f73ce72aae9 100644 --- a/src/wccp/WccpLocal.h +++ b/src/wccp/WccpLocal.h @@ -180,7 +180,7 @@ struct RouterId { ); uint32_t m_addr; ///< Identifying router IP address. - uint32_t m_recv_id; ///< Recieve ID (sequence #). + uint32_t m_recv_id; ///< Receive ID (sequence #). }; /** Sect 5.7.1: Router Identity Element. @@ -486,7 +486,7 @@ class MaskValueSetElt ValueElt &operator[](int idx ///< Index of target element. ); //@} - /// Calcuate the size of an element with @a n values. + /// Calculate the size of an element with @a n values. static size_t calcSize(uint32_t n ///< Number of values. ); /// Get the size (length) of this element. @@ -1030,7 +1030,7 @@ class SecurityComp : public CompWithHeader static void setDefaultOption(Option opt ///< Type of security. ); - /// Set messsage local security key. + /// Set message local security key. self &setKey(const char *key ///< Shared key. ); @@ -1067,7 +1067,7 @@ class ServiceComp : public CompWithHeader struct raw_t : public super::raw_t, public ServiceGroup { }; - ServiceComp(); ///< Default constructor, no member intialization. + ServiceComp(); ///< Default constructor, no member initialization. /// @name Accessors //@{ @@ -1636,7 +1636,7 @@ class AltAssignComp : public CompWithHeader RouterAssignListElt m_routers; ///< Routers. }; - /// Force virtual desctructor. + /// Force virtual destructor. virtual ~AltAssignComp() {} /// @name Accessors //@{ @@ -1691,7 +1691,7 @@ class AltHashAssignComp : public AltAssignComp uint32_t getCacheCount() const; //@} - /// Force virtual desctructor. + /// Force virtual destructor. virtual ~AltHashAssignComp() {} /// Fill out the component from an @c Assignment. virtual self &fill(MsgBuffer &buffer, ///< Target storage. @@ -1724,7 +1724,7 @@ class AltMaskAssignComp : public AltAssignComp typedef AltMaskAssignComp self; ///< Self reference type. typedef AltAssignComp super; ///< Parent type. - /// Force virtual desctructor. + /// Force virtual destructor. virtual ~AltMaskAssignComp() {} /// Fill out the component from an @c Assignment. virtual self &fill(MsgBuffer &buffer, ///< Target storage. @@ -1775,7 +1775,7 @@ class CmdComp : public CompWithHeader //@} /// Write basic serialization data. - /// Elements must be filled in seperately and after invoking this method. + /// Elements must be filled in separately and after invoking this method. self &fill(MsgBuffer &buffer, ///< Component storage. cmd_t cmd, ///< Command type. uint32_t data ///< Command data. @@ -1861,7 +1861,7 @@ class QueryComp : public CompWithHeader uint32_t routerAddr, ///< Router identifying address. uint32_t toAddr, ///< Destination address. uint32_t cacheAddr, ///< Cache identifying address. - uint32_t recvId ///< Recieve ID. + uint32_t recvId ///< Receive ID. ); /// Validate an existing structure. @@ -2021,7 +2021,7 @@ class BaseMsg size_t getCount() const; /// Validate security option. - /// @note This presumes a sublcass has already successfully parsed. + /// @note This presumes a subclass has already successfully parsed. bool validateSecurity() const; // Common starting components for all messages. @@ -2073,7 +2073,7 @@ class ISeeYouMsg : public BaseMsg typedef ISeeYouMsg self; ///< Self reference type. /// Fill out message structure. - /// Router ID and view data must be filled in seperately. + /// Router ID and view data must be filled in separately. void fill(detail::router::GroupData const &group, ///< Service groupc context. SecurityOption sec_opt, ///< Security option. detail::Assignment &assign, ///< Cache assignment data. @@ -2213,7 +2213,7 @@ class Impl : public ts::IntrusivePtrCounter /// @return 0 for success, -errno on error. virtual int housekeeping() = 0; - /// Recieve and process a message. + /// Receive and process a message. /// @return 0 for success, -ERRNO on system error. virtual ts::Rv handleMessage(); @@ -2382,7 +2382,7 @@ namespace detail RouterBag::iterator findRouter(uint32_t addr ///< IP address of cache. ); - /// Set an intial router for a service group. + /// Set an initial router for a service group. self &seedRouter(uint32_t addr ///< IP address for router. ); /// Remove a seed router. @@ -2467,7 +2467,7 @@ class CacheImpl : public Impl ServiceGroup::Result *result = 0 ///< [out] Result for service creation. ); - /** Set an intial router for a service group. + /** Set an initial router for a service group. This is needed to bootstrap the protocol. If the router is already seeded, this call is silently ignored. */ diff --git a/src/wccp/WccpMsg.cc b/src/wccp/WccpMsg.cc index 0b64990481c..f054b862fe7 100644 --- a/src/wccp/WccpMsg.cc +++ b/src/wccp/WccpMsg.cc @@ -1796,7 +1796,7 @@ ISeeYouMsg::parse(ts::Buffer const &buffer) // Optional components. // Test for alternates here - // At most one of the asssignments but never both. + // At most one of the assignments but never both. // Can be omitted. m_assignment.parse(m_buffer); m_map.parse(m_buffer); diff --git a/src/wccp/WccpUtil.h b/src/wccp/WccpUtil.h index 3d986acda69..f8225257901 100644 --- a/src/wccp/WccpUtil.h +++ b/src/wccp/WccpUtil.h @@ -81,7 +81,7 @@ ts::Errata &logf(ts::Errata &err, ///< Target errata. ts::Errata log(ts::Errata::Code code, ///< Severity level. char const *text ///< Message text. ); -/// Return an Errata populated with a printf styleformatted string. +/// Return an Errata populated with a printf style formatted string. /// Use message ID 0. /// @return @a err. ts::Errata logf(ts::Errata::Code code, ///< Severity level. From 774f55f107bee5316b152afc12b70c59dcb333f3 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Thu, 25 Apr 2019 16:27:28 +0800 Subject: [PATCH 501/526] fixes spelling in include --- include/ts/TsException.h | 2 +- include/ts/experimental.h | 4 ++-- include/ts/remap.h | 10 ++++----- include/ts/ts.h | 18 +++++++-------- include/tscore/AcidPtr.h | 4 ++-- include/tscore/Allocator.h | 2 +- include/tscore/ArgParser.h | 4 ++-- include/tscore/AtomicBit.h | 2 +- include/tscore/BufferWriter.h | 22 +++++++++---------- include/tscore/BufferWriterForward.h | 2 +- include/tscore/CryptoHash.h | 2 +- include/tscore/Extendible.h | 6 ++--- include/tscore/HostLookup.h | 2 +- include/tscore/I_Layout.h | 2 +- include/tscore/IntrusiveHashMap.h | 2 +- include/tscore/IpMap.h | 4 ++-- include/tscore/MemArena.h | 2 +- include/tscore/ParseRules.h | 4 ++-- include/tscore/Ptr.h | 2 +- include/tscore/Scalar.h | 4 ++-- include/tscore/SimpleTokenizer.h | 2 +- include/tscore/TextBuffer.h | 2 +- include/tscore/Tokenizer.h | 12 +++++----- include/tscore/ink_base64.h | 2 +- include/tscore/ink_cap.h | 2 +- include/tscore/ink_code.h | 2 +- include/tscore/ink_endian.h | 2 +- include/tscore/ink_file.h | 2 +- include/tscore/ink_hrtime.h | 4 ++-- include/tscore/ink_inet.h | 2 +- include/tscore/ink_lockfile.h | 2 +- include/tscore/ink_resolver.h | 4 ++-- include/tscore/signals.h | 2 +- include/tscpp/api/Continuation.h | 4 ++-- include/tscpp/api/GlobalPlugin.h | 6 ++--- include/tscpp/api/GzipDeflateTransformation.h | 6 ++--- include/tscpp/api/GzipInflateTransformation.h | 8 +++---- include/tscpp/api/Headers.h | 14 ++++++------ include/tscpp/api/Logger.h | 8 +++---- include/tscpp/api/Plugin.h | 8 +++---- include/tscpp/api/RemapPlugin.h | 2 +- include/tscpp/api/Stat.h | 2 +- include/tscpp/api/Transaction.h | 6 ++--- include/tscpp/api/TransactionPlugin.h | 8 +++---- include/tscpp/api/TransformationPlugin.h | 4 ++-- include/tscpp/api/noncopyable.h | 2 +- include/tscpp/util/IntrusiveDList.h | 12 +++++----- include/tscpp/util/MemSpan.h | 4 ++-- include/tscpp/util/TextView.h | 12 +++++----- include/wccp/Wccp.h | 2 +- 50 files changed, 124 insertions(+), 124 deletions(-) diff --git a/include/ts/TsException.h b/include/ts/TsException.h index ca833d0c6ff..ff808947430 100644 --- a/include/ts/TsException.h +++ b/include/ts/TsException.h @@ -35,7 +35,7 @@ namespace ts /** Base class for ATS exception. Clients should subclass as appropriate. This is intended to carry pre-allocated text along so that it can be thrown without any - addditional memory allocation. + additional memory allocation. */ class Exception { diff --git a/include/ts/experimental.h b/include/ts/experimental.h index 8f9a818af01..fa8e7ead89c 100644 --- a/include/ts/experimental.h +++ b/include/ts/experimental.h @@ -192,7 +192,7 @@ tsapi TSReturnCode TSHttpTxnInfoIntGet(TSHttpTxn txnp, TSHttpTxnInfoKey key, TSM /**************************************************************************** * TSHttpTxnCacheLookupCountGet - * Return: TS_SUCESS/TS_ERROR + * Return: TS_SUCCESS/TS_ERROR ****************************************************************************/ tsapi TSReturnCode TSHttpTxnCacheLookupCountGet(TSHttpTxn txnp, int *lookup_count); tsapi TSReturnCode TSHttpTxnServerRespIgnore(TSHttpTxn txnp); @@ -390,7 +390,7 @@ tsapi void TSFetchWriteData(TSFetchSM fetch_sm, const void *data, size_t len); tsapi ssize_t TSFetchReadData(TSFetchSM fetch_sm, void *buf, size_t len); /* - * Lanuch FetchSM to do http request, before calling this API, + * Launch FetchSM to do http request, before calling this API, * you should append http request header into fetch sm through * TSFetchWriteData() API * diff --git a/include/ts/remap.h b/include/ts/remap.h index edc1c23e116..cc263fdd4ea 100644 --- a/include/ts/remap.h +++ b/include/ts/remap.h @@ -43,7 +43,7 @@ typedef struct _tsremap_api_info { typedef struct _tm_remap_request_info { /* Important: You should *not* release these buf pointers or TSMLocs from your plugin! */ - /* these URL mloc's are read only, use normal ts/ts.h APIs for accesing */ + /* these URL mloc's are read only, use normal ts/ts.h APIs for accessing */ TSMLoc mapFromUrl; TSMLoc mapToUrl; @@ -61,7 +61,7 @@ typedef struct _tm_remap_request_info { /* This is the type returned by the TSRemapDoRemap() callback */ typedef enum { - TSREMAP_NO_REMAP = 0, /* No remaping was done, continue with next in chain */ + TSREMAP_NO_REMAP = 0, /* No remapping was done, continue with next in chain */ TSREMAP_DID_REMAP = 1, /* Remapping was done, continue with next in chain */ TSREMAP_NO_REMAP_STOP = 2, /* No remapping was done, and stop plugin chain evaluation */ TSREMAP_DID_REMAP_STOP = 3, /* Remapping was done, but stop plugin chain evaluation */ @@ -88,7 +88,7 @@ typedef enum { */ tsapi TSReturnCode TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size); -/* This gets called everytime remap.config is reloaded. This is complementary +/* This gets called every time remap.config is reloaded. This is complementary to TSRemapInit() which gets called when the plugin is first loaded. You can not fail, or cause reload to stop here, it's merely a notification. Optional function. @@ -99,7 +99,7 @@ tsapi void TSRemapConfigReload(void); /* Remap new request Mandatory interface function. Remap API plugin can/should use SDK API function calls inside this function! - return: TSREMAP_NO_REMAP - No remaping was done, continue with next in chain + return: TSREMAP_NO_REMAP - No remapping was done, continue with next in chain TSREMAP_DID_REMAP - Remapping was done, continue with next in chain TSREMAP_NO_REMAP_STOP - No remapping was done, and stop plugin chain evaluation TSREMAP_DID_REMAP_STOP - Remapping was done, but stop plugin chain evaluation @@ -113,7 +113,7 @@ tsapi void TSRemapDone(void); /* Plugin new instance. Create new plugin processing entry for unique remap record. First two arguments in argv vector are - fromURL and toURL from remap record. Please keep in mind that fromURL and toURL will be converted to canonical view. - Return: TS_SUCESS + Return: TS_SUCCESS TS_ERROR - instance creation error */ tsapi TSReturnCode TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size); diff --git a/include/ts/ts.h b/include/ts/ts.h index b15641449c4..55bae84c822 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -1309,7 +1309,7 @@ tsapi TSReturnCode TSHttpTxnCacheLookupStatusGet(TSHttpTxn txnp, int *lookup_sta tsapi TSReturnCode TSHttpTxnTransformRespGet(TSHttpTxn txnp, TSMBuffer *bufp, TSMLoc *offset); /** Set the @a port value for the inbound (user agent) connection in the transaction @a txnp. - This is used primarily where the conection is synthetic and therefore does not have a port. + This is used primarily where the connection is synthetic and therefore does not have a port. @note @a port is in @b host @b order. */ tsapi void TSHttpTxnClientIncomingPortSet(TSHttpTxn txnp, int port); @@ -1452,8 +1452,8 @@ tsapi TSReturnCode TSHttpTxnServerPacketDscpSet(TSHttpTxn txnp, int dscp); /** Sets an error type body to a transaction. Note that both string arguments must be allocated with TSmalloc() or TSstrdup(). The mimetype argument is - optional, if not provided it defaults to "text/html". Sending an emptry - string would prevent setting a content type header (but that is not adviced). + optional, if not provided it defaults to "text/html". Sending an empty + string would prevent setting a content type header (but that is not advised). @param txnp HTTP transaction whose parent proxy to get. @param buf The body message (must be heap allocated). @@ -1877,7 +1877,7 @@ tsapi TSAction TSCacheWrite(TSCont contp, TSCacheKey key); anything. The user does not get any vconnection from the cache, since no data needs to be transferred. When the cache calls contp back with TS_EVENT_CACHE_REMOVE, the remove has already - been commited. + been committed. @param contp continuation that the cache calls back reporting the success or failure of the remove. @@ -2019,7 +2019,7 @@ tsapi void TSStatIntDecrement(int the_stat, TSMgmtInt amount); tsapi TSMgmtInt TSStatIntGet(int the_stat); tsapi void TSStatIntSet(int the_stat, TSMgmtInt value); /* Currently not supported. */ -/* tsapi TSeturnCode TSStatFloatGet(int the_stat, float* value); */ +/* tsapi TSReturnCode TSStatFloatGet(int the_stat, float* value); */ /* tsapi TSReturnCode TSStatFloatSet(int the_stat, float value); */ tsapi TSReturnCode TSStatFindName(const char *name, int *idp); @@ -2033,7 +2033,7 @@ tsapi void TSDebug(const char *tag, const char *format_str, ...) TS_PRINTFLIKE(2 Output a debug line even if the debug tag is turned off, as long as debugging is enabled. Could be used as follows: @code - TSDebugSpecifc(TSHttpTxnDebugGet(txn), "plugin_tag" , "Hello World from transaction %p", txn); + TSDebugSpecific(TSHttpTxnDebugGet(txn), "plugin_tag" , "Hello World from transaction %p", txn); @endcode will be printed if the plugin_tag is enabled or the transaction specific debugging is turned on for txn. @@ -2397,7 +2397,7 @@ tsapi TSReturnCode TSHttpTxnMilestoneGet(TSHttpTxn txnp, TSMilestonesType milest tsapi int TSHttpTxnIsCacheable(TSHttpTxn txnp, TSMBuffer request, TSMBuffer response); /** - Return a string respresentation for a TSServerState value. This is useful for plugin debugging. + Return a string representation for a TSServerState value. This is useful for plugin debugging. @param state the value of this TSServerState @@ -2406,7 +2406,7 @@ tsapi int TSHttpTxnIsCacheable(TSHttpTxn txnp, TSMBuffer request, TSMBuffer resp tsapi const char *TSHttpServerStateNameLookup(TSServerState state); /** - Return a string respresentation for a TSHttpHookID value. This is useful for plugin debugging. + Return a string representation for a TSHttpHookID value. This is useful for plugin debugging. @param hook the value of this TSHttpHookID @@ -2415,7 +2415,7 @@ tsapi const char *TSHttpServerStateNameLookup(TSServerState state); tsapi const char *TSHttpHookNameLookup(TSHttpHookID hook); /** - Return a string respresentation for a TSEvent value. This is useful for plugin debugging. + Return a string representation for a TSEvent value. This is useful for plugin debugging. @param event the value of this TSHttpHookID diff --git a/include/tscore/AcidPtr.h b/include/tscore/AcidPtr.h index 6656705b916..bc4fc6fdab2 100644 --- a/include/tscore/AcidPtr.h +++ b/include/tscore/AcidPtr.h @@ -35,7 +35,7 @@ ////////////////////////////////////////////////////////// // Lock Pool /// Intended to make datasets thread safe by assigning locks to stripes of data, kind of like a bloom filter. -/** Allocates a fixed number of locks and retrives one with a hash. +/** Allocates a fixed number of locks and retrieves one with a hash. */ template struct LockPool { /** @@ -152,7 +152,7 @@ template class AcidPtr /////////////////////////////////////////// /// AcidCommitPtr -/// a globally exclusive pointer, for commiting changes to AcidPtr. +/// a globally exclusive pointer, for committing changes to AcidPtr. /** used for COPY_SWAP functionality. * 1. copy data (construct) * 2. overwrite data (scope) diff --git a/include/tscore/Allocator.h b/include/tscore/Allocator.h index 3de986a8c45..c03f996ab84 100644 --- a/include/tscore/Allocator.h +++ b/include/tscore/Allocator.h @@ -23,7 +23,7 @@ Provides three classes - Allocator for allocating memory blocks of fixed size - ClassAllocator for allocating objects - - SpaceClassAllocator for allocating sparce objects (most members uninitialized) + - SpaceClassAllocator for allocating sparse objects (most members uninitialized) These class provides a efficient way for handling dynamic allocation. The fast allocator maintains its own freepool of objects from diff --git a/include/tscore/ArgParser.h b/include/tscore/ArgParser.h index 95a5a2d64ee..45b0e416252 100644 --- a/include/tscore/ArgParser.h +++ b/include/tscore/ArgParser.h @@ -176,7 +176,7 @@ class ArgParser // The help & version messages void help_message(std::string_view err = "") const; void version_message() const; - // Helpr method for parse() + // Helper method for parse() void append_option_data(Arguments &ret, AP_StrVec &args, int index); // The command name and help message std::string _name; @@ -227,7 +227,7 @@ class ArgParser std::string const &key = ""); Command &add_command(std::string const &cmd_name, std::string const &cmd_description, std::string const &cmd_envvar, unsigned cmd_arg_num, Function const &f = nullptr, std::string const &key = ""); - // give a defaut command to this parser + // give a default command to this parser void set_default_command(std::string const &cmd); /** Main parsing function @return The Arguments object available for program using diff --git a/include/tscore/AtomicBit.h b/include/tscore/AtomicBit.h index 25d399f5339..72581d3aad3 100644 --- a/include/tscore/AtomicBit.h +++ b/include/tscore/AtomicBit.h @@ -31,7 +31,7 @@ ////////////////////////////////////////////////////// /// AtomicBit for inplace atomic bit operations -/* useful when you refernce a bit packed into a byte (unit_8) as a bool&, +/* useful when you reference a bit packed into a byte (unit_8) as a bool&, * you want a bit to 'walk and talk' like an std::atomic or std::atomic_flag. * In practice this is constructed at time of the operation(s), * storing it would defeat the purpose of packing the bits. diff --git a/include/tscore/BufferWriter.h b/include/tscore/BufferWriter.h index 57b6c570f67..42610fa0159 100644 --- a/include/tscore/BufferWriter.h +++ b/include/tscore/BufferWriter.h @@ -105,7 +105,7 @@ class BufferWriter invalidate the current auxiliary buffer (contents and address). Care must be taken to not write to data beyond this plus @c remaining bytes. Usually the - safest mechanism is to create a @c FixedBufferWriter on the auxillary buffer and write to that. + safest mechanism is to create a @c FixedBufferWriter on the auxiliary buffer and write to that. @code ts::FixedBufferWriter subw(w.auxBuffer(), w.remaining()); @@ -124,7 +124,7 @@ class BufferWriter /** Advance the buffer position @a n bytes. This treats the next @a n bytes as being written without changing the content. This is useful - only in conjuction with @a auxBuffer to indicate that @a n bytes of the auxillary buffer has + only in conjunction with @a auxBuffer to indicate that @a n bytes of the auxiliary buffer has been written by some other mechanism. @internal Concrete subclasses @b must override this to advance in a way consistent with the @@ -155,7 +155,7 @@ class BufferWriter } /// Get the remaining buffer space. - /// @return Number of additional characters that can be written without causing an error condidtion. + /// @return Number of additional characters that can be written without causing an error condition. size_t remaining() const { @@ -178,12 +178,12 @@ class BufferWriter /** BufferWriter print. This prints its arguments to the @c BufferWriter @a w according to the format @a fmt. The format - string is based on Python style formating, each argument substitution marked by braces, {}. Each - specification has three parts, a @a name, a @a specifier, and an @a extention. These are + string is based on Python style formatting, each argument substitution marked by braces, {}. Each + specification has three parts, a @a name, a @a specifier, and an @a extension. These are separated by colons. The name should be either omitted or a number, the index of the argument to use. If omitted the place in the format string is used as the argument index. E.g. "{} {} {}", "{} {1} {}", and "{0} {1} {2}" are equivalent. Using an explicit index does not reset the - position of subsequent substiations, therefore "{} {0} {}" is equivalent to "{0} {0} {2}". + position of subsequent substitutions, therefore "{} {0} {}" is equivalent to "{0} {0} {2}". */ template BufferWriter &print(TextView fmt, Rest &&... rest); /** Print overload to take arguments as a tuple instead of explicitly. @@ -378,9 +378,9 @@ class FixedBufferWriter : public BufferWriter /** Get a @c FixedBufferWriter for the unused output buffer. - If @a reserve is non-zero then the buffer size for the auxillary writer will be @a reserve bytes + If @a reserve is non-zero then the buffer size for the auxiliary writer will be @a reserve bytes smaller than the remaining buffer. This "reserves" space for additional output after writing - to the auxillary buffer, in a manner similar to @c clip / @c extend. + to the auxiliary buffer, in a manner similar to @c clip / @c extend. */ FixedBufferWriter auxWriter(size_t reserve = 0) @@ -526,8 +526,8 @@ namespace bw_fmt } /// This exists only to expand the index sequence into an array of formatters for the tuple type - /// @a TUPLE. Due to langauge limitations it cannot be done directly. The formatters can be - /// accessed via standard array access in constrast to templated tuple access. The actual array is + /// @a TUPLE. Due to language limitations it cannot be done directly. The formatters can be + /// accessed via standard array access in contrast to templated tuple access. The actual array is /// static and therefore at run time the only operation is loading the address of the array. template ArgFormatterSignature * @@ -625,7 +625,7 @@ BufferWriter::printv(TextView fmt, std::tuple const &args) int arg_idx = 0; // the next argument index to be processed. while (fmt.size()) { - // Next string piece of interest is an (optional) literal and then an (optinal) format specifier. + // Next string piece of interest is an (optional) literal and then an (optional) format specifier. // There will always be a specifier except for the possible trailing literal. std::string_view lit_v; std::string_view spec_v; diff --git a/include/tscore/BufferWriterForward.h b/include/tscore/BufferWriterForward.h index 9bc874c57df..50faa9ecdc1 100644 --- a/include/tscore/BufferWriterForward.h +++ b/include/tscore/BufferWriterForward.h @@ -60,7 +60,7 @@ struct BWFSpec { // @a _min is unsigned because there's no point in an invalid default, 0 works fine. unsigned int _min = 0; ///< Minimum width. int _prec = -1; ///< Precision - unsigned int _max = std::numeric_limits::max(); ///< Maxium width + unsigned int _max = std::numeric_limits::max(); ///< Maximum width int _idx = -1; ///< Positional "name" of the specification. std::string_view _name; ///< Name of the specification. std::string_view _ext; ///< Extension if provided. diff --git a/include/tscore/CryptoHash.h b/include/tscore/CryptoHash.h index c9e1835f264..6fe34e1bc2f 100644 --- a/include/tscore/CryptoHash.h +++ b/include/tscore/CryptoHash.h @@ -126,7 +126,7 @@ class CryptoContextBase /// Convenience - compute final @a hash for @a data. /// @note This is just as fast as the previous style, as a new context must be initialized - /// everytime this is done. + /// every time this is done. bool hash_immediate(CryptoHash &hash, void const *data, int length); }; diff --git a/include/tscore/Extendible.h b/include/tscore/Extendible.h index e4f46826dd7..add2f915845 100644 --- a/include/tscore/Extendible.h +++ b/include/tscore/Extendible.h @@ -68,7 +68,7 @@ areStaticsFrozen() * @brief Allows code (and Plugins) to declare member variables during system init. * * The size of this structure is actually zero, so it will not change the size of your derived class. - * But new and delete are overriden to use allocate enough bytes of the derived type + added fields. + * But new and delete are overridden to use allocate enough bytes of the derived type + added fields. * All bool's are packed to save space using the *Bit methods. * This API is focused on thread safe data types that allow minimally blocked reading. * This is templated so static variables are instanced per Derived type. B/c we need to have different @@ -196,7 +196,7 @@ template struct Extendible { }; // end Schema struct private: - // Extendible convience methods + // Extendible convenience methods char *this_as_char_ptr(); char const *this_as_char_ptr() const; @@ -392,7 +392,7 @@ void Extendible::Schema::call_construct(char *ext_as_char_ptr) { ++instance_count; // don't allow schema modification - // init all extendible memory to 0, incase constructors don't + // init all extendible memory to 0, in case constructors don't memset(ext_as_char_ptr + sizeof(Derived_t), 0, alloc_size - sizeof(Derived_t)); for (auto const &elm : fields) { diff --git a/include/tscore/HostLookup.h b/include/tscore/HostLookup.h index 58a0ec344dc..1d5bdb7302e 100644 --- a/include/tscore/HostLookup.h +++ b/include/tscore/HostLookup.h @@ -36,7 +36,7 @@ #include #include -// HostLookup constantss +// HostLookup constants constexpr int HOST_TABLE_DEPTH = 3; // Controls the max number of levels in the logical tree constexpr int HOST_ARRAY_MAX = 8; // Sets the fixed array size diff --git a/include/tscore/I_Layout.h b/include/tscore/I_Layout.h index 18aa1415463..eb4d2609117 100644 --- a/include/tscore/I_Layout.h +++ b/include/tscore/I_Layout.h @@ -65,7 +65,7 @@ struct Layout { /** Return file path relative to dir - Store the path to buf. The buf should be large eough to store + Store the path to buf. The buf should be large enough to store Example usage: Layout::relative_to(default_layout()->sysconfdir, "foo.bar"); */ static void relative_to(char *buf, size_t bufsz, std::string_view dir, std::string_view file); diff --git a/include/tscore/IntrusiveHashMap.h b/include/tscore/IntrusiveHashMap.h index 3e2b5c9d13d..704b964e1e6 100644 --- a/include/tscore/IntrusiveHashMap.h +++ b/include/tscore/IntrusiveHashMap.h @@ -298,7 +298,7 @@ template class IntrusiveHashMap Bucket *bucket_for(key_type key); - ExpansionPolicy _expansion_policy{DEFAULT_EXPANSION_POLICY}; ///< When to exand the table. + ExpansionPolicy _expansion_policy{DEFAULT_EXPANSION_POLICY}; ///< When to expand the table. size_t _expansion_limit{DEFAULT_EXPANSION_LIMIT}; ///< Limit value for expansion. // noncopyable diff --git a/include/tscore/IpMap.h b/include/tscore/IpMap.h index 568374ebfc1..50a9fbe2434 100644 --- a/include/tscore/IpMap.h +++ b/include/tscore/IpMap.h @@ -75,7 +75,7 @@ namespace detail data. Marking takes a painter's algorithm approach -- any marking overwrites any previous marking on an address. Details of marking calls are discarded and only the final results are kept. That is, - a client cannot unmark expliticly any previous marking. Only a + a client cannot unmark explicitly any previous marking. Only a specific range of addresses can be unmarked. Both IPv4 and IPv6 are supported in the same map. Mixed ranges are @@ -310,7 +310,7 @@ class IpMap /** Test for membership. - @note Covenience overload for IPv4. + @note Convenience overload for IPv4. @return @c true if the address is in the map, @c false if not. If the address is in the map and @a ptr is not @c nullptr, @c *ptr diff --git a/include/tscore/MemArena.h b/include/tscore/MemArena.h index 3a3519a0b2e..8d0d8ccd9ae 100644 --- a/include/tscore/MemArena.h +++ b/include/tscore/MemArena.h @@ -168,7 +168,7 @@ class MemArena /** Release all memory. - Empties the entire arena and deallocates all underlying memory. The hint for the next reservered block size will + Empties the entire arena and deallocates all underlying memory. The hint for the next reserved block size will be @a n if @a n is not zero, otherwise it will be the sum of all allocations when this method was called. @return @c *this diff --git a/include/tscore/ParseRules.h b/include/tscore/ParseRules.h index 847522e7d7e..e69e7378cb6 100644 --- a/include/tscore/ParseRules.h +++ b/include/tscore/ParseRules.h @@ -136,7 +136,7 @@ class ParseRules ////////////////// static CTypeResult is_escape(const char *seq); // % - static CTypeResult is_uchar(const char *seq); // starts unresrvd or is escape + static CTypeResult is_uchar(const char *seq); // starts unreserved or is escape static CTypeResult is_pchar(const char *seq); // uchar,:,@,&,=,+ (see code) /////////////////// @@ -766,7 +766,7 @@ ParseRules::strlen_eow(const char *s) // // This function is the same as strstr(), except that it accepts strings // that are terminated with '\r', '\n' or null. -// It returns a pointer to the first occurance of s2 within s1 (or null). +// It returns a pointer to the first occurrence of s2 within s1 (or null). ////////////////////////////////////////////////////////////////////////////// inline const char * ParseRules::strstr_eow(const char *s1, const char *s2) diff --git a/include/tscore/Ptr.h b/include/tscore/Ptr.h index 559f346feb3..3a79e01b9dd 100644 --- a/include/tscore/Ptr.h +++ b/include/tscore/Ptr.h @@ -143,7 +143,7 @@ template class Ptr } // Return the raw pointer as a RefCount object. Typically - // this is for keeping a collection of heterogenous objects. + // this is for keeping a collection of ogenous objects. RefCountObj * object() const { diff --git a/include/tscore/Scalar.h b/include/tscore/Scalar.h index 06c339832e0..5eb0dffa463 100644 --- a/include/tscore/Scalar.h +++ b/include/tscore/Scalar.h @@ -224,7 +224,7 @@ template class Scalar static_assert(N > 0, "The scaling factor (1st template argument) must be a positive integer"); static_assert(std::is_integral::value, "The counter type (2nd template argument) must be an integral type"); - constexpr Scalar(); ///< Default contructor. + constexpr Scalar(); ///< Default constructor. ///< Construct to have @a n scaled units. explicit constexpr Scalar(Counter n); /// Copy constructor. @@ -872,7 +872,7 @@ template Scalar operator/(Scalar lhs, I n) { - static_assert(std::is_integral::value, "Scalar divsion only support integral types."); + static_assert(std::is_integral::value, "Scalar division only support integral types."); return Scalar(lhs) /= n; } diff --git a/include/tscore/SimpleTokenizer.h b/include/tscore/SimpleTokenizer.h index 15471ef66d9..82a257fff9f 100644 --- a/include/tscore/SimpleTokenizer.h +++ b/include/tscore/SimpleTokenizer.h @@ -127,7 +127,7 @@ class SimpleTokenizer { } - // NOTE: The input strring 's' is overwritten for mode OVERWRITE_INPUT_STRING. + // NOTE: The input string 's' is overwritten for mode OVERWRITE_INPUT_STRING. SimpleTokenizer(const char *s, char delimiter = ' ', unsigned mode = 0, char escape = '\\') : _delimiter(delimiter), _mode(mode), _escape(escape) { diff --git a/include/tscore/TextBuffer.h b/include/tscore/TextBuffer.h index 8ef01debe41..65dd1ec30a4 100644 --- a/include/tscore/TextBuffer.h +++ b/include/tscore/TextBuffer.h @@ -25,7 +25,7 @@ /**************************************************************************** * - * TextBuffer.h - A self-expanding buffer, primarly meant for strings + * TextBuffer.h - A self-expanding buffer, primarily meant for strings * * * diff --git a/include/tscore/Tokenizer.h b/include/tscore/Tokenizer.h index 562279b0f1f..a86c96abc0a 100644 --- a/include/tscore/Tokenizer.h +++ b/include/tscore/Tokenizer.h @@ -43,26 +43,26 @@ * * There are three memory options. * SHARE_TOKS - this modifies the original string passed in - * through Intialize() and shares its space. NULLs + * through Initialize() and shares its space. NULLs * are inserted into string after each token. Choosing - * this option means the user is reponsible for not + * this option means the user is responsible for not * deallocating the string storage before deallocating * the tokenizer object - * COPY_TOKS - this option copies the orginial string and + * COPY_TOKS - this option copies the original string and * leaves the original unchanged. The deallocation of the * original string and the deallocation of the Tokenizer * object are now independent. * Note: If neither SHARE_TOKS or COPY_TOKS is selected, COPY_TOKS * is the default * ALLOW_EMPTY_TOKENS: If multiple delimiters appear next to each - * other, each delimiter creates a token someof which + * other, each delimiter creates a token some of which * will be zero length. The default is to skip repeated * delimiters * * Tokenizer(const char* StrOfDelimit) - a string that contains * the delimiters for tokenizing. This string is copied. * - * Intialize(char* str, TokenizerOpts opt) - Submits a string + * Initialize(char* str, TokenizerOpts opt) - Submits a string * to be tokenized according to the memory options listed above * * ReUse() - Allows the object to be reused for a new string @@ -76,7 +76,7 @@ * is intended to be used on a small number of tokens * * iterFirst(tok_iter_state* state) - Returns the first - * token and intializes state argument for subsequent + * token and initializes state argument for subsequent * calls to iterNext. If no tokens exist, NULL is * returned * diff --git a/include/tscore/ink_base64.h b/include/tscore/ink_base64.h index b536fd6d1bd..a3c4d510afc 100644 --- a/include/tscore/ink_base64.h +++ b/include/tscore/ink_base64.h @@ -25,7 +25,7 @@ /* * Base64 encoding and decoding as according to RFC1521. Similar to uudecode. - * See RFC1521 for specificiation. + * See RFC1521 for specification. * * RFC 1521 requires inserting line breaks for long lines. The basic web * authentication scheme does not require them. This implementation is diff --git a/include/tscore/ink_cap.h b/include/tscore/ink_cap.h index 4a2c7d57ca1..6ffaccbbf36 100644 --- a/include/tscore/ink_cap.h +++ b/include/tscore/ink_cap.h @@ -52,7 +52,7 @@ extern FILE *elevating_fopen(const char *path, const char *mode); // chmod a file, elevating if necessary extern int elevating_chmod(const char *path, int perm); -/// @c stat a file, evelating only if needed. +/// @c stat a file, elevating only if needed. extern int elevating_stat(const char *path, struct stat *buff); /** Control generate of core file on crash. diff --git a/include/tscore/ink_code.h b/include/tscore/ink_code.h index 2b08574f064..c10dd1e2af4 100644 --- a/include/tscore/ink_code.h +++ b/include/tscore/ink_code.h @@ -32,7 +32,7 @@ typedef MD5_CTX INK_DIGEST_CTX; /* - Wrappers around the MD5 functions, all of this should be depericated and just use the functions directly + Wrappers around the MD5 functions, all of this should be deprecated and just use the functions directly */ inkcoreapi int ink_code_md5(unsigned const char *input, int input_length, unsigned char *sixteen_byte_hash_pointer); diff --git a/include/tscore/ink_endian.h b/include/tscore/ink_endian.h index 30b3132598f..cee79d45b1b 100644 --- a/include/tscore/ink_endian.h +++ b/include/tscore/ink_endian.h @@ -1,6 +1,6 @@ /** @file * - * Endian convertion routines + * Endian conversion routines * * @section license License * diff --git a/include/tscore/ink_file.h b/include/tscore/ink_file.h index fb0487c6985..6cb656f4583 100644 --- a/include/tscore/ink_file.h +++ b/include/tscore/ink_file.h @@ -45,7 +45,7 @@ #include #endif -// Darwin keeps statafs(2) in ... +// Darwin keeps statfs(2) in ... #if HAVE_SYS_MOUNT_H #include #endif diff --git a/include/tscore/ink_hrtime.h b/include/tscore/ink_hrtime.h index f321476913a..9358b6358db 100644 --- a/include/tscore/ink_hrtime.h +++ b/include/tscore/ink_hrtime.h @@ -44,7 +44,7 @@ char *int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int * ////////////////////////////////////////////////////////////////////////////// // -// Factors to multiply units by to obtain coresponding ink_hrtime values. +// Factors to multiply units by to obtain corresponding ink_hrtime values. // ////////////////////////////////////////////////////////////////////////////// @@ -81,7 +81,7 @@ char *int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int * #define HRTIME_USECONDS(_x) ((_x)*HRTIME_USECOND) #define HRTIME_NSECONDS(_x) ((_x)*HRTIME_NSECOND) -// gratuituous wrappers +// gratuitous wrappers static inline ink_hrtime ink_hrtime_from_years(unsigned int years) diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index 39403374375..ff6c10406e5 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -117,7 +117,7 @@ union IpEndpoint { in_port_t &port(); /// Port in network order. in_port_t port() const; - /// Port in host horder. + /// Port in host order. in_port_t host_order_port() const; operator sockaddr *() { return &sa; } diff --git a/include/tscore/ink_lockfile.h b/include/tscore/ink_lockfile.h index 8f5aa2ef6f7..dab70612374 100644 --- a/include/tscore/ink_lockfile.h +++ b/include/tscore/ink_lockfile.h @@ -79,7 +79,7 @@ class Lockfile // If the lock file open succeeds, it closes the lock file releasing // the lock. // - // The intial signal can be used to generate a core from the process while + // The initial signal can be used to generate a core from the process while // still ensuring it dies. void Kill(int sig, int initial_sig = 0, const char *pname = nullptr); void KillGroup(int sig, int initial_sig = 0, const char *pname = nullptr); diff --git a/include/tscore/ink_resolver.h b/include/tscore/ink_resolver.h index fee408470ed..b9af473f772 100644 --- a/include/tscore/ink_resolver.h +++ b/include/tscore/ink_resolver.h @@ -109,7 +109,7 @@ #define INK_RES_AAONLY 0x00000004 /*%< authoritative answers only (!IMPL)*/ #define INK_RES_USEVC 0x00000008 /*%< use virtual circuit */ #define INK_RES_PRIMARY 0x00000010 /*%< query primary server only (!IMPL) */ -#define INK_RES_IGNTC 0x00000020 /*%< ignore trucation errors */ +#define INK_RES_IGNTC 0x00000020 /*%< ignore truncation errors */ #define INK_RES_RECURSE 0x00000040 /*%< recursion desired */ #define INK_RES_DEFNAMES 0x00000080 /*%< use default domain name */ #define INK_RES_STAYOPEN 0x00000100 /*%< Keep TCP socket open */ @@ -181,7 +181,7 @@ enum HostResStyle { /// Strings for host resolution styles extern const char *const HOST_RES_STYLE_STRING[]; -/// Caclulate the effective resolution preferences. +/// Calculate the effective resolution preferences. extern HostResStyle ats_host_res_from(int family, ///< Connection family HostResPreferenceOrder ///< Preference ordering. ); diff --git a/include/tscore/signals.h b/include/tscore/signals.h index 5e23ab4070b..54fc9e5583f 100644 --- a/include/tscore/signals.h +++ b/include/tscore/signals.h @@ -50,6 +50,6 @@ bool signal_is_masked(int signo); // Test whether the signal is being handled by the given handler. bool signal_check_handler(int signo, signal_handler_t handler); -// Start a thread to test whether signals have the expected handler. Apparantly useful for +// Start a thread to test whether signals have the expected handler. Apparently useful for // finding pthread bugs in some version of DEC Unix. void signal_start_check_thread(signal_handler_t handler); diff --git a/include/tscpp/api/Continuation.h b/include/tscpp/api/Continuation.h index ef25dabc9bb..1557ecc964b 100644 --- a/include/tscpp/api/Continuation.h +++ b/include/tscpp/api/Continuation.h @@ -41,7 +41,7 @@ class Continuation TSContDataSet(_cont, static_cast(this)); } - // Create "empty" continuation, can only be populated by move assignement. + // Create "empty" continuation, can only be populated by move assignment. // Continuation() {} @@ -131,7 +131,7 @@ class Continuation } protected: - // Distinct continuation behavior is acheived by overriding this function in a derived continutation type. + // Distinct continuation behavior is achieved by overriding this function in a derived continuation type. // virtual int _run(TSEvent event, void *edata) = 0; diff --git a/include/tscpp/api/GlobalPlugin.h b/include/tscpp/api/GlobalPlugin.h index 1df60bddc6c..1d76ed5d8ff 100644 --- a/include/tscpp/api/GlobalPlugin.h +++ b/include/tscpp/api/GlobalPlugin.h @@ -33,7 +33,7 @@ struct GlobalPluginState; * @brief The interface used when creating a GlobalPlugin. * * A GlobalPlugin is a Plugin that will fire for a given hook on all Transactions. - * In otherwords, a GlobalPlugin is not tied to a specific plugin, a Transaction + * In other words, a GlobalPlugin is not tied to a specific plugin, a Transaction * specific plugin would be a TransactionPlugin. * * Depending on the @@ -60,9 +60,9 @@ class GlobalPlugin : public Plugin /** * registerHook is the mechanism used to attach a global hook. * - * \note Whenever you register a hook you must have the appropriate callback definied in your GlobalPlugin + * \note Whenever you register a hook you must have the appropriate callback defined in your GlobalPlugin * see HookType and Plugin for the correspond HookTypes and callback methods. If you fail to implement the - * callback, a default implmentation will be used that will only resume the Transaction. + * callback, a default implementation will be used that will only resume the Transaction. * * @param HookType the type of hook you wish to register * @see HookType diff --git a/include/tscpp/api/GzipDeflateTransformation.h b/include/tscpp/api/GzipDeflateTransformation.h index 039965eded4..10cfb621283 100644 --- a/include/tscpp/api/GzipDeflateTransformation.h +++ b/include/tscpp/api/GzipDeflateTransformation.h @@ -42,7 +42,7 @@ namespace transformations * * The GzipDeflateTransformation is a helper transformation that can be used * to easily compress content. For a full example of GzipDeflateTransformation - * and GzipInflateTransformation see examples/gzip_transformation/. + * and GzipInflateTransformation see example/cppapi/gzip_transformation/. * * @note GzipDeflateTransformation DOES NOT set Content-Encoding headers, it is the * users responsibility to set any applicable headers. @@ -54,7 +54,7 @@ namespace transformations public: /** * A full example of how to use GzipDeflateTransformation and GzipInflateTransformation is available - * in examples/gzip_tranformation/ + * in example/cppapi/gzip_transformation/ * * @param transaction As with any TransformationPlugin you must pass in the transaction * @param type because the GzipDeflateTransformation can be used with both requests and responses @@ -74,7 +74,7 @@ namespace transformations /** * Any TransformationPlugin must implement handleInputComplete(), this method will - * finalize the gzip compression and flush any remaining data and the epilouge. + * finalize the gzip compression and flush any remaining data and the epilogue. */ void handleInputComplete() override; diff --git a/include/tscpp/api/GzipInflateTransformation.h b/include/tscpp/api/GzipInflateTransformation.h index c664c182e0d..358fdc7b658 100644 --- a/include/tscpp/api/GzipInflateTransformation.h +++ b/include/tscpp/api/GzipInflateTransformation.h @@ -41,12 +41,12 @@ namespace transformations * * The GzipInflateTransformation is a helper transformation that can be used * to easily decompress gzipped content. For a full example of GzipInflateTransformation - * and GzipDeflateTransformation see examples/gzip_transformation/. + * and GzipDeflateTransformation see example/cppapi/gzip_transformation/. * * @note GzipDeflateTransformation DOES NOT set or check Content-Encoding headers, it is the - * users responsibility to set any applicable headers and check that the content is acctually + * users responsibility to set any applicable headers and check that the content is actually * gzipped by checking the Content-Encoding header before creating a GzipInflateTransformation, - * see examples/gzip_transformation/ for a full example. + * see example/cppapi/gzip_transformation/ for a full example. * * @see GzipDeflateTransformation */ @@ -55,7 +55,7 @@ namespace transformations public: /** * A full example of how to use GzipInflateTransformation and GzipDeflateTransformation is available - * in examples/gzip_tranformation/ + * in example/cppapi/gzip_transformation/. * * @param transaction As with any TransformationPlugin you must pass in the transaction * @param type because the GzipInflateTransformation can be used with both requests and responses diff --git a/include/tscpp/api/Headers.h b/include/tscpp/api/Headers.h index 25e1ce18660..c956ff4a518 100644 --- a/include/tscpp/api/Headers.h +++ b/include/tscpp/api/Headers.h @@ -36,7 +36,7 @@ class Response; /** * @brief A HeaderFieldName is a lightweight wrapper around a string that allows for case insensitive comparisons. - * Because header field names must be case insensitive this allows easy case insentive comparisons of names. + * Because header field names must be case insensitive this allows easy case insensitive comparisons of names. * */ class HeaderFieldName @@ -75,7 +75,7 @@ class HeaderFieldName std::string str(); /** - * @return a const char * which points to the name of this HeaderFIeldName + * @return a const char * which points to the name of this HeaderFieldName */ const char *c_str(); @@ -119,7 +119,7 @@ class header_field_value_iterator : public std::iterator class IntrusiveDList value_type *operator->() const; /// Convenience conversion to pointer type - /// Because of how this list is normally used, being able to pass an iterator as a pointer is quite convienent. + /// Because of how this list is normally used, being able to pass an iterator as a pointer is quite convenient. /// If the iterator isn't valid, it converts to @c nullptr. operator value_type *() const; @@ -150,7 +150,7 @@ template class IntrusiveDList protected: // These are stored non-const to make implementing @c iterator easier. This class provides the required @c const // protection. - list_type *_list{nullptr}; ///< Needed to descrement from @c end() position. + list_type *_list{nullptr}; ///< Needed to decrement from @c end() position. typename list_type::value_type *_v{nullptr}; ///< Referenced element. /// Internal constructor for containers. @@ -242,11 +242,11 @@ template class IntrusiveDList self_type &append(value_type *v); /// Remove the first element of the list. - /// @return A poiner to the removed item, or @c nullptr if the list was empty. + /// @return A pointer to the removed item, or @c nullptr if the list was empty. value_type *take_head(); /// Remove the last element of the list. - /// @return A poiner to the removed item, or @c nullptr if the list was empty. + /// @return A pointer to the removed item, or @c nullptr if the list was empty. value_type *take_tail(); /// Insert a new element @a elt after @a target. @@ -354,7 +354,7 @@ template class IntrusiveDList */ template struct IntrusiveLinkage { static T *&next_ptr(T *thing); ///< Retrieve reference to next pointer. - static T *&prev_ptr(T *thing); ///< Retrive reference to previous pointer. + static T *&prev_ptr(T *thing); ///< Retrieve reference to previous pointer. }; template diff --git a/include/tscpp/util/MemSpan.h b/include/tscpp/util/MemSpan.h index f0a2890b3d0..9779e021a21 100644 --- a/include/tscpp/util/MemSpan.h +++ b/include/tscpp/util/MemSpan.h @@ -224,7 +224,7 @@ class MemSpan */ self_type remove_prefix(void const *p); - /** Shringt the span from the front. + /** Shrink the span from the front. * * @param n The number of bytes to remove. * @return @c *this @@ -254,7 +254,7 @@ class MemSpan */ self_type &remove_suffix(void const *p); - /** Shringt the span from the back. + /** Shrink the span from the back. * * @param n The number of bytes to remove. * @return @c *this diff --git a/include/tscpp/util/TextView.h b/include/tscpp/util/TextView.h index ce620dfc8ed..e82044f7bc5 100644 --- a/include/tscpp/util/TextView.h +++ b/include/tscpp/util/TextView.h @@ -99,7 +99,7 @@ class TextView : public std::string_view /** Construct explicitly with a pointer and size. If @a n is negative it is treated as 0. - @internal Overload for convience, otherwise get "narrow conversion" errors. + @internal Overload for convenience, otherwise get "narrow conversion" errors. */ constexpr TextView(const char *ptr, ///< Pointer to buffer. int n ///< Size of buffer. @@ -116,7 +116,7 @@ class TextView : public std::string_view Construct directly from an array of characters. All elements of the array are included in the view unless the last element is nul, in which case it is elided. - If this is inapropriate then a constructor with an explicit size should be used. + If this is inappropriate then a constructor with an explicit size should be used. @code TextView a("A literal string"); @@ -291,10 +291,10 @@ class TextView : public std::string_view /// Overload to provide better return type. self_type &remove_prefix(size_t n); - /// Remove the prefix delimited by the first occurence of @a c. + /// Remove the prefix delimited by the first occurrence of @a c. self_type &remove_prefix_at(char c); - /// Remove the prefix delimited by the first occurence of a character for which @a pred is @c true. + /// Remove the prefix delimited by the first occurrence of a character for which @a pred is @c true. template self_type &remove_prefix_if(F const &pred); /** Split a prefix from the view on the character at offset @a n. @@ -391,10 +391,10 @@ class TextView : public std::string_view /// Overload to provide better return type. self_type &remove_suffix(size_t n); - /// Remove a suffix, delimited by the last occurence of @c c. + /// Remove a suffix, delimited by the last occurrence of @c c. self_type &remove_suffix_at(char c); - /// Remove a suffix, delimited by the last occurence of a character for which @a pred is @c true. + /// Remove a suffix, delimited by the last occurrence of a character for which @a pred is @c true. template self_type &remove_suffix_if(F const &f); /** Split the view to get a suffix of size @a n. diff --git a/include/wccp/Wccp.h b/include/wccp/Wccp.h index 40dff8fd4b6..ec1424cc93d 100644 --- a/include/wccp/Wccp.h +++ b/include/wccp/Wccp.h @@ -255,7 +255,7 @@ class EndPoint /// Perform house keeping, including sending outbound messages. int housekeeping(); - /// Recieve and process a message on the socket. + /// Receive and process a message on the socket. /// @return 0 for success, -ERRNO on system error. ts::Rv handleMessage(); From affba2de28e1fad411e373b890b89ce4ca5dfd39 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 7 Aug 2018 07:45:44 -0700 Subject: [PATCH 502/526] Fixes spelling in iocore --- iocore/aio/AIO.cc | 2 +- iocore/cache/Cache.cc | 16 +++++++-------- iocore/cache/CacheDir.cc | 4 ++-- iocore/cache/CacheHosting.cc | 8 ++++---- iocore/cache/CachePages.cc | 2 +- iocore/cache/CacheRead.cc | 8 ++++---- iocore/cache/CacheWrite.cc | 6 +++--- iocore/cache/P_CacheDir.h | 2 +- iocore/cache/P_CacheHosting.h | 2 +- iocore/cache/P_CacheInternal.h | 4 ++-- iocore/cache/RamCacheCLFUS.cc | 4 ++-- iocore/cache/Store.cc | 8 ++++---- iocore/dns/DNS.cc | 4 ++-- iocore/dns/DNSConnection.cc | 2 +- iocore/dns/test_P_DNS.cc | 2 +- iocore/eventsystem/EventSystem.cc | 2 +- iocore/eventsystem/I_Action.h | 10 +++++----- iocore/eventsystem/I_Continuation.h | 2 +- iocore/eventsystem/I_EThread.h | 24 +++++++++++------------ iocore/eventsystem/I_Event.h | 6 +++--- iocore/eventsystem/I_EventProcessor.h | 8 ++++---- iocore/eventsystem/I_IOBuffer.h | 10 +++++----- iocore/eventsystem/I_Lock.h | 4 ++-- iocore/eventsystem/I_Processor.h | 4 ++-- iocore/eventsystem/I_Thread.h | 2 +- iocore/eventsystem/I_VConnection.h | 16 +++++++-------- iocore/eventsystem/P_Freer.h | 2 +- iocore/eventsystem/P_IOBuffer.h | 2 +- iocore/eventsystem/Thread.cc | 2 +- iocore/eventsystem/UnixEThread.cc | 2 +- iocore/hostdb/HostDB.cc | 16 +++++++-------- iocore/hostdb/I_HostDBProcessor.h | 8 ++++---- iocore/hostdb/P_RefCountCacheSerializer.h | 2 +- iocore/hostdb/test_P_HostDB.cc | 2 +- iocore/net/I_Net.h | 4 ++-- iocore/net/I_NetProcessor.h | 2 +- iocore/net/I_NetVConnection.h | 18 ++++++++--------- iocore/net/I_UDPNet.h | 4 ++-- iocore/net/NetVCTest.cc | 4 ++-- iocore/net/OCSPStapling.cc | 2 +- iocore/net/P_Connection.h | 4 ++-- iocore/net/P_NetVCTest.h | 2 +- iocore/net/P_NetVConnection.h | 2 +- iocore/net/P_SSLConfig.h | 2 +- iocore/net/P_UDPNet.h | 2 +- iocore/net/P_UnixNet.h | 4 ++-- iocore/net/P_UnixNetProcessor.h | 2 +- iocore/net/P_UnixNetVConnection.h | 2 +- iocore/net/SSLConfig.cc | 2 +- iocore/net/SSLNetVConnection.cc | 8 ++++---- iocore/net/SSLNextProtocolSet.cc | 4 ++-- iocore/net/SSLSessionTicket.cc | 2 +- iocore/net/SSLUtils.cc | 8 ++++---- iocore/net/Socks.cc | 6 +++--- iocore/net/UnixNet.cc | 2 +- iocore/net/UnixNetVConnection.cc | 2 +- iocore/net/YamlSNIConfig.cc | 2 +- iocore/net/test_P_Net.cc | 2 +- iocore/net/test_certlookup.cc | 2 +- iocore/utils/I_OneWayMultiTunnel.h | 2 +- iocore/utils/I_OneWayTunnel.h | 2 +- 61 files changed, 148 insertions(+), 148 deletions(-) diff --git a/iocore/aio/AIO.cc b/iocore/aio/AIO.cc index e908542668b..48e08b475d8 100644 --- a/iocore/aio/AIO.cc +++ b/iocore/aio/AIO.cc @@ -538,7 +538,7 @@ DiskHandler::mainAIOEvent(int event, Event *e) if (ret < 0) { Debug("aio", "io_submit failed: %s (%d)", strerror(-ret), -ret); } else { - Fatal("could not sumbit IOs, io_submit(%p, %d, %p) returned %d", ctx, num, cbs, ret); + Fatal("could not submit IOs, io_submit(%p, %d, %p) returned %d", ctx, num, cbs, ret); } } } diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 8b6ab8d8c30..d28fa4cb768 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -246,7 +246,7 @@ cache_stats_bytes_used_cb(const char *name, RecDataT data_type, RecData *data, R RecRawStatSyncSum(name, data_type, data, rsb, id); RecGetGlobalRawStatSum(rsb, (int)cache_bytes_total_stat, &total); percent_full = (float)used / (float)total * 100; - // The perent_full float below gets rounded down + // The percent_full float below gets rounded down RecSetGlobalRawStatSum(rsb, (int)cache_percent_full_stat, (int64_t)percent_full); } @@ -851,7 +851,7 @@ CacheProcessor::cacheInitialized() int caches_ready = 0; int cache_init_ok = 0; /* allocate ram size in proportion to the disk space the - volume accupies */ + volume occupies */ int64_t total_size = 0; // count in HTTP & MIXT uint64_t total_cache_bytes = 0; // bytes that can used in total_size uint64_t total_direntries = 0; // all the direntries in the cache @@ -1138,7 +1138,7 @@ vol_init_data_internal(Vol *d) off_t total_entries = (d->len - (d->start - d->skip)) / cache_config_min_average_object_size; // step2: calculate the number of buckets off_t total_buckets = total_entries / DIR_DEPTH; - // step3: calculate the number of segments, no semgent has more than 16384 buckets + // step3: calculate the number of segments, no segment has more than 16384 buckets d->segments = (total_buckets + (((1 << 16) - 1) / DIR_DEPTH)) / ((1 << 16) / DIR_DEPTH); // step4: divide total_buckets into segments on average. d->buckets = (total_buckets + d->segments - 1) / d->segments; @@ -1531,7 +1531,7 @@ Vol::handle_recover_from_data(int event, void * /* data ATS_UNUSED */) s += round_to_approx_size(doc->len); continue; } - // case 3 - we have already recoverd some data and + // case 3 - we have already recovered some data and // (doc->sync_serial < last_sync_serial) || // (doc->sync_serial > header->sync_serial + 1). // if we are too close to the end, wrap around @@ -1855,7 +1855,7 @@ build_vol_hash_table(CacheHostRecord *cp) for (int i = 0; i < VOL_HASH_TABLE_SIZE; i++) { ttable[i] = VOL_HASH_EMPTY; } - // generate random numbers proportaion to allocation + // generate random numbers proportional to allocation rtable_pair *rtable = (rtable_pair *)ats_malloc(sizeof(rtable_pair) * rtable_size); int rindex = 0; for (int i = 0; i < num_vols; i++) { @@ -2154,7 +2154,7 @@ unmarshal_helper(Doc *doc, Ptr &buf, int &okay) @internal I looked at doing this in place (rather than a copy & modify) but - The in place logic would be even worse than this mess - It wouldn't save you that much, since you end up doing inserts early in the buffer. - Without extreme care in the logic it could end up doing more copying thatn + Without extreme care in the logic it could end up doing more copying than the simpler copy & modify. @internal This logic presumes the existence of some slack at the end of the buffer, which @@ -2221,7 +2221,7 @@ upgrade_doc_version(Ptr &buf) } Doc *n_doc = reinterpret_cast(buf->data()); // access as current version. // For now the base header size is the same. If that changes we'll need to handle the v22/23 case here - // as with the v21 and shift the content down to accomodate the bigger header. + // as with the v21 and shift the content down to accommodate the bigger header. ink_assert(sizeof(*n_doc) == sizeof(*doc)); n_doc->doc_type = CACHE_FRAG_TYPE_HTTP; // We converted so adjust doc_type. @@ -2795,7 +2795,7 @@ cplist_reconfigure() } } - /* change percentages in the config patitions to absolute value */ + /* change percentages in the config partitions to absolute value */ off_t tot_space_in_blks = 0; off_t blocks_per_vol = VOL_BLOCK_SIZE / STORE_BLOCK_SIZE; /* sum up the total space available on all the disks. diff --git a/iocore/cache/CacheDir.cc b/iocore/cache/CacheDir.cc index 109538f176f..a7a1255dec4 100644 --- a/iocore/cache/CacheDir.cc +++ b/iocore/cache/CacheDir.cc @@ -558,7 +558,7 @@ dir_probe(const CacheKey *key, Vol *d, Dir *result, Dir **last_collision) if (collision) { if (collision == e) { collision = nullptr; - // increment collison stat + // increment collision stat // Note: dir_probe could be called multiple times // for the same document and so the collision stat // may not accurately reflect the number of documents @@ -973,7 +973,7 @@ sync_cache_dir_on_shutdown() int r = pwrite(d->fd, d->agg_buffer, d->agg_buf_pos, d->header->write_pos); if (r != d->agg_buf_pos) { - ink_assert(!"flusing agg buffer failed"); + ink_assert(!"flushing agg buffer failed"); continue; } d->header->last_write_pos = d->header->write_pos; diff --git a/iocore/cache/CacheHosting.cc b/iocore/cache/CacheHosting.cc index a57d3bbaf10..5a6a7cec839 100644 --- a/iocore/cache/CacheHosting.cc +++ b/iocore/cache/CacheHosting.cc @@ -100,8 +100,8 @@ CacheHostMatcher::Match(const char *rdata, int rlen, CacheHostResult *result) CacheHostRecord *data_ptr; bool r; - // Check to see if there is any work to do before makeing - // the stirng copy + // Check to see if there is any work to do before making + // the string copy if (num_el <= 0) { return; } @@ -776,7 +776,7 @@ ConfigVolumes::BuildListFromString(char *config_file_path, char *file_buf) return; } -/* Test the cache volumeing with different configurations */ +/* Test the cache volume with different configurations */ #define MEGS_128 (128 * 1024 * 1024) #define ROUND_TO_VOL_SIZE(_x) (((_x) + (MEGS_128 - 1)) & ~(MEGS_128 - 1)) extern CacheDisk **gdisks; @@ -866,7 +866,7 @@ create_config(RegressionTest *t, int num) total_space += vol_blocks; } - // make sure we have atleast 1280 M bytes + // make sure we have at least 1280 M bytes if (total_space<(10 << 27)>> STORE_BLOCK_SHIFT) { rprintf(t, "Not enough space for 10 volume\n"); return 0; diff --git a/iocore/cache/CachePages.cc b/iocore/cache/CachePages.cc index 089cb5908ab..33f62beef5c 100644 --- a/iocore/cache/CachePages.cc +++ b/iocore/cache/CachePages.cc @@ -548,7 +548,7 @@ ShowCache::lookup_regex(int event, Event *e) " form.elements[0].value += urllist[c]+ \"%%0D%%0A\";\n" " }\n" " if (form.elements[0].value == \"\"){\n" - " alert(\"Please select atleast one url before clicking delete\");\n" + " alert(\"Please select at least one url before clicking delete\");\n" " return true;\n" "}\n" " srcfile=\"./delete_url?url=\" + form.elements[0].value;\n" diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index 92179469b73..085ffca4de4 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -838,7 +838,7 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS // an object needs to be outside the aggregation window in order to be // be evacuated as it is read if (!dir_agg_valid(vol, &dir)) { - // a directory entry which is nolonger valid may have been overwritten + // a directory entry which is no longer valid may have been overwritten if (!dir_valid(vol, &dir)) { last_collision = nullptr; } @@ -864,7 +864,7 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS last_collision = nullptr; goto Lread; } - if (!(doc->key == key)) { // collisiion + if (!(doc->key == key)) { // collision goto Lread; } // success @@ -1041,7 +1041,7 @@ CacheVC::openReadStartHead(int event, Event *e) // an object needs to be outside the aggregation window in order to be // be evacuated as it is read if (!dir_agg_valid(vol, &dir)) { - // a directory entry which is nolonger valid may have been overwritten + // a directory entry which is no longer valid may have been overwritten if (!dir_valid(vol, &dir)) { last_collision = nullptr; } @@ -1085,7 +1085,7 @@ CacheVC::openReadStartHead(int event, Event *e) if (buf) { HTTPCacheAlt *alt = reinterpret_cast(doc->hdr()); int32_t alt_length = 0; - // count should be reasonable, as vector is initialized and unlikly to be too corrupted + // count should be reasonable, as vector is initialized and unlikely to be too corrupted // by bad disk data - count should be the number of successfully unmarshalled alts. for (int32_t i = 0; i < vector.count(); ++i) { CacheHTTPInfo *info = vector.get(i); diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index 48d16cb1f2b..55ec3ebd837 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -400,7 +400,7 @@ CacheVC::evacuateReadHead(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */ if (!io.ok()) { goto Ldone; } - // a directory entry which is nolonger valid may have been overwritten + // a directory entry which is no longer valid may have been overwritten if (!dir_valid(vol, &dir)) { last_collision = nullptr; goto Lcollision; @@ -500,7 +500,7 @@ CacheVC::evacuateDocDone(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) dir_overwrite(&doc->key, vol, &dir, &overwrite_dir); } // if the tag in the overwrite_dir matches the first_key in the - // document, then it has to be the vector. We gaurantee that + // document, then it has to be the vector. We guarantee that // the first_key and the earliest_key will never collide (see // Cache::open_write). Once we know its the vector, we can // safely overwrite the first_key in the directory. @@ -677,7 +677,7 @@ Vol::evacuateDocReadDone(int event, Event *e) b->f.unused = 87; } // if the tag in the c->dir does match the first_key in the - // document, then it has to be the earliest fragment. We gaurantee that + // document, then it has to be the earliest fragment. We guarantee that // the first_key and the earliest_key will never collide (see // Cache::open_write). if (!dir_head(&b->dir) || !dir_compare_tag(&b->dir, &doc->first_key)) { diff --git a/iocore/cache/P_CacheDir.h b/iocore/cache/P_CacheDir.h index 576224f167f..58c1c9ffd3c 100644 --- a/iocore/cache/P_CacheDir.h +++ b/iocore/cache/P_CacheDir.h @@ -299,7 +299,7 @@ void sync_cache_dir_on_shutdown(); extern Dir empty_dir; -// Inline Funtions +// Inline Functions #define dir_in_seg(_s, _i) ((Dir *)(((char *)(_s)) + (SIZEOF_DIR * (_i)))) diff --git a/iocore/cache/P_CacheHosting.h b/iocore/cache/P_CacheHosting.h index 765c695ec3f..90822e2c44b 100644 --- a/iocore/cache/P_CacheHosting.h +++ b/iocore/cache/P_CacheHosting.h @@ -98,7 +98,7 @@ class CacheHostMatcher HostLookup *host_lookup; // Data structure to do the lookups CacheHostRecord *data_array; // array of all data items int array_len; // the length of the arrays - int num_el; // the number of itmems in the tree + int num_el; // the number of items in the tree CacheType type; }; diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h index 7ba0e240ca4..1a61b7194c0 100644 --- a/iocore/cache/P_CacheInternal.h +++ b/iocore/cache/P_CacheInternal.h @@ -433,7 +433,7 @@ struct CacheVC : public CacheVConnection { // Start Region C // These variables are memset to 0 when the structure is freed. // The size of this region is size_to_init which is initialized - // in the CacheVC constuctor. It assumes that vio is the start + // in the CacheVC constructor. It assumes that vio is the start // of this region. // NOTE: NOTE: NOTE: If vio is NOT the start, then CHANGE the // size_to_init initialization @@ -578,7 +578,7 @@ free_CacheVC(CacheVC *cont) ink_assert(!cont->is_io_in_progress()); ink_assert(!cont->od); /* calling cont->io.action = nullptr causes compile problem on 2.6 solaris - release build....wierd??? For now, null out continuation and mutex + release build....weird??? For now, null out continuation and mutex of the action separately */ cont->io.action.continuation = nullptr; cont->io.action.mutex = nullptr; diff --git a/iocore/cache/RamCacheCLFUS.cc b/iocore/cache/RamCacheCLFUS.cc index 3fe4af0a416..cca0ee1b73a 100644 --- a/iocore/cache/RamCacheCLFUS.cc +++ b/iocore/cache/RamCacheCLFUS.cc @@ -35,7 +35,7 @@ #endif #define REQUIRED_COMPRESSION 0.9 // must get to this size or declared incompressible -#define REQUIRED_SHRINK 0.8 // must get to this size or keep orignal buffer (with padding) +#define REQUIRED_SHRINK 0.8 // must get to this size or keep original buffer (with padding) #define HISTORY_HYSTERIA 10 // extra temporary history #define ENTRY_OVERHEAD 256 // per-entry overhead to consider when computing cache value/size #define LZMA_BASE_MEMLIMIT (64 * 1024 * 1024) @@ -53,7 +53,7 @@ struct RamCacheCLFUSEntry { uint32_t auxkey1; uint32_t auxkey2; uint64_t hits; - uint32_t size; // memory used including paddding in buffer + uint32_t size; // memory used including padding in buffer uint32_t len; // actual data length uint32_t compressed_len; union { diff --git a/iocore/cache/Store.cc b/iocore/cache/Store.cc index 38deecd286d..778306ac1e5 100644 --- a/iocore/cache/Store.cc +++ b/iocore/cache/Store.cc @@ -46,7 +46,7 @@ make_span_error(int error) switch (error) { case ENOENT: return SPAN_ERROR_NOT_FOUND; - case EPERM: /* fallthru */ + case EPERM: /* fallthrough */ case EACCES: return SPAN_ERROR_NO_ACCESS; default: @@ -213,7 +213,7 @@ Span::errorstr(span_error_t serr) return "unsupported cache file type"; case SPAN_ERROR_MEDIA_PROBE: return "failed to probe device geometry"; - case SPAN_ERROR_UNKNOWN: /* fallthru */ + case SPAN_ERROR_UNKNOWN: /* fallthrough */ default: return "unknown error"; } @@ -652,7 +652,7 @@ void Store::spread_alloc(Store &s, unsigned int blocks, bool mmapable) { // - // Count the eligable disks.. + // Count the eligible disks.. // int mmapable_disks = 0; for (unsigned k = 0; k < n_disks; k++) { @@ -738,7 +738,7 @@ Store::try_realloc(Store &s, Store &diff) } // -// Stupid grab first availabled space allocator +// Stupid grab first available space allocator // void Store::alloc(Store &s, unsigned int blocks, bool one_only, bool mmapable) diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 41173551891..509ae2970e9 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -577,7 +577,7 @@ DNSHandler::startEvent(int /* event ATS_UNUSED */, Event *e) /** Initial state of the DSNHandler. Can reinitialize the running DNS - hander to a new nameserver. + handler to a new nameserver. */ int DNSHandler::startEvent_sdns(int /* event ATS_UNUSED */, Event *e) @@ -1192,7 +1192,7 @@ DNSEntry::mainEvent(int event, Event *e) } else { domains = nullptr; } - Debug("dns", "enqueing query %s", qname); + Debug("dns", "enqueuing query %s", qname); DNSEntry *dup = get_entry(dnsH, qname, qtype); if (dup) { Debug("dns", "collapsing NS request"); diff --git a/iocore/dns/DNSConnection.cc b/iocore/dns/DNSConnection.cc index fc4cb32ff75..5a1408de282 100644 --- a/iocore/dns/DNSConnection.cc +++ b/iocore/dns/DNSConnection.cc @@ -80,7 +80,7 @@ DNSConnection::trigger() // Since the periodic check is removed, we need to call // this when it's triggered by EVENTIO_DNS_CONNECTION. - // The handler should be pionting to DNSHandler::mainEvent. + // The handler should be pointing to DNSHandler::mainEvent. // We can schedule an immediate event or call the handler // directly, and since both arguments are not being used // passing in 0 and nullptr will do the job. diff --git a/iocore/dns/test_P_DNS.cc b/iocore/dns/test_P_DNS.cc index fecca6d0c8f..378fa948ca4 100644 --- a/iocore/dns/test_P_DNS.cc +++ b/iocore/dns/test_P_DNS.cc @@ -55,7 +55,7 @@ struct NetTesterSM : public Continuation { fflush(stdout); break; case VC_EVENT_READ_COMPLETE: - /* FALLSTHROUGH */ + /* FALLTHROUGH */ case VC_EVENT_EOS: r = reader->read_avail(); str = new char[r + 10]; diff --git a/iocore/eventsystem/EventSystem.cc b/iocore/eventsystem/EventSystem.cc index c7f1e99c509..f5d5dc03ae3 100644 --- a/iocore/eventsystem/EventSystem.cc +++ b/iocore/eventsystem/EventSystem.cc @@ -37,7 +37,7 @@ ink_event_system_init(ts::ModuleVersion v) int config_max_iobuffer_size = DEFAULT_MAX_BUFFER_SIZE; int iobuffer_advice = 0; - // For backwards compatability make sure to allow thread_freelist_size + // For backwards compatibility make sure to allow thread_freelist_size // This needs to change in 6.0 REC_EstablishStaticConfigInt32(thread_freelist_high_watermark, "proxy.config.allocator.thread_freelist_size"); diff --git a/iocore/eventsystem/I_Action.h b/iocore/eventsystem/I_Action.h index 444e2115d08..edd9159d5ce 100644 --- a/iocore/eventsystem/I_Action.h +++ b/iocore/eventsystem/I_Action.h @@ -78,7 +78,7 @@ Allocation policy: Actions are allocated by the Processor performing the actions. - It is the processor's responsbility to handle deallocation once + It is the processor's responsibility to handle deallocation once the action is complete or cancelled. A state machine MUST NOT access an action once the operation that returned the Action has completed or it has cancelled the Action. @@ -88,11 +88,11 @@ class Action { public: /** - Contination that initiated this action. + Continuation that initiated this action. The reference to the initiating continuation is only used to verify that the action is being cancelled by the correct - continuation. This field should not be accesed or modified + continuation. This field should not be accessed or modified directly by the state machine. */ @@ -103,7 +103,7 @@ class Action Keeps a reference to the Continuation's lock to preserve the access to the cancelled field valid even when the state machine - has been deallocated. This field should not be accesed or + has been deallocated. This field should not be accessed or modified directly by the state machine. */ @@ -114,7 +114,7 @@ class Action cancelled. This flag is set after a call to cancel or cancel_action and - it should not be accesed or modified directly by the state + it should not be accessed or modified directly by the state machine. */ diff --git a/iocore/eventsystem/I_Continuation.h b/iocore/eventsystem/I_Continuation.h index a7cceb6b5b6..b4531a95415 100644 --- a/iocore/eventsystem/I_Continuation.h +++ b/iocore/eventsystem/I_Continuation.h @@ -23,7 +23,7 @@ @section details Details Continuations have a handleEvent() method to invoke them. Users - can determine the behavior of a Continuation by suppling a + can determine the behavior of a Continuation by supplying a "ContinuationHandler" (member function name) which is invoked when events arrive. This function can be changed with the "setHandler" method. diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index a0518fd24c6..7e86c8e0649 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -56,7 +56,7 @@ enum ThreadType { The EThread class is the type of thread created and managed by the Event System. It is one of the available interfaces for - schedulling events in the event system (another two are the Event + scheduling events in the event system (another two are the Event and EventProcessor classes). In order to handle events, each EThread object has two event @@ -73,7 +73,7 @@ enum ThreadType { Scheduling Interface: - There are eight schedulling functions provided by EThread and + There are eight scheduling functions provided by EThread and they are a wrapper around their counterparts in EventProcessor. @see EventProcessor @@ -121,7 +121,7 @@ class EThread : public Thread continuation's handler. See the the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return Reference to an Event object representing the schedulling + @return Reference to an Event object representing the scheduling of this callback. */ @@ -143,7 +143,7 @@ class EThread : public Thread continuation's handler. See the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -163,7 +163,7 @@ class EThread : public Thread continuation's handler. See the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -177,14 +177,14 @@ class EThread : public Thread to occur every time 'aperiod' elapses. It is scheduled on this EThread. - @param c Continuation to call back everytime 'aperiod' elapses. + @param c Continuation to call back every time 'aperiod' elapses. @param aperiod Duration of the time period between callbacks. @param callback_event Event code to be passed back to the continuation's handler. See the Remarks section in the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -202,7 +202,7 @@ class EThread : public Thread continuation's handler. See the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -223,7 +223,7 @@ class EThread : public Thread continuation's handler. See the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -244,7 +244,7 @@ class EThread : public Thread EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -257,14 +257,14 @@ class EThread : public Thread Schedules the callback to the continuation 'c' to occur every time 'aperiod' elapses. It is scheduled on this EThread. - @param c Continuation to call back everytime 'aperiod' elapses. + @param c Continuation to call back every time 'aperiod' elapses. @param aperiod Duration of the time period between callbacks. @param callback_event Event code to be passed back to the continuation's handler. See the Remarks section in the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ diff --git a/iocore/eventsystem/I_Event.h b/iocore/eventsystem/I_Event.h index 3f708db0f16..e598adda288 100644 --- a/iocore/eventsystem/I_Event.h +++ b/iocore/eventsystem/I_Event.h @@ -166,7 +166,7 @@ class Event : public Action Instructs the event object to reschedule itself at the time specified in atimeout_at on the EventProcessor. - @param atimeout_at Time at which to callcallback. See the Remarks section. + @param atimeout_at Time at which to call the callback. See the Remarks section. @param callback_event Event code to return at the completion of this event. See the Remarks section. */ @@ -177,7 +177,7 @@ class Event : public Action Instructs the event object to reschedule itself at the time specified in atimeout_at on the EventProcessor. - @param atimeout_in Time at which to callcallback. See the Remarks section. + @param atimeout_in Time at which to call the callback. See the Remarks section. @param callback_event Event code to return at the completion of this event. See the Remarks section. */ @@ -188,7 +188,7 @@ class Event : public Action the event object to reschedule itself to callback every 'aperiod' from now. - @param aperiod Time period at which to callcallback. See the Remarks section. + @param aperiod Time period at which to call the callback. See the Remarks section. @param callback_event Event code to return at the completion of this event. See the Remarks section. */ diff --git a/iocore/eventsystem/I_EventProcessor.h b/iocore/eventsystem/I_EventProcessor.h index 2bf90103109..031ca71d2ac 100644 --- a/iocore/eventsystem/I_EventProcessor.h +++ b/iocore/eventsystem/I_EventProcessor.h @@ -73,8 +73,8 @@ class EThread; thread is independent of the thread groups and it exists as long as your continuation handle executes and there are events to process. In the latter, you call @c registerEventType to get an event type and then @c spawn_event_theads which creates the threads in the group of that - type. Such threads require events to be scheduled on a specicif thread in the group or for the - grouop in general using the event type. Note that between these two calls @c + type. Such threads require events to be scheduled on a specific thread in the group or for the + group in general using the event type. Note that between these two calls @c EThread::schedule_spawn can be used to set up per thread initialization. Callback event codes: @@ -204,11 +204,11 @@ class EventProcessor : public Processor /** Schedules the continuation on a specific thread group to receive an event periodically. Requests the EventProcessor to schedule the - callback to the continuation 'c' everytime 'aperiod' elapses. The + callback to the continuation 'c' every time 'aperiod' elapses. The callback is handled by a thread in the specified thread group (event_type). - @param c Continuation to call back everytime 'aperiod' elapses. + @param c Continuation to call back every time 'aperiod' elapses. @param aperiod duration of the time period between callbacks. @param event_type thread group id (or event type) specifying the group of threads on which to schedule the callback. diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h index 88a54cf9d35..90b599a4732 100644 --- a/iocore/eventsystem/I_IOBuffer.h +++ b/iocore/eventsystem/I_IOBuffer.h @@ -29,7 +29,7 @@ might wish to ensure that an entire line will come in before consuming the data. In such a case, the water_mark should be set to the largest possible size of the string. (appropriate error handling should take - care of exessively long strings). + care of excessively long strings). In all other cases, especially when all data will be consumed, the water_mark should be set to 0 (the default). @@ -418,7 +418,7 @@ class IOBufferBlock : public RefCountObj Create a copy of the IOBufferBlock. Creates and returns a copy of this IOBufferBlock that references the same data that this IOBufferBlock (it does not allocate an another buffer). The cloned block will not - have a writable space since the original IOBufferBlock mantains the + have a writable space since the original IOBufferBlock maintains the ownership for writing data to the block. @return copy of this IOBufferBlock. @@ -715,7 +715,7 @@ class IOBufferReader /** Clears all fields in this IOBuffeReader, rendering it unusable. Drops - the reference to the IOBufferBlock list, the accesor, MIOBuffer and + the reference to the IOBufferBlock list, the accessor, MIOBuffer and resets this reader's state. You have to set those fields in order to use this object again. @@ -820,7 +820,7 @@ class IOBufferReader @param offset number of the bytes to skip over before beginning the operation. @return -1 if c is not found, otherwise position of the first - ocurrence. + occurrence. */ inkcoreapi int64_t memchr(char c, int64_t len = INT64_MAX, int64_t offset = 0); @@ -1082,7 +1082,7 @@ class MIOBuffer /** Returns the amount of space of available for writing on the first - writable block on the block chain (the one that would be reutrned + writable block on the block chain (the one that would be returned by first_write_block()). */ diff --git a/iocore/eventsystem/I_Lock.h b/iocore/eventsystem/I_Lock.h index 00177253de0..c061d266b16 100644 --- a/iocore/eventsystem/I_Lock.h +++ b/iocore/eventsystem/I_Lock.h @@ -129,7 +129,7 @@ inkcoreapi extern void lock_taken(const SourceLocation &, const char *handler); A ProxyMutex object has an ink_mutex member (defined in ink_mutex.h) which is a wrapper around the platform dependent mutex type. This - member allows the ProxyMutex to provide the functionallity required + member allows the ProxyMutex to provide the functionality required by the users of the class without the burden of platform specific function calls. @@ -234,7 +234,7 @@ class ProxyMutex : public RefCountObj } }; -// The ClassAlocator for ProxyMutexes +// The ClassAllocator for ProxyMutexes extern inkcoreapi ClassAllocator mutexAllocator; inline bool diff --git a/iocore/eventsystem/I_Processor.h b/iocore/eventsystem/I_Processor.h index cce9a01cf64..291c7386d3b 100644 --- a/iocore/eventsystem/I_Processor.h +++ b/iocore/eventsystem/I_Processor.h @@ -41,7 +41,7 @@ class Thread; processors in the IO Core. A processor is multithreaded subsystem specialized in some type of task or application. For example, the Event System module includes the EventProcessor which provides - schedulling services, the Net module includes the NetProcessor + scheduling services, the Net module includes the NetProcessor which provides networking services, etc. You cannot create objects of the Processor class and its methods @@ -76,7 +76,7 @@ class Processor virtual int get_thread_count(); /** - This function attemps to stop the processor. Please refer to + This function attempts to stop the processor. Please refer to the documentation on each processor to determine if it is supported. diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index e2c46b3391c..54cb9b0ee0b 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -147,7 +147,7 @@ class Thread This gets a cached copy of the time so it is very fast and reasonably accurate. The cached time is updated every time the actual operating system time is fetched which is at least every 10ms and generally more frequently. - @note The cached copy shared among threads which means the cached copy is udpated + @note The cached copy shared among threads which means the cached copy is updated for all threads if any thread updates it. */ static ink_hrtime get_hrtime(); diff --git a/iocore/eventsystem/I_VConnection.h b/iocore/eventsystem/I_VConnection.h index bc72ba5fe9b..193c62bd3d4 100644 --- a/iocore/eventsystem/I_VConnection.h +++ b/iocore/eventsystem/I_VConnection.h @@ -54,7 +54,7 @@ static constexpr int TS_VCONN_MAX_USER_ARG = 4; #define VC_EVENT_READ_READY VC_EVENT_EVENTS_START /** - Any data in the accociated buffer *will be written* when the + Any data in the associated buffer *will be written* when the Continuation returns. */ @@ -73,7 +73,7 @@ static constexpr int TS_VCONN_MAX_USER_ARG = 4; #define VC_EVENT_ERROR EVENT_ERROR /** - VC_EVENT_INACTIVITY_TIMEOUT indiates that the operation (read or write) has: + VC_EVENT_INACTIVITY_TIMEOUT indicates that the operation (read or write) has: -# been enabled for more than the inactivity timeout period (for a read, there has been space in the buffer) (for a write, there has been data in the buffer) @@ -85,7 +85,7 @@ static constexpr int TS_VCONN_MAX_USER_ARG = 4; #define VC_EVENT_INACTIVITY_TIMEOUT (VC_EVENT_EVENTS_START + 5) /** - Total time for some operation has been exeeded, regardless of any + Total time for some operation has been exceeded, regardless of any intermediate progress. */ @@ -101,10 +101,10 @@ static constexpr int TS_VCONN_MAX_USER_ARG = 4; // VC_EVENT_READ_READ occurs when data *has been written* into // the associated buffer. // -// VC_EVENT_ERROR indicates that some error has occured. The +// VC_EVENT_ERROR indicates that some error has occurred. The // "data" will be either 0 if the errno is unavailable or errno. // -// VC_EVENT_INTERVAL indidates that an interval timer has expired. +// VC_EVENT_INTERVAL indicates that an interval timer has expired. // // @@ -260,7 +260,7 @@ class VConnection : public Continuation must call this function to indicate that the VConnection can be deallocated. After a close has been called, the VConnection and underlying processor must not send any more events related - to this VConnection to the state machine. Likeswise, the state + to this VConnection to the state machine. Likewise, the state machine must not access the VConnection or any VIOs obtained from it after calling this method. @@ -335,7 +335,7 @@ class VConnection : public Continuation @param id Identifier associated to interpret the data field @param data Value or pointer with state machine or VConnection data. - @return True if the oparation is successful. + @return True if the operation is successful. */ virtual bool @@ -356,7 +356,7 @@ class VConnection : public Continuation @param id Identifier associated to interpret the data field. @param data Value or pointer with state machine or VConnection data. - @return True if the oparation is successful. + @return True if the operation is successful. */ virtual bool diff --git a/iocore/eventsystem/P_Freer.h b/iocore/eventsystem/P_Freer.h index c620c573d31..374af33622b 100644 --- a/iocore/eventsystem/P_Freer.h +++ b/iocore/eventsystem/P_Freer.h @@ -27,7 +27,7 @@ #include "I_Tasks.h" // Note that these should not be used for memory that wishes to retain -// NUMA socket affinity. We'll potentially return these on an arbitarily +// NUMA socket affinity. We'll potentially return these on an arbitrarily // selected processor/socket. template struct DeleterContinuation : public Continuation { diff --git a/iocore/eventsystem/P_IOBuffer.h b/iocore/eventsystem/P_IOBuffer.h index 4968ff90e39..c13202d0259 100644 --- a/iocore/eventsystem/P_IOBuffer.h +++ b/iocore/eventsystem/P_IOBuffer.h @@ -709,7 +709,7 @@ TS_INLINE char &IOBufferReader::operator[](int64_t i) } ink_release_assert(!"out of range"); - // Never used, just to satisfy compilers not undersatnding the fatality of ink_release_assert(). + // Never used, just to satisfy compilers not understanding the fatality of ink_release_assert(). return default_ret; } diff --git a/iocore/eventsystem/Thread.cc b/iocore/eventsystem/Thread.cc index e239bf3a5b1..90faf9d8c56 100644 --- a/iocore/eventsystem/Thread.cc +++ b/iocore/eventsystem/Thread.cc @@ -73,7 +73,7 @@ Thread::~Thread() /////////////////////////////////////////////// struct thread_data_internal { - ThreadFunction f; ///< Function to excecute in the thread. + ThreadFunction f; ///< Function to execute in the thread. Thread *me; ///< The class instance. char name[MAX_THREAD_NAME_LENGTH]; ///< Name for the thread. }; diff --git a/iocore/eventsystem/UnixEThread.cc b/iocore/eventsystem/UnixEThread.cc index d0f896d3412..a57932295a5 100644 --- a/iocore/eventsystem/UnixEThread.cc +++ b/iocore/eventsystem/UnixEThread.cc @@ -128,7 +128,7 @@ EThread::process_event(Event *e, int calling_code) return; } Continuation *c_temp = e->continuation; - // Make sure that the contination is locked before calling the handler + // Make sure that the continuation is locked before calling the handler e->continuation->handleEvent(calling_code, e); ink_assert(!e->in_the_priority_queue); ink_assert(c_temp == e->continuation); diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 9f4110a83ac..179a040a5ad 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -392,7 +392,7 @@ HostDBProcessor::start(int, size_t) statPagesManager.register_http("hostdb", register_ShowHostDB); // - // Register configuration callback, and establish configuation links + // Register configuration callback, and establish configuration links // REC_EstablishStaticConfigInt32(hostdb_ttl_mode, "proxy.config.hostdb.ttl_mode"); REC_EstablishStaticConfigInt32(hostdb_disable_reverse_lookup, "proxy.config.cache.hostdb.disable_reverse_lookup"); @@ -897,7 +897,7 @@ HostDBProcessor::setby(const char *hostname, int len, sockaddr const *ip, HostDB } return; } - // Create a continuation to do a deaper probe in the background + // Create a continuation to do a deeper probe in the background HostDBContinuation *c = hostDBContAllocator.alloc(); c->init(hash); @@ -920,7 +920,7 @@ HostDBProcessor::setby_srv(const char *hostname, int len, const char *target, Ho hash.db_mark = HOSTDB_MARK_SRV; hash.refresh(); - // Create a continuation to do a deaper probe in the background + // Create a continuation to do a deeper probe in the background HostDBContinuation *c = hostDBContAllocator.alloc(); c->init(hash); @@ -1414,7 +1414,7 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) } // We have seen cases were the action.mutex != action.continuation.mutex. - // Since reply_to_cont will call the hanlder on the action.continuation, it is important that we hold + // Since reply_to_cont will call the handler on the action.continuation, it is important that we hold // that mutex. bool need_to_reschedule = true; MUTEX_TRY_LOCK(lock, action.mutex, thread); @@ -1559,7 +1559,7 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) reply_to_cont(action.continuation, r.get()); } - // If it suceeds or it was a remote probe, we are done + // If it succeeds or it was a remote probe, we are done // if (r) { hostdb_cont_free(this); @@ -1830,7 +1830,7 @@ struct ShowHostDB : public ShowCont { if (event == EVENT_INTERVAL) { HostDBInfo *r = reinterpret_cast(e); if (output_json && records_seen++ > 0) { - CHECK_SHOW(show(",")); // we need to seperate records + CHECK_SHOW(show(",")); // we need to separate records } showOne(r, false, event, e); if (r->round_robin) { @@ -1852,7 +1852,7 @@ struct ShowHostDB : public ShowCont { for (int i = 0; i < rr_data->rrcount; i++) { showOne(&rr_data->info(i), true, event, e, rr_data); if (output_json) { - CHECK_SHOW(show("}")); // we need to seperate records + CHECK_SHOW(show("}")); // we need to separate records if (i < (rr_data->rrcount - 1)) CHECK_SHOW(show(",")); } @@ -2333,7 +2333,7 @@ struct HostDBRegressionContinuation : public Continuation { hostDBProcessor.getbyname_re(this, hostnames[i++], 0); return EVENT_CONT; } else { - rprintf(test, "HostDBTestRR: %d outstanding %d succcess %d failure\n", outstanding, success, failure); + rprintf(test, "HostDBTestRR: %d outstanding %d success %d failure\n", outstanding, success, failure); if (success == hosts) { *status = REGRESSION_TEST_PASSED; } else { diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h index 25871315f97..3d15faf8423 100644 --- a/iocore/hostdb/I_HostDBProcessor.h +++ b/iocore/hostdb/I_HostDBProcessor.h @@ -51,7 +51,7 @@ struct HostDBContinuation; // IP address. // // Since host information is relatively small, we can afford to have -// a reasonable size memory cache, and use a (relatively) sparce +// a reasonable size memory cache, and use a (relatively) sparse // disk representation to decrease # of seeks. // extern int hostdb_enable; @@ -84,7 +84,7 @@ makeHostHash(const char *string) // // This structure contains the host information required by // the application. Except for the initial fields it -// is treated as opacque by the database. +// is treated as opaque by the database. // union HostDBApplicationInfo { @@ -395,13 +395,13 @@ struct HostDBRoundRobin { struct HostDBCache; struct HostDBHash; -// Prototype for inline completion functionf or +// Prototype for inline completion function or // getbyname_imm() typedef void (Continuation::*cb_process_result_pfn)(HostDBInfo *r); Action *iterate(Continuation *cont); -/** The Host Databse access interface. */ +/** The Host Database access interface. */ struct HostDBProcessor : public Processor { friend struct HostDBSync; // Public Interface diff --git a/iocore/hostdb/P_RefCountCacheSerializer.h b/iocore/hostdb/P_RefCountCacheSerializer.h index 41b80b5867f..e76b7513564 100644 --- a/iocore/hostdb/P_RefCountCacheSerializer.h +++ b/iocore/hostdb/P_RefCountCacheSerializer.h @@ -292,7 +292,7 @@ RefCountCacheSerializer::finalize_sync() return error; } - // Don't bother checking for errors on the close since theere's nothing we can do about it at + // Don't bother checking for errors on the close since there's nothing we can do about it at // this point anyway. socketManager.close(dirfd); socketManager.close(this->fd); diff --git a/iocore/hostdb/test_P_HostDB.cc b/iocore/hostdb/test_P_HostDB.cc index 2768b43d96d..7609c865d45 100644 --- a/iocore/hostdb/test_P_HostDB.cc +++ b/iocore/hostdb/test_P_HostDB.cc @@ -56,7 +56,7 @@ struct NetTesterSM : public Continuation { fflush(stdout); break; case VC_EVENT_READ_COMPLETE: - /* FALLSTHROUGH */ + /* FALLTHROUGH */ case VC_EVENT_EOS: r = reader->read_avail(); str = new char[r + 10]; diff --git a/iocore/net/I_Net.h b/iocore/net/I_Net.h index 81ed6c3edbe..050777c3875 100644 --- a/iocore/net/I_Net.h +++ b/iocore/net/I_Net.h @@ -22,9 +22,9 @@ @section details Details - Net subsystem is a layer on top the operations sytem network apis. It + Net subsystem is a layer on top the operations system network apis. It provides an interface for accepting/creating new connection oriented - (TCP) and connection less (UDP) connetions and for reading/writing + (TCP) and connection less (UDP) connections and for reading/writing data through these. The net system can manage 1000s of connections very efficiently. Another advantage of using the net system is that the SMs dont have be concerned about differences in the net apis of diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index 570e7f3496f..5f94cd639b9 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -240,7 +240,7 @@ class NetProcessor : public Processor object. @code - netProcesors.accept(my_cont, ...); + netProcessor.accept(my_cont, ...); netProcessor.connect_re(my_cont, ...); @endcode diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index e1e95044777..6f40159b30b 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -258,13 +258,13 @@ struct NetVCOptions { if (&that != this) { /* * It is odd but necessary to null the scoped string pointer here - * and then explicitly call release on them in the string assignements + * and then explicitly call release on them in the string assignments * below. * We a memcpy from that to this. This will put that's string pointers into * this's memory. Therefore we must first explicitly null out * this's original version of the string. The release after the * memcpy removes the extra reference to that's copy of the string - * Removing the release will eventualy cause a double free crash + * Removing the release will eventually cause a double free crash */ sni_servername = nullptr; // release any current name. ssl_servername = nullptr; @@ -358,7 +358,7 @@ class NetVConnection : public AnnotatedVConnection c->handleEvent(VC_EVENT_ERROR, vio) - signified that error occured during write. + signified that error occurred during write. @@ -368,7 +368,7 @@ class NetVConnection : public AnnotatedVConnection when it is destroyed. @param c continuation to be called back after (partial) write - @param nbytes no of bytes to write, if unknown msut be set to INT64_MAX + @param nbytes no of bytes to write, if unknown must be set to INT64_MAX @param buf source of data @param owner @return vio pointer @@ -378,11 +378,11 @@ class NetVConnection : public AnnotatedVConnection /** Closes the vconnection. A state machine MUST call do_io_close() - when it has finished with a VConenction. do_io_close() indicates + when it has finished with a VConnection. do_io_close() indicates that the VConnection can be deallocated. After a close has been called, the VConnection and underlying processor must NOT send any more events related to this VConnection to the state machine. - Likeswise, state machine must not access the VConnectuion or + Likewise, state machine must not access the VConnection or any returned VIOs after calling close. lerrno indicates whether a close is a normal close or an abort. The difference between a normal close and an abort depends on the underlying type of @@ -401,7 +401,7 @@ class NetVConnection : public AnnotatedVConnection IO_SHUTDOWN_READWRITE. Once a side of a VConnection is shutdown, no further I/O can be done on that side of the connections and the underlying processor MUST NOT send any further events - (INCLUDING TIMOUT EVENTS) to the state machine. The state machine + (INCLUDING TIMEOUT EVENTS) to the state machine. The state machine MUST NOT use any VIOs from a shutdown side of a connection. Even if both sides of a connection are shutdown, the state machine MUST still call do_io_close() when it wishes the @@ -438,7 +438,7 @@ class NetVConnection : public AnnotatedVConnection //////////////////////////////////////////////////////////// // Set the timeouts associated with this connection. // - // active_timeout is for the total elasped time of // + // active_timeout is for the total elapsed time of // // the connection. // // inactivity_timeout is the elapsed time from the time // // a read or a write was scheduled during which the // @@ -461,7 +461,7 @@ class NetVConnection : public AnnotatedVConnection that it does not keep any connections open for a really long time. - Timeout symantics: + Timeout semantics: Should a timeout occur, the state machine for the read side of the NetVConnection is signaled first assuming that a read has diff --git a/iocore/net/I_UDPNet.h b/iocore/net/I_UDPNet.h index 9e4667dc38c..846c3a78e07 100644 --- a/iocore/net/I_UDPNet.h +++ b/iocore/net/I_UDPNet.h @@ -46,7 +46,7 @@ class UDPNetProcessor : public Processor public: int start(int n_upd_threads, size_t stacksize) override = 0; - // this function was interanal intially.. this is required for public and + // this function was internal initially.. this is required for public and // interface probably should change. bool CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action **status, NetVCOptions &opt); @@ -69,7 +69,7 @@ class UDPNetProcessor : public Processor @param recv_bufsize (optional) Socket buffer size for sending. Limits how much can be queued by OS before we read it. @return Action* Always returns ACTION_RESULT_DONE if socket was - created successfuly, or ACTION_IO_ERROR if not. + created successfully, or ACTION_IO_ERROR if not. */ inkcoreapi Action *UDPBind(Continuation *c, sockaddr const *addr, int send_bufsize = 0, int recv_bufsize = 0); diff --git a/iocore/net/NetVCTest.cc b/iocore/net/NetVCTest.cc index bda22bd7390..41ac801d068 100644 --- a/iocore/net/NetVCTest.cc +++ b/iocore/net/NetVCTest.cc @@ -26,7 +26,7 @@ NetVCTest.cc Description: - Unit test for infastructure for VConnections implementing the + Unit test for infrastructure for VConnections implementing the NetVConnection interface @@ -67,7 +67,7 @@ NVC_test_def netvc_tests_def[] = { {"itimeout", 6000, 8000, 10, 10, 512, 10, VC_EVENT_READ_COMPLETE, VC_EVENT_INACTIVITY_TIMEOUT}, {"itimeout", 10, 10, 6000, 8000, 512, 20, VC_EVENT_EOS, VC_EVENT_WRITE_COMPLETE}, - // Test the small transfer code one byts at a time + // Test the small transfer code one byte at a time {"smallt", 400, 400, 500, 500, 1, 15, VC_EVENT_READ_COMPLETE, VC_EVENT_WRITE_COMPLETE}, {"smallt", 500, 500, 400, 400, 1, 15, VC_EVENT_READ_COMPLETE, VC_EVENT_WRITE_COMPLETE}, diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index 2ebae671673..5a3121eeb8a 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -29,7 +29,7 @@ #include "P_SSLUtils.h" #include "SSLStats.h" -// Maxiumum OCSP stapling response size. +// Maximum OCSP stapling response size. // This should be the response for a single certificate and will typically include the responder certificate chain, // so 10K should be more than enough. #define MAX_STAPLING_DER 10240 diff --git a/iocore/net/P_Connection.h b/iocore/net/P_Connection.h index 9741fd7c37d..8c8d4ce80ac 100644 --- a/iocore/net/P_Connection.h +++ b/iocore/net/P_Connection.h @@ -42,7 +42,7 @@ The accept call is a blocking call while connect is non-blocking. They returns a new Connection instance which is an handle to the newly created connection. The connection `q instance can be used later for read/writes - using an intance of IOProcessor class. + using an instance of IOProcessor class. **************************************************************************/ @@ -143,7 +143,7 @@ struct Connection { private: // Don't want copy constructors to avoid having the deconstructor on - // temporarly copies close the file descriptor too soon. Use move instead + // temporarily copies close the file descriptor too soon. Use move instead Connection(Connection const &); protected: diff --git a/iocore/net/P_NetVCTest.h b/iocore/net/P_NetVCTest.h index eedbb8d2319..f98193218dd 100644 --- a/iocore/net/P_NetVCTest.h +++ b/iocore/net/P_NetVCTest.h @@ -26,7 +26,7 @@ P_NetVCTest.h Description: - Unit test for infastructure for VConnections implementing the + Unit test for infrastructure for VConnections implementing the NetVConnection interface diff --git a/iocore/net/P_NetVConnection.h b/iocore/net/P_NetVConnection.h index 3d85e7a1d35..679cf0b10ed 100644 --- a/iocore/net/P_NetVConnection.h +++ b/iocore/net/P_NetVConnection.h @@ -40,7 +40,7 @@ NetVConnection::get_remote_addr() inline IpEndpoint const & NetVConnection::get_remote_endpoint() { - get_remote_addr(); // Make sure the vallue is filled in + get_remote_addr(); // Make sure the value is filled in return remote_addr; } diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index c0dc504f17d..87a4a33e0ad 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -46,7 +46,7 @@ struct ssl_ticket_key_block; // // struct SSLConfigParams // -// configuration parameters as they apear in the global +// configuration parameters as they appear in the global // configuration file. ///////////////////////////////////////////////////////////// diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index 1be6d86dbdb..be2d896ef8d 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -193,7 +193,7 @@ class PacketQueue UDPPacketInternal *p; // pull in all the stuff from long-term slot lastPullLongTermQ = t; - // this is to handle wierdoness where someone is trying to queue a + // this is to handle weirdness where someone is trying to queue a // packet to be sent in SLOT_TIME_MSEC * N_SLOTS * (2+)---the packet // will get back to longTermQ and we'll have an infinite loop. while ((p = longTermQ.dequeue()) != nullptr) diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h index 0440b7238dc..7d591b01f9c 100644 --- a/iocore/net/P_UnixNet.h +++ b/iocore/net/P_UnixNet.h @@ -343,7 +343,7 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler /** Release a netvc and free it. - @param netvc UnixNetVConnection to be deattached. + @param netvc UnixNetVConnection to be detached. */ void free_netvc(UnixNetVConnection *netvc); @@ -353,7 +353,7 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler void _close_vc(UnixNetVConnection *vc, ink_hrtime now, int &handle_event, int &closed, int &total_idle_time, int &total_idle_count); - /// Static method used as the callbackf for runtime configuration updates. + /// Static method used as the callback for runtime configuration updates. static int update_nethandler_config(const char *name, RecDataT, RecData data, void *); }; diff --git a/iocore/net/P_UnixNetProcessor.h b/iocore/net/P_UnixNetProcessor.h index b076c0ae00f..5c19df4f8d5 100644 --- a/iocore/net/P_UnixNetProcessor.h +++ b/iocore/net/P_UnixNetProcessor.h @@ -66,7 +66,7 @@ extern UnixNetProcessor unix_netProcessor; // // Set up a thread to receive events from the NetProcessor // This function should be called for all threads created to -// accept such events by the EventProcesor. +// accept such events by the EventProcessor. // extern void initialize_thread_for_net(EThread *thread); diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 46111bcad92..e4e0483239f 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -142,7 +142,7 @@ class UnixNetVConnection : public NetVConnection //////////////////////////////////////////////////////////// // Set the timeouts associated with this connection. // - // active_timeout is for the total elasped time of // + // active_timeout is for the total elapsed time of // // the connection. // // inactivity_timeout is the elapsed time from the time // // a read or a write was scheduled during which the // diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index ca61712e4ca..aa82eb08c44 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -514,7 +514,7 @@ SSLCertificateConfig::reconfigure() // twice the healthcheck period to simulate a loading a large certificate set. if (is_action_tag_set("test.multicert.delay")) { const int secs = 60; - Debug("ssl", "delaying certificate reload by %dsecs", secs); + Debug("ssl", "delaying certificate reload by %d secs", secs); ink_hrtime_sleep(HRTIME_SECONDS(secs)); } diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 070e4141ab2..e4c9fbf4402 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -371,7 +371,7 @@ SSLNetVConnection::read_raw_data() // If we have already moved some bytes successfully, adjust total_read to reflect reality // If any read succeeded, we should return success if (r != rattempted) { - // If the first read failds, we should return error + // If the first read fails, we should return error if (r <= 0 && total_read > rattempted) { r = total_read - rattempted; } else { @@ -408,7 +408,7 @@ SSLNetVConnection::read_raw_data() } else { Debug("proxyprotocol", "[SSLNetVConnection::read_raw_data] proxy protocol DOES NOT have a configured whitelist of trusted IPs but " - "proxy protocol is ernabled on this port - processing all connections"); + "proxy protocol is enabled on this port - processing all connections"); } if (ssl_has_proxy_v1(this, buffer, &r)) { @@ -1215,7 +1215,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // Clean up the epoll entry for signalling SSL_clear_mode(ssl, SSL_MODE_ASYNC); this->ep.stop(); - // Rectivate the socket, ready to rock + // Reactivate the socket, ready to rock PollDescriptor *pd = get_PollDescriptor(this_ethread()); this->ep.start( pd, this, @@ -1550,7 +1550,7 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) // Reenabling from the handshake callback // - // Originally, we would wait for the callback to go again to execute additinonal + // Originally, we would wait for the callback to go again to execute additional // hooks, but since the callbacks are associated with the context and the context // can be replaced by the plugin, it didn't seem reasonable to assume that the // callback would be executed again. So we walk through the rest of the hooks diff --git a/iocore/net/SSLNextProtocolSet.cc b/iocore/net/SSLNextProtocolSet.cc index 2adefce1e0c..04c6330e3e0 100644 --- a/iocore/net/SSLNextProtocolSet.cc +++ b/iocore/net/SSLNextProtocolSet.cc @@ -29,7 +29,7 @@ // For currently defined protocol strings, see // http://technotes.googlecode.com/git/nextprotoneg.html. The OpenSSL // documentation tells us to return a string in "wire format". The -// draft NPN RFC helpfuly refuses to document the wire format. The +// draft NPN RFC helpfully refuses to document the wire format. The // above link says we need to send length-prefixed strings, but does // not say how many bytes the length is. For the record, it's 1. @@ -78,7 +78,7 @@ create_npn_advertisement(const SSLNextProtocolSet::NextProtocolEndpoint::list_ty return false; } -// copies th eprotocols but not the endpoints +// copies the protocols but not the endpoints SSLNextProtocolSet * SSLNextProtocolSet::clone() const diff --git a/iocore/net/SSLSessionTicket.cc b/iocore/net/SSLSessionTicket.cc index 07eea13914d..9346b29b655 100644 --- a/iocore/net/SSLSessionTicket.cc +++ b/iocore/net/SSLSessionTicket.cc @@ -31,7 +31,7 @@ #include "SSLStats.h" #include "P_SSLConfig.h" -// Remvoe this when drop OpenSSL 1.0.2 support +// Remove this when drop OpenSSL 1.0.2 support #ifndef evp_md_func #ifdef OPENSSL_NO_SHA256 #define evp_md_func EVP_sha1() diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index d2e2e97ae2b..1e390fda57a 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -382,7 +382,7 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) const char *servername = nullptr; const unsigned char *p; size_t remaining, len; - // Parse the servrer name if the get extension call succeeds and there are more than 2 bytes to parse + // Parse the server name if the get extension call succeeds and there are more than 2 bytes to parse if (SSL_client_hello_get0_ext(s, TLSEXT_TYPE_server_name, &p, &remaining) && remaining > 2) { // Parse to get to the name, originally from test/handshake_helper.c in openssl tree /* Extract the length of the supplied list of names. */ @@ -979,7 +979,7 @@ SSLMultiCertConfigLoader::check_server_cert_now(X509 *cert, const char *certname timeCmpValue = X509_cmp_current_time(X509_get_notBefore(cert)); if (timeCmpValue == 0) { - // an error occured parsing the time, which we'll call a bogosity + // an error occurred parsing the time, which we'll call a bogosity Error("invalid certificate %s: unable to parse notBefore time", certname); return -3; } else if (timeCmpValue > 0) { @@ -990,7 +990,7 @@ SSLMultiCertConfigLoader::check_server_cert_now(X509 *cert, const char *certname timeCmpValue = X509_cmp_current_time(X509_get_notAfter(cert)); if (timeCmpValue == 0) { - // an error occured parsing the time, which we'll call a bogosity + // an error occurred parsing the time, which we'll call a bogosity Error("invalid certificate %s: unable to parse notAfter time", certname); return -3; } else if (timeCmpValue < 0) { @@ -1762,7 +1762,7 @@ SSLConnect(SSL *ssl) } /** - Load certificats to SSL_CTX + Load certificates to SSL_CTX @static */ bool diff --git a/iocore/net/Socks.cc b/iocore/net/Socks.cc index a1a59999b96..ab1044f5bf9 100644 --- a/iocore/net/Socks.cc +++ b/iocore/net/Socks.cc @@ -480,7 +480,7 @@ loadSocksConfiguration(socks_conf_struct *socks_conf_stuff) socks_conf_stuff->server_connect_timeout = REC_ConfigReadInteger("proxy.config.socks.server_connect_timeout"); socks_conf_stuff->socks_timeout = REC_ConfigReadInteger("proxy.config.socks.socks_timeout"); - Debug("Socks", "server connect timeout: %d socks respnonse timeout %d", socks_conf_stuff->server_connect_timeout, + Debug("Socks", "server connect timeout: %d socks response timeout %d", socks_conf_stuff->server_connect_timeout, socks_conf_stuff->socks_timeout); socks_conf_stuff->per_server_connection_attempts = REC_ConfigReadInteger("proxy.config.socks.per_server_connection_attempts"); @@ -636,7 +636,7 @@ socks5BasicAuthHandler(int event, unsigned char *p, void (**h_ptr)(void)) break; case 0xff: - Debug("Socks", "None of the Socks authentcations is acceptable " + Debug("Socks", "None of the Socks authentications is acceptable " "to the server"); *h_ptr = nullptr; ret = -1; @@ -655,7 +655,7 @@ socks5BasicAuthHandler(int event, unsigned char *p, void (**h_ptr)(void)) break; default: - // This should be inpossible + // This should be impossible ink_assert(!"bad case value"); ret = -1; break; diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc index 76a10a46620..63721b96154 100644 --- a/iocore/net/UnixNet.cc +++ b/iocore/net/UnixNet.cc @@ -152,7 +152,7 @@ PollCont::do_poll(ink_hrtime timeout) poll_timeout = net_config_poll_timeout; } } -// wait for fd's to tigger, or don't wait if timeout is 0 +// wait for fd's to trigger, or don't wait if timeout is 0 #if TS_USE_EPOLL pollDescriptor->result = epoll_wait(pollDescriptor->epoll_fd, pollDescriptor->ePoll_Triggered_Events, POLL_DESCRIPTOR_SIZE, poll_timeout); diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 456fa95dda6..5ecc216b304 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -1377,7 +1377,7 @@ UnixNetVConnection::migrateToCurrentThread(Continuation *cont, EThread *t) // Try to get the mutex lock for NetHandler of this NetVC MUTEX_TRY_LOCK(lock_src, this->nh->mutex, t); if (lock_src.is_locked()) { - // Deattach this NetVC from original NetHandler & InactivityCop. + // Detach this NetVC from original NetHandler & InactivityCop. this->nh->stopCop(this); this->nh->stopIO(this); // Put this NetVC into current NetHandler & InactivityCop. diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc index 6ba226e3580..8fbb4d81320 100644 --- a/iocore/net/YamlSNIConfig.cc +++ b/iocore/net/YamlSNIConfig.cc @@ -147,7 +147,7 @@ template <> struct convert { } // remove before 9.0.0 release - // backwards compatibiity + // backwards compatibility if (node[TS_verify_origin_server]) { auto value = node[TS_verify_origin_server].as(); YamlSNIConfig::Level level = static_cast(LEVEL_DESCRIPTOR.get(value)); diff --git a/iocore/net/test_P_Net.cc b/iocore/net/test_P_Net.cc index f56dd19d2c5..185b1586e65 100644 --- a/iocore/net/test_P_Net.cc +++ b/iocore/net/test_P_Net.cc @@ -55,7 +55,7 @@ struct NetTesterSM : public Continuation { fflush(stdout); break; case VC_EVENT_READ_COMPLETE: - /* FALLSTHROUGH */ + /* FALLTHROUGH */ case VC_EVENT_EOS: r = reader->read_avail(); str = new char[r + 10]; diff --git a/iocore/net/test_certlookup.cc b/iocore/net/test_certlookup.cc index 31913b6593f..96a8f362328 100644 --- a/iocore/net/test_certlookup.cc +++ b/iocore/net/test_certlookup.cc @@ -168,7 +168,7 @@ load_hostnames_csv(const char *fname, SSLCertLookup &lookup) SSLCertContext ctx_cc(ctx); // The input should have 2 comma-separated fields; this is the format that you get when - // you download the top 1M sites from alexa. + // you download the top 1M sites from Alexa. // // For example: // 1,google.com diff --git a/iocore/utils/I_OneWayMultiTunnel.h b/iocore/utils/I_OneWayMultiTunnel.h index 9351851e1a7..c4973e6ffe1 100644 --- a/iocore/utils/I_OneWayMultiTunnel.h +++ b/iocore/utils/I_OneWayMultiTunnel.h @@ -36,7 +36,7 @@ #define ONE_WAY_MULTI_TUNNEL_LIMIT 4 /** - A generic state machine that connects a source virtual conection to + A generic state machine that connects a source virtual connection to multiple target virtual connections. A OneWayMultiTunnel is similar to the OneWayTunnel module. However, instead of connection one source to one target, it connects multiple virtual connections - a source diff --git a/iocore/utils/I_OneWayTunnel.h b/iocore/utils/I_OneWayTunnel.h index 8a835641107..d74401a88ab 100644 --- a/iocore/utils/I_OneWayTunnel.h +++ b/iocore/utils/I_OneWayTunnel.h @@ -45,7 +45,7 @@ typedef void (*Transform_fn)(MIOBufferAccessor &in_buf, MIOBufferAccessor &out_buf); /** - A generic state machine that connects two virtual conections. A + A generic state machine that connects two virtual connections. A OneWayTunnel is a module that connects two virtual connections, a source vc and a target vc, and copies the data between source and target. Once the tunnel is started using the init() call, it handles all the events From ed7c2001d8f9f689d0853db7e80078d59ba22c4e Mon Sep 17 00:00:00 2001 From: Bill Chen Date: Thu, 25 Apr 2019 08:03:35 -0700 Subject: [PATCH 503/526] cppcheck: Fix issues found in I_IOBuffer.h --- iocore/eventsystem/I_IOBuffer.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h index 90b599a4732..1742016c220 100644 --- a/iocore/eventsystem/I_IOBuffer.h +++ b/iocore/eventsystem/I_IOBuffer.h @@ -585,6 +585,7 @@ class IOBufferChain const_iterator() = default; ///< Default constructor. /// Copy constructor. + // cppcheck-suppress noExplicitConstructor; copy constructor const_iterator(self_type const &that); /// Assignment. @@ -1298,6 +1299,7 @@ class MIOBuffer #endif MIOBuffer(void *b, int64_t bufsize, int64_t aWater_mark); + // cppcheck-suppress noExplicitConstructor; allow implicit conversion MIOBuffer(int64_t default_size_index); MIOBuffer(); ~MIOBuffer(); @@ -1372,7 +1374,7 @@ class MIOBuffer_tracker const char *loc; public: - MIOBuffer_tracker(const char *_loc) : loc(_loc) {} + explicit MIOBuffer_tracker(const char *_loc) : loc(_loc) {} MIOBuffer * operator()(int64_t size_index = default_large_iobuffer_size) { @@ -1393,7 +1395,7 @@ class Empty_MIOBuffer_tracker const char *loc; public: - Empty_MIOBuffer_tracker(const char *_loc) : loc(_loc) {} + explicit Empty_MIOBuffer_tracker(const char *_loc) : loc(_loc) {} MIOBuffer * operator()(int64_t size_index = default_large_iobuffer_size) { @@ -1431,7 +1433,7 @@ class IOBufferBlock_tracker const char *loc; public: - IOBufferBlock_tracker(const char *_loc) : loc(_loc) {} + explicit IOBufferBlock_tracker(const char *_loc) : loc(_loc) {} IOBufferBlock * operator()() { @@ -1477,7 +1479,7 @@ class IOBufferData_tracker const char *loc; public: - IOBufferData_tracker(const char *_loc) : loc(_loc) {} + explicit IOBufferData_tracker(const char *_loc) : loc(_loc) {} IOBufferData * operator()(int64_t size_index = default_large_iobuffer_size, AllocType type = DEFAULT_ALLOC) { From c3ef1210b44c59a0ad7d9874c39b81b512780be5 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Thu, 26 Jul 2018 13:04:53 -0700 Subject: [PATCH 504/526] Remove commented out includes --- example/cppapi/globalhook/GlobalHookPlugin.cc | 2 +- include/tscore/CryptoHash.h | 2 -- iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc | 1 - iocore/net/P_SNIActionPerformer.h | 1 - iocore/net/P_UnixNetProcessor.h | 2 -- iocore/net/test_I_Net.cc | 1 - proxy/http/Http1ClientSession.h | 1 - src/traffic_server/EventName.cc | 1 - src/tscore/unit_tests/test_Scalar.cc | 4 +--- 9 files changed, 2 insertions(+), 13 deletions(-) diff --git a/example/cppapi/globalhook/GlobalHookPlugin.cc b/example/cppapi/globalhook/GlobalHookPlugin.cc index 9fd245e491f..e8acbe9d660 100644 --- a/example/cppapi/globalhook/GlobalHookPlugin.cc +++ b/example/cppapi/globalhook/GlobalHookPlugin.cc @@ -17,9 +17,9 @@ */ #include + #include "tscpp/api/GlobalPlugin.h" #include "tscpp/api/PluginInit.h" -//#include<../ts/Diags.h> using namespace atscppapi; using namespace std; diff --git a/include/tscore/CryptoHash.h b/include/tscore/CryptoHash.h index 6fe34e1bc2f..1cb7848ca30 100644 --- a/include/tscore/CryptoHash.h +++ b/include/tscore/CryptoHash.h @@ -28,10 +28,8 @@ /// Apache Traffic Server commons. #if TS_ENABLE_FIPS == 1 -// #include "tscore/SHA256.h" #define CRYPTO_HASH_SIZE (256 / 8) #else -// #include "tscore/ink_code.h" #define CRYPTO_HASH_SIZE (128 / 8) #endif #define CRYPTO_HEX_SIZE ((CRYPTO_HASH_SIZE * 2) + 1) diff --git a/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc b/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc index 759a3e9b22b..955bea04db5 100644 --- a/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc +++ b/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc @@ -29,7 +29,6 @@ #include "I_EventSystem.h" #include "tscore/I_Layout.h" -//#include "tscore/ink_string.h" #include "diags.i" #include "I_MIOBufferWriter.h" diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 4c27c73e90c..957ff3c055a 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -31,7 +31,6 @@ #pragma once #include "I_EventSystem.h" -//#include"P_UnixNetProcessor.h" #include #include "P_SSLNextProtocolAccept.h" #include "tscore/ink_inet.h" diff --git a/iocore/net/P_UnixNetProcessor.h b/iocore/net/P_UnixNetProcessor.h index 5c19df4f8d5..7cd2208d452 100644 --- a/iocore/net/P_UnixNetProcessor.h +++ b/iocore/net/P_UnixNetProcessor.h @@ -69,5 +69,3 @@ extern UnixNetProcessor unix_netProcessor; // accept such events by the EventProcessor. // extern void initialize_thread_for_net(EThread *thread); - -//#include "UnixNet.h" diff --git a/iocore/net/test_I_Net.cc b/iocore/net/test_I_Net.cc index 9b6a79e18f9..9961bc00b8e 100644 --- a/iocore/net/test_I_Net.cc +++ b/iocore/net/test_I_Net.cc @@ -29,7 +29,6 @@ /* * Choose a net test application */ -//#include "NetTest-http-server.c" #include "NetTest-simple-proxy.c" int diff --git a/proxy/http/Http1ClientSession.h b/proxy/http/Http1ClientSession.h index 74d7a99ff4c..3fafd2d451a 100644 --- a/proxy/http/Http1ClientSession.h +++ b/proxy/http/Http1ClientSession.h @@ -31,7 +31,6 @@ #pragma once -//#include "libts.h" #include "P_Net.h" #include "InkAPIInternal.h" #include "HTTP.h" diff --git a/src/traffic_server/EventName.cc b/src/traffic_server/EventName.cc index e45be296351..1eb9b58ec74 100644 --- a/src/traffic_server/EventName.cc +++ b/src/traffic_server/EventName.cc @@ -26,7 +26,6 @@ #include #include "P_EventSystem.h" -// #include "I_Disk.h" unused #include "I_Cache.h" #include "I_Net.h" #include "I_HostDB.h" diff --git a/src/tscore/unit_tests/test_Scalar.cc b/src/tscore/unit_tests/test_Scalar.cc index 6e8956054ac..d38c2e80119 100644 --- a/src/tscore/unit_tests/test_Scalar.cc +++ b/src/tscore/unit_tests/test_Scalar.cc @@ -22,10 +22,8 @@ */ #include + #include "tscore/Scalar.h" -//#include -//#include -//#include using Bytes = ts::Scalar<1, off_t>; using Paragraphs = ts::Scalar<16, off_t>; From efe823c6811a375c57749e89f0d5487897c21263 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Tue, 30 Apr 2019 07:18:51 -0700 Subject: [PATCH 505/526] Fixes spelling in miscellaneous files in configs, contrib, doc, example and tools --- README | 6 +++--- ci/docker/yum/Dockerfile | 2 +- configs/logging.yaml.default | 2 +- contrib/install_trafficserver.sh | 2 +- contrib/python/compare_RecordsConfigcc.py | 2 +- doc/admin-guide/plugins/prefetch.en.rst | 2 +- doc/developer-guide/core-architecture/rpc.en.rst | 2 +- doc/uml/RPC-sequence-diagram.uml | 4 ++-- doc/uml/RPC-states.uml | 2 +- example/response_header_1/response_header_1.c | 2 +- example/thread_pool/psi.c | 2 +- example/thread_pool/test/SDKTest/psi_server.c | 2 +- tools/compare_servers.pl | 2 +- tools/http_load/http_load.c | 2 +- tools/http_load/merge_stats.pl | 2 +- tools/jtest/README.zh.md | 4 ++-- tools/jtest/jtest.cc | 8 ++++---- tools/traffic_primer | 4 ++-- 18 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README b/README index 1a5f5457f3b..707c767040f 100644 --- a/README +++ b/README @@ -11,13 +11,13 @@ plugins to build large scale web applications. |-- ci/ ................... Quality assurance and other CI tools and configs |-- configs/ .............. Configurations |-- contrib/ .............. Various contributed auxiliary pieces - |-- doc/ .................. Documentations for Traffic Server + |-- doc/ .................. Documentation for Traffic Server |-- admin-guide/ ...... Admin guide documentations |-- appendices/ ....... Appendices of Traffic Server - |-- developer-guide/ .. Documentaions for developers + |-- developer-guide/ .. Documentation for developers |-- dot/ .............. Graphviz source files for docs pictures |-- static/ ........... Static resources - |-- uml/ .............. Documentations in UML + |-- uml/ .............. Documentation in UML |-- example/ .............. Example plugins |-- iocore/ ............... |-- aio/ .............. Asynchronous I/O core diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index eeff9ca0fce..774fa26c881 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -21,7 +21,7 @@ # limitations under the License. ################################################################################ -# These can (should?) be overriden from the command line with ----build-arg, e.g. +# These can (should?) be overridden from the command line with ----build-arg, e.g. # docker build --build-arg OS_VERSION=7 --build-arg OS_TYPE=centos # ARG OS_VERSION=28 diff --git a/configs/logging.yaml.default b/configs/logging.yaml.default index ea593ee4ad9..da116d6758e 100644 --- a/configs/logging.yaml.default +++ b/configs/logging.yaml.default @@ -3,7 +3,7 @@ # Documentation on logging: # https://docs.trafficserver.apache.org/en/8.0.x/admin-guide/logging/index.en.html # -# Documentaion on logging.yaml file format: +# Documentation on logging.yaml file format: # https://docs.trafficserver.apache.org/en/8.0.x/admin-guide/files/logging.yaml.en.html # # Example log configurations: diff --git a/contrib/install_trafficserver.sh b/contrib/install_trafficserver.sh index e8269434ffe..4223efa9b50 100644 --- a/contrib/install_trafficserver.sh +++ b/contrib/install_trafficserver.sh @@ -136,7 +136,7 @@ function dev() { git clone git://git.apache.org/trafficserver.git cd $EC2_EPHEMERAL/$trafficserver - #swtich to dev build + #switch to dev build git checkout -b remotes/origin/dev #------------------------------------------------------ } diff --git a/contrib/python/compare_RecordsConfigcc.py b/contrib/python/compare_RecordsConfigcc.py index d41ac6d2036..b01a133c963 100644 --- a/contrib/python/compare_RecordsConfigcc.py +++ b/contrib/python/compare_RecordsConfigcc.py @@ -65,7 +65,7 @@ if m: rc_in[m.group(1)] = (m.group(2), m.group(3)) -# Process records.comfig documentation. +# Process records.config documentation. # eg. .. ts:cv:: CONFIG proxy.config.proxy_binary STRING traffic_server with open("%s/doc/admin-guide/files/records.config.en.rst" % src_dir) as fh: doc_re = re.compile(r'ts:cv:: CONFIG (\S+)\s+(\S+)\s+(\S+)') diff --git a/doc/admin-guide/plugins/prefetch.en.rst b/doc/admin-guide/plugins/prefetch.en.rst index 93a7e3cf582..9b3b8c69c98 100644 --- a/doc/admin-guide/plugins/prefetch.en.rst +++ b/doc/admin-guide/plugins/prefetch.en.rst @@ -212,7 +212,7 @@ compromises: object** would be cached as well. * **Don't fetch the response body** and **never cache** at the **front-tier**. The **front-tier** marks the prefetch request with a special API header defined - by ``--api-header`` plugin parameter. When recieved the **back-tier** responds + by ``--api-header`` plugin parameter. When received the **back-tier** responds right away before actually fetching the object (without a body), it just schedules the real prefetch at the **back-tier**. ``Cache-Control: no-store`` is used to make sure the prefetch request response is never cached at the **front-tier**. diff --git a/doc/developer-guide/core-architecture/rpc.en.rst b/doc/developer-guide/core-architecture/rpc.en.rst index 0b5cdeaa3f6..1e43989c406 100644 --- a/doc/developer-guide/core-architecture/rpc.en.rst +++ b/doc/developer-guide/core-architecture/rpc.en.rst @@ -63,7 +63,7 @@ Runtime Structure Message Passing =============== -Sequence diagram for a command sent from |TCtl| to when it is recieved by a plugin. +Sequence diagram for a command sent from |TCtl| to when it is received by a plugin. .. figure:: ../../uml/images/RPC-sequence-diagram.svg diff --git a/doc/uml/RPC-sequence-diagram.uml b/doc/uml/RPC-sequence-diagram.uml index 17d543dbe9a..5a5c0d3d33a 100644 --- a/doc/uml/RPC-sequence-diagram.uml +++ b/doc/uml/RPC-sequence-diagram.uml @@ -38,8 +38,8 @@ traffic_server -[#blue]-> local_rpc : //unserialize message// traffic_server -> plugin : invoke hook and attach ""TSPluginMsg"" plugin -> plugin : hooks ""TS_LIFECYCLE_MSG_HOOK""\nand reads ""TSPluginMsg"" traffic_manager -[#green]-> remote_rpc : TS_ERR_OK -note right : traffic_manager has no idea if\nplugin actually recieved message +note right : traffic_manager has no idea if\nplugin actually received message remote_rpc -> traffic_ctl : parse response -@enduml \ No newline at end of file +@enduml diff --git a/doc/uml/RPC-states.uml b/doc/uml/RPC-states.uml index ad73b58935c..c95ade5c634 100644 --- a/doc/uml/RPC-states.uml +++ b/doc/uml/RPC-states.uml @@ -20,7 +20,7 @@ state "LocalManager\nruns in traffic_manager main event loop" as traffic_manager state "poll socket" as tm_poll_sock : ""mgmt_select"" state "new connection" as tm_new_process : register new client socket fd\nto start getting msgs from\nclient state "new message" as tm_new_message : new message from process - state "handle msg" as tm_handle_msg : ""handleMgmtMsgFromProcesses"" will\noften ""signalAlarm"" with recieved\nmsg_id + state "handle msg" as tm_handle_msg : ""handleMgmtMsgFromProcesses"" will\noften ""signalAlarm"" with received\nmsg_id tm_poll_sock -d-> tm_new_process : new process trying to connect tm_poll_sock -d-> tm_new_message : msg from ""traffic_server"" diff --git a/example/response_header_1/response_header_1.c b/example/response_header_1/response_header_1.c index 65f173a8b5d..9a0430f5438 100644 --- a/example/response_header_1/response_header_1.c +++ b/example/response_header_1/response_header_1.c @@ -195,7 +195,7 @@ modify_header(TSHttpTxn txnp) * Additional 200/304 processing can go here, if so desired. */ - /* Caller reneables */ + /* Caller reenables */ } static int diff --git a/example/thread_pool/psi.c b/example/thread_pool/psi.c index c21eefc3cd7..4793c699cdd 100644 --- a/example/thread_pool/psi.c +++ b/example/thread_pool/psi.c @@ -438,7 +438,7 @@ _basename(const char *filename) data continuation for the current transaction Output : data->psi_buffer contains the file content - data->psi_sucess 0 if include failed, 1 if success + data->psi_success 0 if include failed, 1 if success Return Value: 0 if failure 1 if success diff --git a/example/thread_pool/test/SDKTest/psi_server.c b/example/thread_pool/test/SDKTest/psi_server.c index 71d21244ce8..cfa209ef0c1 100644 --- a/example/thread_pool/test/SDKTest/psi_server.c +++ b/example/thread_pool/test/SDKTest/psi_server.c @@ -29,7 +29,7 @@ * - PSI header * - PSI include in body * - * Ratio for generating PSI response is specifid in config file. + * Ratio for generating PSI response is specified in config file. * * Added Options in Synth_server.config - * psi_ratio : percentage of response with psi embedded we want to generate diff --git a/tools/compare_servers.pl b/tools/compare_servers.pl index 4c485da6c24..a7bfaaa534b 100755 --- a/tools/compare_servers.pl +++ b/tools/compare_servers.pl @@ -79,7 +79,7 @@ ($$) my @test_headers = qw(ETag Cache-Control Connection Accept-Ranges Server Content-Type Access-Control-Allow-Methods Access-Control-Allow-Origin Strict-Transport-Security); - my $return_val = 0; # header valuse match + my $return_val = 0; # header value match if ($verbose >= 3) { foreach my $field ($response1->header_field_names) { diff --git a/tools/http_load/http_load.c b/tools/http_load/http_load.c index 9268f7432c6..da8c74c6bff 100644 --- a/tools/http_load/http_load.c +++ b/tools/http_load/http_load.c @@ -2846,7 +2846,7 @@ close_connection(int cnum) url_num = connections[cnum].url_num; /* Only check to update got_bytes, byte count errors and/or checksums - if the request was succesful (i.e. no HTTP error). */ + if the request was successful (i.e. no HTTP error). */ if (connections[cnum].http_status >= 0 && connections[cnum].http_status < 400) { if (do_checksum) { if (!urls[url_num].got_checksum) { diff --git a/tools/http_load/merge_stats.pl b/tools/http_load/merge_stats.pl index db406004907..317460bb5f4 100644 --- a/tools/http_load/merge_stats.pl +++ b/tools/http_load/merge_stats.pl @@ -66,7 +66,7 @@ } print "Total runs: ", $runs, "\n"; -printf "%d fetches on %d conns, %d max parallell, %.5e bytes in %d seconds\n", +printf "%d fetches on %d conns, %d max parallel, %.5e bytes in %d seconds\n", $fetches, $conns, $parallel, $bytes, $seconds / $runs; print $mean_bytes/ $runs, " mean bytes/fetch\n"; printf "%.2f fetches/sec, %.5e bytes/sec\n", $fetches_sec, $bytes_sec; diff --git a/tools/jtest/README.zh.md b/tools/jtest/README.zh.md index c7d43f4e7cf..286ad9e8b7c 100644 --- a/tools/jtest/README.zh.md +++ b/tools/jtest/README.zh.md @@ -190,8 +190,8 @@ hash是jtest、ats里无处不在的，如何让hash互相影响，甚至测试h -N, --alternates int 0 Number of Alternates -e, --client_rate int 0 Clients Per Sec -o, --abort_retry_speed int 0 Abort/Retry Speed - - , --abort_retry_bytes int 0 Abort/Retry Threshhold (bytes) - - , --abort_retry_secs int 5 Abort/Retry Threshhold (secs) + - , --abort_retry_bytes int 0 Abort/Retry Threshold (bytes) + - , --abort_retry_secs int 5 Abort/Retry Threshold (secs) -W, --reload_rate dbl 0.000 Reload Rate * -D，用于生成url的随机数，如果有多个jtest并发运行，可以对这个随机的seed进行区分以控制cache的多小等 diff --git a/tools/jtest/jtest.cc b/tools/jtest/jtest.cc index 6f612aa1ff0..da22fe1202a 100644 --- a/tools/jtest/jtest.cc +++ b/tools/jtest/jtest.cc @@ -92,7 +92,7 @@ #define MAX_BUFSIZE (65536 + 4096) // -// Contants +// Constants // #define MAXFDS 65536 #define HEADER_DONE -1 @@ -294,7 +294,7 @@ static const ArgumentDescription argument_descriptions[] = { {"alternates", 'N', "Number of Alternates", "I", &alternates, "JTEST_ALTERNATES", nullptr}, {"client_rate", 'e', "Clients Per Sec", "I", &client_rate, "JTEST_CLIENT_RATE", nullptr}, {"abort_retry_speed", 'o', "Abort/Retry Speed", "I", &abort_retry_speed, "JTEST_ABORT_RETRY_SPEED", nullptr}, - {"abort_retry_bytes", ' ', "Abort/Retry Threshhold (bytes)", "I", &abort_retry_bytes, "JTEST_ABORT_RETRY_THRESHHOLD_BYTES", + {"abort_retry_bytes", ' ', "Abort/Retry Threshold (bytes)", "I", &abort_retry_bytes, "JTEST_ABORT_RETRY_THRESHHOLD_BYTES", nullptr}, {"abort_retry_secs", ' ', "Abort/Retry Threshhold (secs)", "I", &abort_retry_secs, "JTEST_ABORT_RETRY_THRESHHOLD_SECS", nullptr}, {"reload_rate", 'W', "Reload Rate", "D", &reload_rate, "JTEST_RELOAD_RATE", nullptr}, @@ -2523,7 +2523,7 @@ read_response(int sock) } if (check_content && !cl) { if (verbose || verbose_errors) { - printf("missiing Content-Length '%s'\n", fd[sock].base_url); + printf("missing Content-Length '%s'\n", fd[sock].base_url); } return read_response_error(sock); } @@ -4280,7 +4280,7 @@ ink_web_canonicalize_url(const char *base_url, const char *emb_url, char *dest_u } else { use_base_host = 1; - /* step 4 - if emb_path preceeded by slash, skip to 7 */ + /* step 4 - if emb_path preceded by slash, skip to 7 */ if (emb.leading_slash != 1) { /* step 5 */ diff --git a/tools/traffic_primer b/tools/traffic_primer index 4c3dc4c0663..2d73c9d79ad 100755 --- a/tools/traffic_primer +++ b/tools/traffic_primer @@ -18,7 +18,7 @@ # Simple script to fetch a URL through one proxy, and then PUSH that response (headers # and body) to a set of hosts. The host:port defaults to localhost:80 for fetching -# the URL, but can be overriden with -h/-p. +# the URL, but can be overridden with -h/-p. # Default values for command line options @@ -72,7 +72,7 @@ push() { if [ $? -gt 0 ]; then echo -e "\tError: ${h}:${PORT} is down" else - echo -en "\tProcesing ${h}:${PORT}..." + echo -en "\tProcessing ${h}:${PORT}..." # PURGE the object first if we -f (force) it if [ $FORCE -gt 0 ]; then echo -n " purged..." From 8e451b7749fec61361d6038b15a91db508cd8a24 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Thu, 25 Apr 2019 17:03:16 +0800 Subject: [PATCH 506/526] Fixes spelling in plugins --- plugins/authproxy/authproxy.cc | 2 +- plugins/background_fetch/background_fetch.cc | 4 ++-- plugins/background_fetch/configs.cc | 2 +- plugins/cachekey/README.md | 2 +- plugins/cachekey/configs.h | 2 +- plugins/cachekey/pattern.cc | 8 ++++---- plugins/cachekey/pattern.h | 2 +- plugins/compress/compress.cc | 6 +++--- plugins/conf_remap/conf_remap.cc | 2 +- plugins/esi/README.combo | 2 +- plugins/esi/esi.cc | 2 +- plugins/esi/fetcher/FetchedDataProcessor.h | 2 +- plugins/esi/lib/EsiGunzip.cc | 2 +- plugins/esi/lib/EsiGzip.cc | 2 +- plugins/esi/lib/EsiProcessor.cc | 8 ++++---- plugins/esi/lib/FailureInfo.h | 6 +++--- plugins/esi/lib/Stats.cc | 4 ++-- plugins/esi/test/sampleProb.cc | 2 +- plugins/experimental/access_control/config.h | 2 +- plugins/experimental/access_control/pattern.cc | 2 +- plugins/experimental/acme/acme.c | 2 +- plugins/experimental/balancer/hash.cc | 2 +- plugins/experimental/buffer_upload/README | 6 +++--- plugins/experimental/cache_range_requests/README | 2 +- .../cache_range_requests/cache_range_requests.cc | 4 ++-- plugins/experimental/certifier/certifier.cc | 6 +++--- plugins/experimental/cookie_remap/cookie_remap.cc | 2 +- plugins/experimental/cookie_remap/cookiejar.cc | 2 +- plugins/experimental/cookie_remap/strip.h | 4 ++-- plugins/experimental/custom_redirect/README | 2 +- plugins/experimental/geoip_acl/README | 4 ++-- plugins/experimental/geoip_acl/acl.cc | 2 +- plugins/experimental/inliner/README | 2 +- plugins/experimental/inliner/ats-inliner.cc | 4 ++-- plugins/experimental/inliner/cache.cc | 2 +- plugins/experimental/inliner/cache.h | 2 +- plugins/experimental/inliner/fetcher.h | 2 +- plugins/experimental/inliner/ts.cc | 2 +- plugins/experimental/inliner/vconnection.h | 2 +- .../experimental/ja3_fingerprint/ja3_fingerprint.cc | 2 +- plugins/experimental/magick/magick.cc | 2 +- plugins/experimental/magick/sign.sh | 2 +- plugins/experimental/magick/test.sh | 2 +- plugins/experimental/magick/verify.sh | 2 +- plugins/experimental/memcache/protocol_binary.h | 2 +- plugins/experimental/metalink/README | 2 +- plugins/experimental/money_trace/README | 4 ++-- plugins/experimental/money_trace/money_trace.cc | 2 +- plugins/experimental/mp4/mp4_meta.cc | 4 ++-- plugins/experimental/mp4/mp4_meta.h | 4 ++-- plugins/experimental/multiplexer/dispatch.cc | 2 +- plugins/experimental/multiplexer/fetcher.h | 2 +- plugins/experimental/prefetch/fetch.h | 2 +- plugins/experimental/prefetch/pattern.cc | 2 +- plugins/experimental/prefetch/plugin.cc | 6 +++--- plugins/experimental/slice/slice.cc | 2 +- plugins/experimental/slice/slice_test.cc | 2 +- plugins/experimental/sslheaders/sslheaders.cc | 4 ++-- plugins/experimental/sslheaders/sslheaders.h | 2 +- plugins/experimental/stream_editor/stream_editor.cc | 8 ++++---- plugins/experimental/system_stats/system_stats.c | 2 +- plugins/experimental/url_sig/README | 2 +- plugins/experimental/url_sig/url_sig.c | 4 ++-- plugins/generator/generator.cc | 2 +- plugins/header_rewrite/conditions.cc | 4 ++-- plugins/header_rewrite/parser.cc | 2 +- plugins/header_rewrite/resources.cc | 2 +- plugins/healthchecks/healthchecks.c | 6 +++--- plugins/lua/ts_lua_cached_response.c | 2 +- plugins/lua/ts_lua_client_request.c | 2 +- plugins/lua/ts_lua_client_response.c | 2 +- plugins/lua/ts_lua_server_request.c | 2 +- plugins/lua/ts_lua_server_response.c | 2 +- plugins/s3_auth/aws_auth_v4.h | 2 +- plugins/s3_auth/aws_auth_v4_wrap.h | 2 +- plugins/s3_auth/s3_auth.cc | 10 +++++----- plugins/s3_auth/unit_tests/test_aws_auth_v4.cc | 2 +- plugins/tcpinfo/tcpinfo.cc | 4 ++-- plugins/xdebug/xdebug_headers.cc | 6 +++--- plugins/xdebug/xdebug_transforms.cc | 4 ++-- 80 files changed, 121 insertions(+), 121 deletions(-) diff --git a/plugins/authproxy/authproxy.cc b/plugins/authproxy/authproxy.cc index 80b4567363f..da8d8c86a45 100644 --- a/plugins/authproxy/authproxy.cc +++ b/plugins/authproxy/authproxy.cc @@ -656,7 +656,7 @@ AuthProxyGlobalHook(TSCont /* cont ATS_UNUSED */, TSEvent event, void *edata) auth->txn = txn; return AuthRequestContext::dispatch(auth->cont, event, edata); } - // fallthru + // fallthrough default: return TS_EVENT_NONE; diff --git a/plugins/background_fetch/background_fetch.cc b/plugins/background_fetch/background_fetch.cc index f6b99c9c208..6ad921661bd 100644 --- a/plugins/background_fetch/background_fetch.cc +++ b/plugins/background_fetch/background_fetch.cc @@ -220,7 +220,7 @@ struct BgFetchData { // This needs the txnp temporarily, so it can copy the pristine request // URL. The txnp is not used once initialize() returns. // -// Upon succesful completion, the struct should be ready to start a +// Upon successful completion, the struct should be ready to start a // background fetch. bool BgFetchData::initialize(TSMBuffer request, TSMLoc req_hdr, TSHttpTxn txnp) @@ -678,7 +678,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo * /* rri */) if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &bufp, &req_hdrs)) { TSMLoc field_loc = TSMimeHdrFieldFind(bufp, req_hdrs, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE); - if (!field_loc) { // Less common case, but also allow If-Range header to triger, but only if Range not present + if (!field_loc) { // Less common case, but also allow If-Range header to trigger, but only if Range not present field_loc = TSMimeHdrFieldFind(bufp, req_hdrs, TS_MIME_FIELD_IF_RANGE, TS_MIME_LEN_IF_RANGE); } diff --git a/plugins/background_fetch/configs.cc b/plugins/background_fetch/configs.cc index 479317b7a9a..d5283b45062 100644 --- a/plugins/background_fetch/configs.cc +++ b/plugins/background_fetch/configs.cc @@ -72,7 +72,7 @@ BgFetchConfig::parseOptions(int argc, const char *argv[]) return true; } -// Read a config file, populare the linked list (chain the BgFetchRule's) +// Read a config file, populate the linked list (chain the BgFetchRule's) bool BgFetchConfig::readConfig(const char *config_file) { diff --git a/plugins/cachekey/README.md b/plugins/cachekey/README.md index 91dc7252e69..6cc59898bbe 100644 --- a/plugins/cachekey/README.md +++ b/plugins/cachekey/README.md @@ -1,7 +1,7 @@ # Description This plugin allows some common cache key manipulations based on various HTTP request elements. It can -* sort query parameters to prevent query parameters reordereding from being a cache miss +* sort query parameters to prevent query parameters reordering from being a cache miss * ignore specific query parameters from the cache key by name or regular expression * ignore all query parameters from the cache key * only use specific query parameters in the cache key by name or regular expression diff --git a/plugins/cachekey/configs.h b/plugins/cachekey/configs.h index 89a67af31fe..947b21931ea 100644 --- a/plugins/cachekey/configs.h +++ b/plugins/cachekey/configs.h @@ -148,7 +148,7 @@ class Configs /** * @brief provides means for post-processing of the plugin parameters to finalize the configuration or to "cache" some of the * decisions for later use. - * @return true if succesful, false if failure. + * @return true if successful, false if failure. */ bool finalize(); diff --git a/plugins/cachekey/pattern.cc b/plugins/cachekey/pattern.cc index 3672b692800..c93490291da 100644 --- a/plugins/cachekey/pattern.cc +++ b/plugins/cachekey/pattern.cc @@ -47,18 +47,18 @@ Pattern::Pattern() : _pattern(""), _replacement("") {} * @return true if successful, false if failure */ bool -Pattern::init(const String &pattern, const String &replacenemt, bool replace) +Pattern::init(const String &pattern, const String &replacement, bool replace) { pcreFree(); _pattern.assign(pattern); - _replacement.assign(replacenemt); + _replacement.assign(replacement); _replace = replace; _tokenCount = 0; if (!compile()) { - CacheKeyDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacenemt.c_str()); + CacheKeyDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacement.c_str()); pcreFree(); return false; } @@ -151,7 +151,7 @@ Pattern::pcreFree() } /** - * @bried Destructor, frees PCRE related resources. + * @brief Destructor, frees PCRE related resources. */ Pattern::~Pattern() { diff --git a/plugins/cachekey/pattern.h b/plugins/cachekey/pattern.h index 749d243f10b..2b36fd70884 100644 --- a/plugins/cachekey/pattern.h +++ b/plugins/cachekey/pattern.h @@ -45,7 +45,7 @@ class Pattern Pattern(); virtual ~Pattern(); - bool init(const String &pattern, const String &replacenemt, bool replace); + bool init(const String &pattern, const String &replacement, bool replace); bool init(const String &config); bool empty() const; bool match(const String &subject); diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 0e748cc50b1..f1be86d9a50 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -40,7 +40,7 @@ using namespace std; using namespace Gzip; // FIXME: custom dictionaries would be nice. configurable/content-type? -// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith +// a GPRS device might benefit from a higher compression ratio, whereas a desktop w. high bandwith // might be served better with little or no compression at all // FIXME: look into compressing from the task thread pool // FIXME: make normalizing accept encoding configurable @@ -136,7 +136,7 @@ data_destroy(Data *data) { TSReleaseAssert(data); - // deflateEnd returnvalue ignore is intentional + // deflateEnd return value ignore is intentional // it would spew log on every client abort deflateEnd(&data->zstrm); @@ -429,7 +429,7 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) { gzip_transform_one(data, upstream_buffer, upstream_length); } else { - warning("No compression supported. Shoudn't come here."); + warning("No compression supported. Shouldn't come here."); } TSIOBufferReaderConsume(upstream_reader, upstream_length); diff --git a/plugins/conf_remap/conf_remap.cc b/plugins/conf_remap/conf_remap.cc index 736b36208ec..34b6198440a 100644 --- a/plugins/conf_remap/conf_remap.cc +++ b/plugins/conf_remap/conf_remap.cc @@ -49,7 +49,7 @@ struct RemapConfigs { int _current = 0; }; -// Helper functionfor the parser +// Helper function for the parser inline TSRecordDataType str_to_datatype(const char *str) { diff --git a/plugins/esi/README.combo b/plugins/esi/README.combo index e4466ab039c..1cd7001fa28 100644 --- a/plugins/esi/README.combo +++ b/plugins/esi/README.combo @@ -12,7 +12,7 @@ The arguments in the plugin.config line in order represent 2) The name of the key used for signature verification (disabled by default) [verification not implemented yet] -3) A colon separated list of headers which, if present on atleast one +3) A colon separated list of headers which, if present on at least one response, will be added to the combo response. A "-" can be supplied as a value for any of these arguments to request diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc index db00073057a..101161834dd 100644 --- a/plugins/esi/esi.cc +++ b/plugins/esi/esi.cc @@ -1537,7 +1537,7 @@ globalHookHandler(TSCont contp, TSEvent event, void *edata) TSDebug(DEBUG_TAG, "[%s] handling cache lookup complete event", __FUNCTION__); if (isCacheObjTransformable(txnp, &intercept_header, &head_only)) { // we make the assumption above that a transformable cache - // object would already have a tranformation. We should revisit + // object would already have a transformation. We should revisit // that assumption in case we change the statement below addTransform(txnp, false, intercept_header, head_only, pOptionInfo); Stats::increment(Stats::N_CACHE_DOCS); diff --git a/plugins/esi/fetcher/FetchedDataProcessor.h b/plugins/esi/fetcher/FetchedDataProcessor.h index c30ac7d745c..9a9fd436cbf 100644 --- a/plugins/esi/fetcher/FetchedDataProcessor.h +++ b/plugins/esi/fetcher/FetchedDataProcessor.h @@ -28,7 +28,7 @@ class FetchedDataProcessor public: FetchedDataProcessor(){}; - virtual void processData(const char *reqeust_url, int request_url_len, const char *response_data, int response_data_len) = 0; + virtual void processData(const char *requeset_url, int request_url_len, const char *response_data, int response_data_len) = 0; virtual ~FetchedDataProcessor(){}; }; diff --git a/plugins/esi/lib/EsiGunzip.cc b/plugins/esi/lib/EsiGunzip.cc index 26bbea9c54c..167b664c285 100644 --- a/plugins/esi/lib/EsiGunzip.cc +++ b/plugins/esi/lib/EsiGunzip.cc @@ -34,7 +34,7 @@ EsiGunzip::EsiGunzip(const char *debug_tag, ComponentBase::Debug debug_func, Com { _init = false; _success = true; - // zlib _zstrm varibles are initialzied when they are required in stream_decode + // zlib _zstrm variables are initialized when they are required in stream_decode // coverity[uninit_member] // coverity[uninit_ctor] } diff --git a/plugins/esi/lib/EsiGzip.cc b/plugins/esi/lib/EsiGzip.cc index 6c0c7cfdea2..0786d593819 100644 --- a/plugins/esi/lib/EsiGzip.cc +++ b/plugins/esi/lib/EsiGzip.cc @@ -32,7 +32,7 @@ using namespace EsiLib; EsiGzip::EsiGzip(const char *debug_tag, ComponentBase::Debug debug_func, ComponentBase::Error error_func) : ComponentBase(debug_tag, debug_func, error_func), _downstream_length(0), _total_data_length(0) { - // Zlib _zstrm varibles are initialized when they are required in runDeflateLoop + // Zlib _zstrm variables are initialized when they are required in runDeflateLoop // coverity[uninit_member] // coverity[uninit_ctor] } diff --git a/plugins/esi/lib/EsiProcessor.cc b/plugins/esi/lib/EsiProcessor.cc index 272c3ef102a..379252b8b6b 100644 --- a/plugins/esi/lib/EsiProcessor.cc +++ b/plugins/esi/lib/EsiProcessor.cc @@ -29,7 +29,7 @@ using std::string; using namespace EsiLib; extern pthread_key_t threadKey; -// this needs to be a fixed address as only the address is used for comparision +// this needs to be a fixed address as only the address is used for comparison const char *EsiProcessor::INCLUDE_DATA_ID_ATTR = reinterpret_cast(0xbeadface); #define FAILURE_INFO_TAG "plugin_esi_failureInfo" @@ -699,7 +699,7 @@ EsiProcessor::_handleHtmlComment(const DocNodeList::iterator &curr_node) _debugLog(_debug_tag, "[%s] parsed %d inner nodes from html comment node", __FUNCTION__, inner_nodes.size()); DocNodeList::iterator next_node = curr_node; ++next_node; - _node_list.splice(next_node, inner_nodes); // insert after curr node for preprocessing + _node_list.splice(next_node, inner_nodes); // insert after curr node for pre-processing return true; } @@ -733,9 +733,9 @@ EsiProcessor::_preprocess(DocNodeList &node_list, int &n_prescanned_nodes) break; case DocNode::TYPE_HTML_COMMENT: /** - * the html comment is a container. + * the html comment is a container. * the esi processor will remove the starting tag "", then keep the innertext (the content within it). + * closure tag "-->", then keep the inner text (the content within it). * * we should call _handleHtmlComment when the node list is parsed * from the content, diff --git a/plugins/esi/lib/FailureInfo.h b/plugins/esi/lib/FailureInfo.h index ba8e61ea073..ffb4a9e04fc 100644 --- a/plugins/esi/lib/FailureInfo.h +++ b/plugins/esi/lib/FailureInfo.h @@ -101,12 +101,12 @@ class FailureInfo : private EsiLib::ComponentBase /* Keep track of the number of windows filled prev*/ size_t _windowsPassed; - /*Used as a deciding factor between attempt/except - * incase prob is complete truth + /* Used as a deciding factor between attempt/except + * in case prob is complete truth */ double _avgOverWindow; public: - /*Was a reqeust made*/ + /*Was a request made*/ bool _requestMade; }; diff --git a/plugins/esi/lib/Stats.cc b/plugins/esi/lib/Stats.cc index 828ed809463..d048b3a47bd 100644 --- a/plugins/esi/lib/Stats.cc +++ b/plugins/esi/lib/Stats.cc @@ -42,7 +42,7 @@ namespace Stats g_system = system; if (g_system) { for (int i = 0; i < Stats::MAX_STAT_ENUM; ++i) { - // FIXME doesn't return avalue. + // FIXME doesn't return a value. g_system->create(i); /* if (!g_system->create(i)) { Utils::ERROR_LOG("[%s] Unable to create stat [%s]", __FUNCTION__, Stats::STAT_NAMES[i]); @@ -56,7 +56,7 @@ namespace Stats increment(Stats::STAT st, int step /* = 1 */) { if (g_system) { - // FIXME doesn't return avalue. + // FIXME doesn't return a value. g_system->increment(st, step); /* if (!g_system->increment(st, step)) { diff --git a/plugins/esi/test/sampleProb.cc b/plugins/esi/test/sampleProb.cc index db460bc1d44..0f22bcc3830 100644 --- a/plugins/esi/test/sampleProb.cc +++ b/plugins/esi/test/sampleProb.cc @@ -129,7 +129,7 @@ isAttemptReq(string URL, FailureData &data) // cout<<"Failure:"< 0) { avg += passFail[i].first / (passFail[i].first + passFail[i].second); - // cout<<"Prob of faillure:"< no extraction */ String _extrValidationHdrName; /** @brief header name to extract the token validation status, if empty => no extraction */ bool _useRedirects = false; /** @brief true - use redirect to set the access token cookie, @todo not used yet */ - Classifier _uriPathScope; /**< @brief blacklist (exclude) and white-list (include) whcih path should have the access control */ + Classifier _uriPathScope; /**< @brief blacklist (exclude) and white-list (include) which path should have the access control */ }; diff --git a/plugins/experimental/access_control/pattern.cc b/plugins/experimental/access_control/pattern.cc index 396e222f4f9..018a4a10276 100644 --- a/plugins/experimental/access_control/pattern.cc +++ b/plugins/experimental/access_control/pattern.cc @@ -160,7 +160,7 @@ Pattern::pcreFree() } /** - * @bried Destructor, frees PCRE related resources. + * @brief Destructor, frees PCRE related resources. */ Pattern::~Pattern() { diff --git a/plugins/experimental/acme/acme.c b/plugins/experimental/acme/acme.c index 65507d9c3bd..afb04578464 100644 --- a/plugins/experimental/acme/acme.c +++ b/plugins/experimental/acme/acme.c @@ -77,7 +77,7 @@ make_absolute_path(char *dest, int dest_len, const char *file, int file_len) for (i = 0; i < file_len; ++i) { char c = file[i]; - /* Assure that only Base64-URL chracter are in the path */ + /* Assure that only Base64-URL character are in the path */ if (!(c == '-' || c == '_' || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) { TSDebug(PLUGIN_NAME, "Invalid Base64 character found, error"); return 0; diff --git a/plugins/experimental/balancer/hash.cc b/plugins/experimental/balancer/hash.cc index d08df3ab861..c6ff0a60331 100644 --- a/plugins/experimental/balancer/hash.cc +++ b/plugins/experimental/balancer/hash.cc @@ -181,7 +181,7 @@ struct HashBalancer : public BalancerInstance { // OK, now look up this hash in the hash ring. lower_bound() finds the first element that is not less than the // target, so the element we find is the first key that is greater than our target. To visualize this in the - // hash ring, that means that each node owns the preceeding keyspace (ie. the node is at the end of each keyspace + // hash ring, that means that each node owns the preceding keyspace (ie. the node is at the end of each keyspace // range). This means that when we wrap, the first node owns the wrapping portion of the keyspace. loc = this->hash_ring.lower_bound(key); if (loc == this->hash_ring.end()) { diff --git a/plugins/experimental/buffer_upload/README b/plugins/experimental/buffer_upload/README index 90a6c2d29bd..24d4b41e7d7 100644 --- a/plugins/experimental/buffer_upload/README +++ b/plugins/experimental/buffer_upload/README @@ -11,7 +11,7 @@ Upload proxy specs for phase I: 1. Memory buffering (buffer the entire POST data in IOBuffer before connecting to OS) 1.1. Memory buffer size is configured with "mem_buffer_size" in config file. Default and minimum value is 32K - You can increase it in the config file. If a request's size is larger than "mem_buffer_size" specifiied + You can increase it in the config file. If a request's size is larger than "mem_buffer_size" specified in config file, then the upload proxy feature will be disabled for this particular request 2. Disk buffering (buffer the entire POST data on disk before connecting to OS) @@ -43,9 +43,9 @@ Upload proxy specs for phase I: 5.1. for now check if the "host" part in the URL is same as the proxy server name, then will do this conversion 5.2. To turn on URL conversion feature, set "convert_url 1" in config file -6. All request headers inlcuding cookies plus the entire POST data will be buffered (either in memory or on disk) +6. All request headers including cookies plus the entire POST data will be buffered (either in memory or on disk) -7. Config file can be explicitly sepcified as a parameter in command line (in plugin.config file) +7. Config file can be explicitly specified as a parameter in command line (in plugin.config file) a sample config file: diff --git a/plugins/experimental/cache_range_requests/README b/plugins/experimental/cache_range_requests/README index 39cd0fcd11d..5495f687d0f 100644 --- a/plugins/experimental/cache_range_requests/README +++ b/plugins/experimental/cache_range_requests/README @@ -23,7 +23,7 @@ Configuration: Or for a global plugin where all range requests are processed, Add cache_range_requests.so to the plugin.config -Parent Selection Mode (consisent-hash only): +Parent Selection Mode (consistent-hash only): default: Parent selection is based solely on the hash of a URL Path In this mode, all partial content of a URL is requested from the same diff --git a/plugins/experimental/cache_range_requests/cache_range_requests.cc b/plugins/experimental/cache_range_requests/cache_range_requests.cc index b0b61ec1cd1..3a25fa69035 100644 --- a/plugins/experimental/cache_range_requests/cache_range_requests.cc +++ b/plugins/experimental/cache_range_requests/cache_range_requests.cc @@ -21,7 +21,7 @@ * This plugin looks for range requests and then creates a new * cache key url so that each individual range requests is written * to the cache as a individual object so that subsequent range - * requests are read accross different disk drives reducing I/O + * requests are read across different disk drives reducing I/O * wait and load averages when there are large numbers of range * requests. */ @@ -94,7 +94,7 @@ create_pluginconfig(int argc, const char *argv[]) } /** - * Destroy pluginconfig data stucture. + * Destroy pluginconfig data structure. */ static void delete_pluginconfig(struct pluginconfig *pc) diff --git a/plugins/experimental/certifier/certifier.cc b/plugins/experimental/certifier/certifier.cc index d2788f84cda..2cbe9e022e8 100644 --- a/plugins/experimental/certifier/certifier.cc +++ b/plugins/experimental/certifier/certifier.cc @@ -108,7 +108,7 @@ class SslLRUList using scoped_SslData = std::unique_ptr; // unordered_map is much faster in terms of insertion/lookup/removal - // Althogh it uses more space than map, the time efficiency should be more important + // Although it uses more space than map, the time efficiency should be more important std::unordered_map cnDataMap; ///< Map from CN to sslData TSMutex list_mutex; @@ -455,7 +455,7 @@ shadow_cert_generator(TSCont contp, TSEvent event, void *edata) if (cert == nullptr) { if (!sign_enabled) { TSDebug(PLUGIN_NAME, "shadow_cert_generator(): No certs found and dynamic generation disabled. Marked as wontdo."); - // There won't be certs avaiable. Mark this servername as wontdo + // There won't be certs available. Mark this servername as wontdo // Pass on as if plugin doesn't exist ssl_list->setup_data_ctx(commonName, localQ, nullptr, nullptr, true); while (!localQ.empty()) { @@ -576,7 +576,7 @@ cert_retriever(TSCont contp, TSEvent event, void *edata) TSVConnReenable(ssl_vc); } - /// For scheduled connections, the schduled continuation will handle the reenabling + /// For scheduled connections, the scheduled continuation will handle the reenabling return TS_SUCCESS; } diff --git a/plugins/experimental/cookie_remap/cookie_remap.cc b/plugins/experimental/cookie_remap/cookie_remap.cc index f6ece83b60b..000986e3cf3 100644 --- a/plugins/experimental/cookie_remap/cookie_remap.cc +++ b/plugins/experimental/cookie_remap/cookie_remap.cc @@ -902,7 +902,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s } //---------------------------------------------------------------------------- -// called whenever we need to perform substiturions on a string; used to replace +// called whenever we need to perform substitutions on a string; used to replace // things like // $url, $unmatched_path, $cr_req_url, and $cr_url_encode // returns 0 if no substitutions, 1 otw. diff --git a/plugins/experimental/cookie_remap/cookiejar.cc b/plugins/experimental/cookie_remap/cookiejar.cc index 8a8c67565a4..da0bb65627c 100644 --- a/plugins/experimental/cookie_remap/cookiejar.cc +++ b/plugins/experimental/cookie_remap/cookiejar.cc @@ -104,7 +104,7 @@ CookieJar::parse(const string &arg, const char *sepstr, bool val_check, bool mai } /* verify that the value is valid according to our configured - * opton and possibly strip out invalid characters. */ + * option and possibly strip out invalid characters. */ if (val_check && verify_value(addme, val_len) != 0) continue; diff --git a/plugins/experimental/cookie_remap/strip.h b/plugins/experimental/cookie_remap/strip.h index cf76f67a64d..5a6e97c3551 100644 --- a/plugins/experimental/cookie_remap/strip.h +++ b/plugins/experimental/cookie_remap/strip.h @@ -38,7 +38,7 @@ extern "C" { #define STRIP_FLAG_LEAVE_WHITESP 0x4 /**< all: avoid trimming spaces */ #define STRIP_FLAG_UNSAFE_QUOTES 0x8 /**< html: dont encode quotes */ #define STRIP_FLAG_UNSAFE_SLASHES 0x10 /**< all: dont encode backslashes */ -#define STRIP_FLAG_UNSAFE_SPACES 0x20 /**< html: stipped tag isnt space */ +#define STRIP_FLAG_UNSAFE_SPACES 0x20 /**< html: stripped tag isnt space */ /** Output the input after stripping all characters that are * unsafe in an HTML context. @@ -59,7 +59,7 @@ extern "C" { * * - leaves a single space character in place of each * sequence of stripped characters if no other space - * preceeded the stripped sequence (e.g., "a b" becomes + * preceded the stripped sequence (e.g., "a b" becomes * "a b", but "ab" becomes "a b") * * @param[in] in character array (string) diff --git a/plugins/experimental/custom_redirect/README b/plugins/experimental/custom_redirect/README index 87448d8745b..9bc9ea92f85 100644 --- a/plugins/experimental/custom_redirect/README +++ b/plugins/experimental/custom_redirect/README @@ -7,7 +7,7 @@ In /home/y/conf/yts/plugin.config, you can add Case 1) means using the default header (x-redirect-url) to specify a URL to redirect to; Case 2) user specifies their specific header name (to replace the default one) to specify a URL to redirect to; -Case 3) user sepcifies the specific return code, if return code matches, then plugin will force to redirect to the +Case 3) user specifies the specific return code, if return code matches, then plugin will force to redirect to the URL specified in standard "Location" header. For simplicity, we recommend to use case 1) diff --git a/plugins/experimental/geoip_acl/README b/plugins/experimental/geoip_acl/README index 4c51e709d61..3e4864d03a7 100644 --- a/plugins/experimental/geoip_acl/README +++ b/plugins/experimental/geoip_acl/README @@ -56,7 +56,7 @@ expressions, and unique rules for match. E.g. Note that the default in the case of no matches on the regular expressions -is to "allow" the request. This can be overriden, see next use case. +is to "allow" the request. This can be overridden, see next use case. 3. You can also combine 1) and 2), and provide defaults in the remap.config @@ -79,7 +79,7 @@ Finally, there's one additional parameter option that can be used: @pparam=html::/some/path.html -This will override the default reponse body for the denied responses with a +This will override the default response body for the denied responses with a custom piece of HTML. This can be useful to explain to your users why they are getting denied access to a particular piece of content. This configuration can be used with any of the use cases described above. diff --git a/plugins/experimental/geoip_acl/acl.cc b/plugins/experimental/geoip_acl/acl.cc index 17dcbad14e2..7a20c2515d6 100644 --- a/plugins/experimental/geoip_acl/acl.cc +++ b/plugins/experimental/geoip_acl/acl.cc @@ -102,7 +102,7 @@ Acl::country_id_by_addr(const sockaddr *addr) const } #endif /* HAVE_GEOIP_H */ -// This is the rest of the ACL baseclass, which is the same for all underlying Geo libraries. +// This is the rest of the ACL base class, which is the same for all underlying Geo libraries. void Acl::read_html(const char *fn) { diff --git a/plugins/experimental/inliner/README b/plugins/experimental/inliner/README index 6c5eea73547..468c2e53045 100644 --- a/plugins/experimental/inliner/README +++ b/plugins/experimental/inliner/README @@ -12,7 +12,7 @@ Inliner: 5. In case the url exists into the cache. Inliner replaces the image with a 1x1 base64 pixel and starts retrieving the content from the cache. 6. Once the image is retrieved, buffers its content into memory. - 7. At the end of the orginal document, Inliner outputs a little JavaScript + 7. At the end of the original document, Inliner outputs a little JavaScript snippet. 8. Inliner outputs every image found into the cache into a JavaScript function call. diff --git a/plugins/experimental/inliner/ats-inliner.cc b/plugins/experimental/inliner/ats-inliner.cc index 1cb2dbe51cc..a895bbf757f 100644 --- a/plugins/experimental/inliner/ats-inliner.cc +++ b/plugins/experimental/inliner/ats-inliner.cc @@ -188,7 +188,7 @@ transform_plugin(TSCont, TSEvent e, void *d) break; default: - assert(false); // UNRECHEABLE + assert(false); // UNREACHABLE break; } @@ -213,5 +213,5 @@ TSPluginInit(int, const char **) return; error: - TSError("[null-tranform] Unable to initialize plugin (disabled).\n"); + TSError("[null-transform] Unable to initialize plugin (disabled).\n"); } diff --git a/plugins/experimental/inliner/cache.cc b/plugins/experimental/inliner/cache.cc index e442806e358..f1375ee0b9e 100644 --- a/plugins/experimental/inliner/cache.cc +++ b/plugins/experimental/inliner/cache.cc @@ -71,7 +71,7 @@ namespace cache TSIOBufferWrite(self->out_->buffer, self->content_.data(), self->content_.size()); break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. break; } return 0; diff --git a/plugins/experimental/inliner/cache.h b/plugins/experimental/inliner/cache.h index 37376462b52..38b80762ab6 100644 --- a/plugins/experimental/inliner/cache.h +++ b/plugins/experimental/inliner/cache.h @@ -79,7 +79,7 @@ namespace cache self->t_.miss(); break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. break; } delete self; diff --git a/plugins/experimental/inliner/fetcher.h b/plugins/experimental/inliner/fetcher.h index 93405f2868f..1956c660133 100644 --- a/plugins/experimental/inliner/fetcher.h +++ b/plugins/experimental/inliner/fetcher.h @@ -291,7 +291,7 @@ template struct HttpTransaction { break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. } return 0; } diff --git a/plugins/experimental/inliner/ts.cc b/plugins/experimental/inliner/ts.cc index be9684e494c..f8b05649594 100644 --- a/plugins/experimental/inliner/ts.cc +++ b/plugins/experimental/inliner/ts.cc @@ -190,7 +190,7 @@ namespace io default: TSError("[" PLUGIN_TAG "] Unknown event: %i", e); - assert(false); // UNREACHEABLE + assert(false); // UNREACHABLE break; } diff --git a/plugins/experimental/inliner/vconnection.h b/plugins/experimental/inliner/vconnection.h index cb9e51fc8fa..6c11ffc7534 100644 --- a/plugins/experimental/inliner/vconnection.h +++ b/plugins/experimental/inliner/vconnection.h @@ -82,7 +82,7 @@ namespace io } } break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. break; } return TS_SUCCESS; diff --git a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc index b94c835ead6..7661d2492f7 100644 --- a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc +++ b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc @@ -229,7 +229,7 @@ custom_get_ja3(SSL *s) custom_get_ja3_prefixed(2, p, len, ja3); ja3 += ','; - // Get extenstions + // Get extensions int *o; std::string eclist, ecpflist; if (SSL_client_hello_get0_ext(s, 0x0a, &p, &len) == 1) { diff --git a/plugins/experimental/magick/magick.cc b/plugins/experimental/magick/magick.cc index 550c818bcd0..f5250b56108 100644 --- a/plugins/experimental/magick/magick.cc +++ b/plugins/experimental/magick/magick.cc @@ -9,7 +9,7 @@ http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or ageed to in writing, software + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and diff --git a/plugins/experimental/magick/sign.sh b/plugins/experimental/magick/sign.sh index e2b1592e73f..8f51b14f06a 100755 --- a/plugins/experimental/magick/sign.sh +++ b/plugins/experimental/magick/sign.sh @@ -10,7 +10,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or ageed to in writing, software +# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/plugins/experimental/magick/test.sh b/plugins/experimental/magick/test.sh index 31fc448edb4..c0b08542f67 100755 --- a/plugins/experimental/magick/test.sh +++ b/plugins/experimental/magick/test.sh @@ -10,7 +10,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or ageed to in writing, software +# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/plugins/experimental/magick/verify.sh b/plugins/experimental/magick/verify.sh index 61ad81e5de6..3de40b4a4b2 100755 --- a/plugins/experimental/magick/verify.sh +++ b/plugins/experimental/magick/verify.sh @@ -10,7 +10,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or ageed to in writing, software +# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/plugins/experimental/memcache/protocol_binary.h b/plugins/experimental/memcache/protocol_binary.h index 1f49d1417d3..8ca18ff98b6 100644 --- a/plugins/experimental/memcache/protocol_binary.h +++ b/plugins/experimental/memcache/protocol_binary.h @@ -69,7 +69,7 @@ typedef enum { } protocol_binary_response_status; /** - * Defintion of the different command opcodes. + * Definition of the different command opcodes. * See section 3.3 Command Opcodes */ typedef enum { diff --git a/plugins/experimental/metalink/README b/plugins/experimental/metalink/README index ecd8d0049e3..8ab54b30a63 100644 --- a/plugins/experimental/metalink/README +++ b/plugins/experimental/metalink/README @@ -41,7 +41,7 @@ header is already cached. If it isn't, then it tries to find a URL that is cached to use instead. It looks in the cache for some object that matches the digest in the Digest header and if it - succeeds, then it rewites the Location header with that object's + succeeds, then it rewrites the Location header with that object's URL. This way a client should get sent to a URL that's already cached diff --git a/plugins/experimental/money_trace/README b/plugins/experimental/money_trace/README index f24cbb1f677..902f860199f 100644 --- a/plugins/experimental/money_trace/README +++ b/plugins/experimental/money_trace/README @@ -1,6 +1,6 @@ Money trace plugin - This is a remap plugin that allows ATS to participate in a distrbuted tracing system based upon + This is a remap plugin that allows ATS to participate in a distributed tracing system based upon the Comcast "Money" distributed tracing and monitoring library. The Comcast "Money" library has its roots in Google's Dapper and Twitters Zipkin systems. A money trace header or session id, is attached to transaction and allows an operator with the appropriate logging systems in place, @@ -21,7 +21,7 @@ Money trace plugin See the documentation at the link above for a complete description on the "X-MoneyTrace" header and how to use and extend it in a distributed tracing system. - To configure and use this plugin, simply add it in the remap.config file where neded. EXAMPLE: + To configure and use this plugin, simply add it in the remap.config file where needed. EXAMPLE: map http://vod.foobar.com http://origin.vod.foobar.com @plugin=money_trace.so diff --git a/plugins/experimental/money_trace/money_trace.cc b/plugins/experimental/money_trace/money_trace.cc index cb1b716e930..3ea43b1d0eb 100644 --- a/plugins/experimental/money_trace/money_trace.cc +++ b/plugins/experimental/money_trace/money_trace.cc @@ -289,7 +289,7 @@ transaction_handler(TSCont contp, TSEvent event, void *edata) mt_send_client_response(txnp, txn_data); break; case TS_EVENT_HTTP_TXN_CLOSE: - LOG_DEBUG("handling transacation close."); + LOG_DEBUG("handling transaction close."); freeTransactionData(txn_data); TSContDestroy(contp); break; diff --git a/plugins/experimental/mp4/mp4_meta.cc b/plugins/experimental/mp4/mp4_meta.cc index b7c4c9e7f61..847565f157a 100644 --- a/plugins/experimental/mp4/mp4_meta.cc +++ b/plugins/experimental/mp4/mp4_meta.cc @@ -415,7 +415,7 @@ Mp4Meta::mp4_read_ftyp_atom(int64_t atom_header_size, int64_t atom_data_size) atom_size = atom_header_size + atom_data_size; - if (meta_avail < atom_size) { // data unsufficient, reasonable from the first level + if (meta_avail < atom_size) { // data insufficient, reasonable from the first level return 0; } @@ -447,7 +447,7 @@ Mp4Meta::mp4_read_moov_atom(int64_t atom_header_size, int64_t atom_data_size) return -1; } - if (meta_avail < atom_size) { // data unsufficient, wait + if (meta_avail < atom_size) { // data insufficient, wait return 0; } diff --git a/plugins/experimental/mp4/mp4_meta.h b/plugins/experimental/mp4/mp4_meta.h index 8ee8f33a19e..d4a11293af1 100644 --- a/plugins/experimental/mp4/mp4_meta.h +++ b/plugins/experimental/mp4/mp4_meta.h @@ -166,7 +166,7 @@ typedef struct { u_char reverved3[2]; u_char matrix[36]; u_char width[4]; - u_char heigth[4]; + u_char height[4]; } mp4_tkhd_atom; typedef struct { @@ -186,7 +186,7 @@ typedef struct { u_char reverved3[2]; u_char matrix[36]; u_char width[4]; - u_char heigth[4]; + u_char height[4]; } mp4_tkhd64_atom; typedef struct { diff --git a/plugins/experimental/multiplexer/dispatch.cc b/plugins/experimental/multiplexer/dispatch.cc index 32803eab6e9..675b23a349f 100644 --- a/plugins/experimental/multiplexer/dispatch.cc +++ b/plugins/experimental/multiplexer/dispatch.cc @@ -47,7 +47,7 @@ Request::Request(const std::string &h, const TSMBuffer b, const TSMLoc l) : host /* * TSHttpHdrLengthGet returns the size with possible "internal" headers * which are not printed by TSHttpHdrPrint. - * Therefore the greater than or equal comparisson + * Therefore the greater than or equal comparison */ assert(TSHttpHdrLengthGet(b, l) >= length); } diff --git a/plugins/experimental/multiplexer/fetcher.h b/plugins/experimental/multiplexer/fetcher.h index 61bc5950206..45e892f2bac 100644 --- a/plugins/experimental/multiplexer/fetcher.h +++ b/plugins/experimental/multiplexer/fetcher.h @@ -272,7 +272,7 @@ template struct HttpTransaction { break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. } return 0; } diff --git a/plugins/experimental/prefetch/fetch.h b/plugins/experimental/prefetch/fetch.h index 206771bd0b0..7c2a1dd7563 100644 --- a/plugins/experimental/prefetch/fetch.h +++ b/plugins/experimental/prefetch/fetch.h @@ -94,7 +94,7 @@ class BgFetchState /* Mechanisms to avoid concurrent fetches and applying limits */ FetchPolicy *_unique = nullptr; /* make sure we never download same object multiple times at the same time */ - TSMutex _lock; /* protects the deduplication object only */ + TSMutex _lock; /* protects the de-duplication object only */ size_t _concurrentFetches = 0; size_t _concurrentFetchesMax = 0; PrefetchMetricInfo _metrics[FETCHES_MAX_METRICS] = { diff --git a/plugins/experimental/prefetch/pattern.cc b/plugins/experimental/prefetch/pattern.cc index 3e408f70cbc..803744330d0 100644 --- a/plugins/experimental/prefetch/pattern.cc +++ b/plugins/experimental/prefetch/pattern.cc @@ -150,7 +150,7 @@ Pattern::pcreFree() } /** - * @bried Destructor, frees PCRE related resources. + * @brief Destructor, frees PCRE related resources. */ Pattern::~Pattern() { diff --git a/plugins/experimental/prefetch/plugin.cc b/plugins/experimental/prefetch/plugin.cc index fede772f6d8..cef738cdc93 100644 --- a/plugins/experimental/prefetch/plugin.cc +++ b/plugins/experimental/prefetch/plugin.cc @@ -285,7 +285,7 @@ appendCacheKey(const TSHttpTxn txnp, const TSMBuffer reqBuffer, String &key) /** * @brief Find out if the object was found fresh in cache. * - * This function finaly controls if the pre-fetch should be scheduled or not. + * This function finely controls if the pre-fetch should be scheduled or not. * @param txnp HTTP transaction structure * @return true - hit fresh, false - miss/stale/skipped or error */ @@ -476,7 +476,7 @@ contHandleFetch(const TSCont contp, TSEvent event, void *edata) /* first-pass */ if (!config.isExactMatch()) { data->_fetchable = state->acquire(data->_cachekey); - PrefetchDebug("request is%sfetchable", data->_fetchable ? " " : " not "); + PrefetchDebug("request is %s fetchable", data->_fetchable ? " " : " not "); } } } @@ -489,7 +489,7 @@ contHandleFetch(const TSCont contp, TSEvent event, void *edata) /* second-pass */ data->_fetchable = state->acquire(data->_cachekey); data->_fetchable = data->_fetchable && state->uniqueAcquire(data->_cachekey); - PrefetchDebug("request is%sfetchable", data->_fetchable ? " " : " not "); + PrefetchDebug("request is %s fetchable", data->_fetchable ? " " : " not "); if (isFetchable(txnp, data)) { if (!data->_fetchable) { diff --git a/plugins/experimental/slice/slice.cc b/plugins/experimental/slice/slice.cc index aab8b7d3f4c..5b5c90b48d9 100644 --- a/plugins/experimental/slice/slice.cc +++ b/plugins/experimental/slice/slice.cc @@ -98,7 +98,7 @@ read_request(TSHttpTxn txnp, Config *const config) data->m_urlloc = newloc; } - // we'll intercept this GET and do it ourselfs + // we'll intercept this GET and do it ourselves TSCont const icontp(TSContCreate(intercept_hook, TSMutexCreate())); TSContDataSet(icontp, (void *)data); // TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, icontp); diff --git a/plugins/experimental/slice/slice_test.cc b/plugins/experimental/slice/slice_test.cc index 60e0246a7be..4fcbc5be7bd 100644 --- a/plugins/experimental/slice/slice_test.cc +++ b/plugins/experimental/slice/slice_test.cc @@ -140,7 +140,7 @@ testParseRange() for (size_t index(0); index < gots.size(); ++index) { if (exps[index] != gots[index] || expsres[index] != gotsres[index]) { - oss << "Eror parsing index: " << index << std::endl; + oss << "Error parsing index: " << index << std::endl; oss << "test: '" << teststrings[index] << "'" << std::endl; oss << "exp: " << exps[index].m_beg << ' ' << exps[index].m_end << std::endl; oss << "expsres: " << (int)expsres[index] << std::endl; diff --git a/plugins/experimental/sslheaders/sslheaders.cc b/plugins/experimental/sslheaders/sslheaders.cc index 3fce092de84..bbda70d304b 100644 --- a/plugins/experimental/sslheaders/sslheaders.cc +++ b/plugins/experimental/sslheaders/sslheaders.cc @@ -274,7 +274,7 @@ TSPluginInit(int argc, const char *argv[]) case SSL_HEADERS_ATTACH_SERVER: TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, hdr->cont); break; - case SSL_HEADERS_ATTACH_BOTH: /* fallthru */ + case SSL_HEADERS_ATTACH_BOTH: /* fallthrough */ case SSL_HEADERS_ATTACH_CLIENT: TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, hdr->cont); TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, hdr->cont); @@ -319,7 +319,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri */) case SSL_HEADERS_ATTACH_SERVER: TSHttpTxnHookAdd(txn, TS_HTTP_SEND_REQUEST_HDR_HOOK, hdr->cont); break; - case SSL_HEADERS_ATTACH_BOTH: /* fallthru */ + case SSL_HEADERS_ATTACH_BOTH: /* fallthrough */ case SSL_HEADERS_ATTACH_CLIENT: TSHttpTxnHookAdd(txn, TS_HTTP_READ_REQUEST_HDR_HOOK, hdr->cont); TSHttpTxnHookAdd(txn, TS_HTTP_SEND_REQUEST_HDR_HOOK, hdr->cont); diff --git a/plugins/experimental/sslheaders/sslheaders.h b/plugins/experimental/sslheaders/sslheaders.h index 7ccb00f2912..99f8fbd4d64 100644 --- a/plugins/experimental/sslheaders/sslheaders.h +++ b/plugins/experimental/sslheaders/sslheaders.h @@ -67,7 +67,7 @@ struct SslHdrExpansion { ExpansionScope scope = SSL_HEADERS_SCOPE_NONE; ExpansionField field = SSL_HEADERS_FIELD_NONE; - // noncopyable but moveable + // noncopyable but movable SslHdrExpansion(const SslHdrExpansion &) = delete; SslHdrExpansion &operator=(const SslHdrExpansion &) = delete; SslHdrExpansion(SslHdrExpansion &&) = default; diff --git a/plugins/experimental/stream_editor/stream_editor.cc b/plugins/experimental/stream_editor/stream_editor.cc index a19552e05b4..7451c91946b 100644 --- a/plugins/experimental/stream_editor/stream_editor.cc +++ b/plugins/experimental/stream_editor/stream_editor.cc @@ -849,10 +849,10 @@ TSPluginInit(int argc, const char *argv[]) } if (rewrites_in != nullptr) { - TSDebug("[stream-editor]", "initialising input filtering"); + TSDebug("[stream-editor]", "initializing input filtering"); inputcont = TSContCreate(streamedit_setup, nullptr); if (inputcont == nullptr) { - TSError("[stream-editor] failed to initialise input filtering!"); + TSError("[stream-editor] failed to initialize input filtering!"); } else { TSContDataSet(inputcont, rewrites_in); TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, inputcont); @@ -862,10 +862,10 @@ TSPluginInit(int argc, const char *argv[]) } if (rewrites_out != nullptr) { - TSDebug("[stream-editor]", "initialising output filtering"); + TSDebug("[stream-editor]", "initializing output filtering"); outputcont = TSContCreate(streamedit_setup, nullptr); if (outputcont == nullptr) { - TSError("[stream-editor] failed to initialise output filtering!"); + TSError("[stream-editor] failed to initialize output filtering!"); } else { TSContDataSet(outputcont, rewrites_out); TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, outputcont); diff --git a/plugins/experimental/system_stats/system_stats.c b/plugins/experimental/system_stats/system_stats.c index 44c91b87982..8bd615f77e8 100644 --- a/plugins/experimental/system_stats/system_stats.c +++ b/plugins/experimental/system_stats/system_stats.c @@ -135,7 +135,7 @@ setNetStat(TSMutex stat_creation_mutex, const char *interface, const char *entry // Generate the ATS stats name snprintf(&stat_name[0], sizeof(stat_name), "%s%s.%s", NET_STATS, interface, entry); - // Determine if this is a toplevel netdev stat, or one from stastistics. + // Determine if this is a toplevel netdev stat, or one from statistics. if (subdir == NULL) { snprintf(&sysfs_name[0], sizeof(sysfs_name), "%s/%s/%s", NET_STATS_DIR, interface, entry); } else { diff --git a/plugins/experimental/url_sig/README b/plugins/experimental/url_sig/README index 9dc4dee3f31..c7e397845db 100644 --- a/plugins/experimental/url_sig/README +++ b/plugins/experimental/url_sig/README @@ -85,7 +85,7 @@ Signing a URL using path parameters instead of using a query string. the file part of the request and are never part of the sign string. Path parameters are separated by a ';' in the path. The complete signature - string is base64 encoded as a single path parameter that is assinged to the + string is base64 encoded as a single path parameter that is assigned to the 'siganchor', and will appear in that path as siganchor=base64string. The following is an example signed request using the path parameter method and with an origin application query string: diff --git a/plugins/experimental/url_sig/url_sig.c b/plugins/experimental/url_sig/url_sig.c index a21861e1753..7857400b268 100644 --- a/plugins/experimental/url_sig/url_sig.c +++ b/plugins/experimental/url_sig/url_sig.c @@ -433,13 +433,13 @@ urlParse(char *url, char *anchor, char *new_path_seg, int new_path_seg_len, char if (strlen(sig_anchor) < signed_seg_len) { memcpy(signed_seg, sig_anchor, strlen(sig_anchor)); } else { - TSError("insuficient space to copy into new_path_seg buffer."); + TSError("insufficient space to copy into new_path_seg buffer."); } } else { // no signature anchor string was found, assum it is in the last path segment. if (strlen(segment[numtoks - 2]) < signed_seg_len) { memcpy(signed_seg, segment[numtoks - 2], strlen(segment[numtoks - 2])); } else { - TSError("insuficient space to copy into new_path_seg buffer."); + TSError("insufficient space to copy into new_path_seg buffer."); return NULL; } } diff --git a/plugins/generator/generator.cc b/plugins/generator/generator.cc index 4dc7f4de045..f4bb95de7fd 100644 --- a/plugins/generator/generator.cc +++ b/plugins/generator/generator.cc @@ -180,7 +180,7 @@ struct GeneratorRequest { ~GeneratorRequest() {} }; -// Destroy a generator request, iincluding the per-txn continuation. +// Destroy a generator request, including the per-txn continuation. static void GeneratorRequestDestroy(GeneratorRequest *grq, TSVIO vio, TSCont contp) { diff --git a/plugins/header_rewrite/conditions.cc b/plugins/header_rewrite/conditions.cc index 16082da045f..8bb0b7e0a97 100644 --- a/plugins/header_rewrite/conditions.cc +++ b/plugins/header_rewrite/conditions.cc @@ -909,7 +909,7 @@ ConditionGeo::get_geo_int(const sockaddr *addr) const } switch (_geo_qual) { - // Country Databse + // Country Database case GEO_QUAL_COUNTRY_ISO: switch (addr->sa_family) { case AF_INET: @@ -974,7 +974,7 @@ ConditionGeo::get_geo_int(const sockaddr *addr) const #else -// No Geo library avaiable, these are just stubs. +// No Geo library available, these are just stubs. const char * ConditionGeo::get_geo_string(const sockaddr *addr) const diff --git a/plugins/header_rewrite/parser.cc b/plugins/header_rewrite/parser.cc index 4fc1bf40671..cc7a70c52b7 100644 --- a/plugins/header_rewrite/parser.cc +++ b/plugins/header_rewrite/parser.cc @@ -95,7 +95,7 @@ Parser::Parser(const std::string &original_line) : _cond(false), _empty(false) } if ((line[i] == '=') || (line[i] == '+')) { - // These are always a seperate token + // These are always a separate token _tokens.push_back(std::string(1, line[i])); continue; } diff --git a/plugins/header_rewrite/resources.cc b/plugins/header_rewrite/resources.cc index cc1379e93b4..bc861c6c2b7 100644 --- a/plugins/header_rewrite/resources.cc +++ b/plugins/header_rewrite/resources.cc @@ -84,7 +84,7 @@ Resources::gather(const ResourceIDs ids, TSHttpHookID hook) return; } if (ids & RSRC_RESPONSE_STATUS) { - TSDebug(PLUGIN_NAME, "\tAdding TXN client esponse status resource"); + TSDebug(PLUGIN_NAME, "\tAdding TXN client response status resource"); resp_status = TSHttpHdrStatusGet(bufp, hdr_loc); } } diff --git a/plugins/healthchecks/healthchecks.c b/plugins/healthchecks/healthchecks.c index 3fe44b95de6..37d7d73ebde 100644 --- a/plugins/healthchecks/healthchecks.c +++ b/plugins/healthchecks/healthchecks.c @@ -206,7 +206,7 @@ hc_thread(void *data ATS_UNUSED) do { HCFileData *next = fdata->_next; - TSDebug(PLUGIN_NAME, "Cleaning up entry from frelist"); + TSDebug(PLUGIN_NAME, "Cleaning up entry from freelist"); TSfree(fdata); fdata = next; } while (fdata); @@ -477,7 +477,7 @@ hc_process_accept(TSCont contp, HCState *my_state) my_state->read_vio = TSVConnRead(my_state->net_vc, contp, my_state->req_buffer, INT64_MAX); } -/* Imlement the server intercept */ +/* Implement the server intercept */ static int hc_intercept(TSCont contp, TSEvent event, void *edata) { @@ -512,7 +512,7 @@ health_check_origin(TSCont contp ATS_UNUSED, TSEvent event ATS_UNUSED, void *eda int path_len = 0; const char *path = TSUrlPathGet(reqp, url_loc, &path_len); - /* Short circuit the / path, common case, and we won't allow healthecks on / */ + /* Short circuit the / path, common case, and we won't allow healthchecks on / */ if (!path || !path_len) { goto cleanup; } diff --git a/plugins/lua/ts_lua_cached_response.c b/plugins/lua/ts_lua_cached_response.c index 28a8ef0b5eb..0fa5e174c0f 100644 --- a/plugins/lua/ts_lua_cached_response.c +++ b/plugins/lua/ts_lua_cached_response.c @@ -163,7 +163,7 @@ ts_lua_cached_response_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->cached_response_bufp, http_ctx->cached_response_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; diff --git a/plugins/lua/ts_lua_client_request.c b/plugins/lua/ts_lua_client_request.c index ae4356d4853..b851eb19d7e 100644 --- a/plugins/lua/ts_lua_client_request.c +++ b/plugins/lua/ts_lua_client_request.c @@ -163,7 +163,7 @@ ts_lua_client_request_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->client_request_bufp, http_ctx->client_request_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; diff --git a/plugins/lua/ts_lua_client_response.c b/plugins/lua/ts_lua_client_response.c index dfeebccb201..224e5e45f5f 100644 --- a/plugins/lua/ts_lua_client_response.c +++ b/plugins/lua/ts_lua_client_response.c @@ -108,7 +108,7 @@ ts_lua_client_response_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->client_response_bufp, http_ctx->client_response_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; diff --git a/plugins/lua/ts_lua_server_request.c b/plugins/lua/ts_lua_server_request.c index d0b26e0800b..608f39e3e4e 100644 --- a/plugins/lua/ts_lua_server_request.c +++ b/plugins/lua/ts_lua_server_request.c @@ -188,7 +188,7 @@ ts_lua_server_request_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->server_request_bufp, http_ctx->server_request_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; diff --git a/plugins/lua/ts_lua_server_response.c b/plugins/lua/ts_lua_server_response.c index 3d991d2ed93..f2c78d49ab8 100644 --- a/plugins/lua/ts_lua_server_response.c +++ b/plugins/lua/ts_lua_server_response.c @@ -181,7 +181,7 @@ ts_lua_server_response_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->server_response_bufp, http_ctx->server_response_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; diff --git a/plugins/s3_auth/aws_auth_v4.h b/plugins/s3_auth/aws_auth_v4.h index d0eac44a1f6..aa929de57be 100644 --- a/plugins/s3_auth/aws_auth_v4.h +++ b/plugins/s3_auth/aws_auth_v4.h @@ -25,7 +25,7 @@ #pragma once #include /* transform() */ -#include /* soze_t */ +#include /* size_t */ #include /* std::string */ #include /* std::stringstream */ #include /* std::map */ diff --git a/plugins/s3_auth/aws_auth_v4_wrap.h b/plugins/s3_auth/aws_auth_v4_wrap.h index 61480a64dd3..72221c3b89b 100644 --- a/plugins/s3_auth/aws_auth_v4_wrap.h +++ b/plugins/s3_auth/aws_auth_v4_wrap.h @@ -18,7 +18,7 @@ /** * @file aws_auth_v4_ts.h - * @brief TS API adaptor and header iterator using the TS API which are swapped with mocks during testing. + * @brief TS API adapter and header iterator using the TS API which are swapped with mocks during testing. * @see aws_auth_v4.h */ diff --git a/plugins/s3_auth/s3_auth.cc b/plugins/s3_auth/s3_auth.cc index 51bd6148acd..77e86fb3057 100644 --- a/plugins/s3_auth/s3_auth.cc +++ b/plugins/s3_auth/s3_auth.cc @@ -41,7 +41,7 @@ #include #include "tscore/ink_config.h" -// Special snowflake here, only availbale when building inside the ATS source tree. +// Special snowflake here, only available when building inside the ATS source tree. #include "tscore/ink_atomic.h" #include "aws_auth_v4.h" @@ -137,7 +137,7 @@ loadRegionMap(StringMap &m, const String &filename) } /////////////////////////////////////////////////////////////////////////////// -// Cache for the secrets file, to avoid reading / loding them repeatedly on +// Cache for the secrets file, to avoid reading / loading them repeatedly on // a reload of remap.config. This gets cached for 60s (not configurable). // class S3Config; @@ -463,7 +463,7 @@ S3Config::parse_config(const std::string &config_fname) continue; } - // Skip trailig white spaces + // Skip trailing white spaces pos2 = pos1; pos1 = pos2 + strlen(pos2) - 1; while ((pos1 > pos2) && isspace(*pos1)) { @@ -647,7 +647,7 @@ S3Request::set_header(const char *header, int header_len, const char *val, int v return ret; } -// dst poinsts to starting offset of dst buffer +// dst points to starting offset of dst buffer // dst_len remaining space in buffer static size_t str_concat(char *dst, size_t dst_len, const char *src, size_t src_len) @@ -1060,7 +1060,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo * /* rri */) if (s3) { TSAssert(s3->valid()); - s3->acquire(); // Increasement ref-count + s3->acquire(); // Increase ref-count // Now schedule the continuation to update the URL when going to origin. // Note that in most cases, this is a No-Op, assuming you have reasonable // cache hit ratio. However, the scheduling is next to free (very cheap). diff --git a/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc b/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc index 025cc2bb976..408ce21f2d8 100644 --- a/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc +++ b/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc @@ -364,7 +364,7 @@ ValidateBench(TsInterface &api, bool signPayload, time_t *now, const char *bench CAPTURE(signedHeaders); CHECK_FALSE(signedHeaders.compare(bench[6])); - /* Test the formating of the date and time */ + /* Test the formatting of the date and time */ char dateTime[sizeof("20170428T010203Z")]; size_t dateTimeLen = getIso8601Time(now, dateTime, sizeof(dateTime)); CAPTURE(String(dateTime, dateTimeLen)); diff --git a/plugins/tcpinfo/tcpinfo.cc b/plugins/tcpinfo/tcpinfo.cc index 588ba3424f5..e8e498e668e 100644 --- a/plugins/tcpinfo/tcpinfo.cc +++ b/plugins/tcpinfo/tcpinfo.cc @@ -53,7 +53,7 @@ // Log format headers. These are emitted once at the start of a log file. Note that we // carefully order the fields so the field ordering is compatible. This lets you change -// the verbosity without breaking a perser that is moderately robust. +// the verbosity without breaking a parser that is moderately robust. static const char *tcpi_headers[] = { "timestamp event client server rtt", "timestamp event client server rtt rttvar last_sent last_recv " @@ -254,7 +254,7 @@ parse_unsigned(const char *str, unsigned long &lval) } if (end && *end != '\0') { - // Not all charaters consumed. + // Not all characters consumed. return false; } diff --git a/plugins/xdebug/xdebug_headers.cc b/plugins/xdebug/xdebug_headers.cc index 51f5d86c20d..aef6871146a 100644 --- a/plugins/xdebug/xdebug_headers.cc +++ b/plugins/xdebug/xdebug_headers.cc @@ -45,15 +45,15 @@ escape_char_for_json(char const &c, bool &parsing_key) case '\t': return {"\\t"}; - // Special header reformating + // Special header reformatting case '\r': return {""}; case '\n': parsing_key = true; - return {"',\r\n\t'"}; // replace new line with pair delemiter + return {"',\r\n\t'"}; // replace new line with pair delimiter case ':': if (parsing_key) { - return {"' : "}; // replace colon after keywith quote + colon + return {"' : "}; // replace colon after key with quote + colon } return {":"}; case ' ': diff --git a/plugins/xdebug/xdebug_transforms.cc b/plugins/xdebug/xdebug_transforms.cc index 1bec6e80c3e..a5bc75a0a73 100644 --- a/plugins/xdebug/xdebug_transforms.cc +++ b/plugins/xdebug/xdebug_transforms.cc @@ -86,7 +86,7 @@ body_transform(TSCont contp, TSEvent event, void *edata) return TS_ERROR; } if (TSVConnClosedGet(contp)) { - // write connection destoried. cleanup. + // write connection destroyed. cleanup. delete data; TSContDestroy(contp); return 0; @@ -106,7 +106,7 @@ body_transform(TSCont contp, TSEvent event, void *edata) } case TS_EVENT_VCONN_WRITE_READY: TSDebug("xdebug_transform", "body_transform(): Event is TS_EVENT_VCONN_WRITE_READY"); - // fallthru + // fall through default: if (!data->output_buffer) { data->output_buffer = TSIOBufferCreate(); From c8c4c18b9e21718ddebd5a4fb369fc52051c4987 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Sat, 27 Apr 2019 09:30:20 +0800 Subject: [PATCH 507/526] Fixes spelling in code in plugins kvDeliiter -> kvDelimiter replacenemt -> replacement --- plugins/experimental/access_control/access_control.cc | 8 ++++---- plugins/experimental/access_control/access_control.h | 2 +- plugins/experimental/access_control/pattern.cc | 6 +++--- plugins/experimental/access_control/pattern.h | 2 +- plugins/experimental/prefetch/pattern.cc | 6 +++--- plugins/experimental/prefetch/pattern.h | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/plugins/experimental/access_control/access_control.cc b/plugins/experimental/access_control/access_control.cc index 7ec8a989c35..7401c5fc8b1 100644 --- a/plugins/experimental/access_control/access_control.cc +++ b/plugins/experimental/access_control/access_control.cc @@ -194,7 +194,7 @@ KvpAccessToken::parse(const StringView token) /* Look for the next KVP */ pos = _token.find(_tokenConfig.pairDelimiter, prev); StringView kvp = _token.substr(prev, pos - prev); - size_t equalsign = kvp.find(_tokenConfig.kvDeliiter); + size_t equalsign = kvp.find(_tokenConfig.kvDelimiter); if (kvp.npos == equalsign) { ERROR_OUT("invalid key-value-pair, missing key-value delimiter"); return _state = INVALID_SYNTAX; @@ -231,11 +231,11 @@ KvpAccessToken::parse(const StringView token) return _state = INVALID_FIELD; } - prev = pos + _tokenConfig.kvDeliiter.size(); + prev = pos + _tokenConfig.kvDelimiter.size(); } while (pos != token.npos); /* Now identify the pay-load which was signed */ - payloadSize += _tokenConfig.messageDigestName.size() + _tokenConfig.kvDeliiter.size(); + payloadSize += _tokenConfig.messageDigestName.size() + _tokenConfig.kvDelimiter.size(); _payload = _token.substr(0, payloadSize); DEBUG_OUT("payload:'" << _payload << "'"); @@ -256,7 +256,7 @@ void KvpAccessTokenBuilder::appendKeyValuePair(const StringView &key, const StringView value) { _buffer.append(_buffer.empty() ? "" : _config.pairDelimiter); - _buffer.append(key).append(_config.kvDeliiter).append(value); + _buffer.append(key).append(_config.kvDelimiter).append(value); } void diff --git a/plugins/experimental/access_control/access_control.h b/plugins/experimental/access_control/access_control.h index 6569cb1ec65..7d860083507 100644 --- a/plugins/experimental/access_control/access_control.h +++ b/plugins/experimental/access_control/access_control.h @@ -79,7 +79,7 @@ struct KvpAccessTokenConfig { StringView messageDigestName = "md"; String pairDelimiter = "&"; - String kvDeliiter = "="; + String kvDelimiter = "="; }; /** diff --git a/plugins/experimental/access_control/pattern.cc b/plugins/experimental/access_control/pattern.cc index 018a4a10276..fc9b4e9a562 100644 --- a/plugins/experimental/access_control/pattern.cc +++ b/plugins/experimental/access_control/pattern.cc @@ -47,18 +47,18 @@ Pattern::Pattern() : _pattern(""), _replacement("") {} * @return true if successful, false if failure */ bool -Pattern::init(const String &pattern, const String &replacenemt, bool replace) +Pattern::init(const String &pattern, const String &replacement, bool replace) { pcreFree(); _pattern.assign(pattern); - _replacement.assign(replacenemt); + _replacement.assign(replacement); _replace = replace; _tokenCount = 0; if (!compile()) { - AccessControlDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacenemt.c_str()); + AccessControlDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacement.c_str()); pcreFree(); return false; } diff --git a/plugins/experimental/access_control/pattern.h b/plugins/experimental/access_control/pattern.h index 46df0baa8f2..f4f37a7a600 100644 --- a/plugins/experimental/access_control/pattern.h +++ b/plugins/experimental/access_control/pattern.h @@ -43,7 +43,7 @@ class Pattern Pattern(); virtual ~Pattern(); - bool init(const String &pattern, const String &replacenemt, bool replace); + bool init(const String &pattern, const String &replacement, bool replace); bool init(const String &config); bool empty() const; bool match(const String &subject); diff --git a/plugins/experimental/prefetch/pattern.cc b/plugins/experimental/prefetch/pattern.cc index 803744330d0..e0ca6ca3f08 100644 --- a/plugins/experimental/prefetch/pattern.cc +++ b/plugins/experimental/prefetch/pattern.cc @@ -47,17 +47,17 @@ Pattern::Pattern() : _pattern(""), _replacement("") {} * @return true if successful, false if failure */ bool -Pattern::init(const String &pattern, const String &replacenemt) +Pattern::init(const String &pattern, const String &replacement) { pcreFree(); _pattern.assign(pattern); - _replacement.assign(replacenemt); + _replacement.assign(replacement); _tokenCount = 0; if (!compile()) { - PrefetchDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacenemt.c_str()); + PrefetchDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacement.c_str()); pcreFree(); return false; } diff --git a/plugins/experimental/prefetch/pattern.h b/plugins/experimental/prefetch/pattern.h index 1045812b110..6d7c5aa5b29 100644 --- a/plugins/experimental/prefetch/pattern.h +++ b/plugins/experimental/prefetch/pattern.h @@ -43,7 +43,7 @@ class Pattern Pattern(); virtual ~Pattern(); - bool init(const String &pattern, const String &replacenemt); + bool init(const String &pattern, const String &replacement); bool init(const String &config); bool empty() const; bool match(const String &subject); From e1bbb9521e805a690dbe1afed7a86c01fdc3377c Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Mon, 6 Aug 2018 21:46:55 -0700 Subject: [PATCH 508/526] Fixes spelling in mgmt --- mgmt/Alarms.cc | 12 ++++++------ mgmt/Alarms.h | 2 +- mgmt/DerivativeMetrics.cc | 4 ++-- mgmt/FileManager.cc | 8 ++++---- mgmt/FileManager.h | 2 +- mgmt/LocalManager.cc | 14 +++++++------- mgmt/MgmtDefs.h | 2 +- mgmt/ProcessManager.cc | 4 ++-- mgmt/ProxyConfig.cc | 2 +- mgmt/RecordsConfig.cc | 4 ++-- mgmt/Rollback.cc | 20 ++++++++++---------- mgmt/Rollback.h | 4 ++-- mgmt/WebMgmtUtils.cc | 30 +++++++++++++++--------------- mgmt/WebMgmtUtils.h | 4 ++-- mgmt/api/CoreAPI.cc | 12 ++++-------- mgmt/api/CoreAPIRemote.cc | 4 ++-- mgmt/api/CoreAPIShared.cc | 8 +++----- mgmt/api/CoreAPIShared.h | 2 +- mgmt/api/INKMgmtAPI.cc | 4 ++-- mgmt/api/NetworkMessage.cc | 2 +- mgmt/api/NetworkMessage.h | 4 ++-- mgmt/api/NetworkUtilsRemote.cc | 8 ++++---- mgmt/api/NetworkUtilsRemote.h | 3 ++- mgmt/api/TSControlMain.cc | 2 +- mgmt/api/include/mgmtapi.h | 2 +- mgmt/utils/MgmtMarshall.cc | 2 +- mgmt/utils/MgmtMarshall.h | 2 +- mgmt/utils/MgmtSocket.cc | 2 +- mgmt/utils/MgmtUtils.cc | 2 +- 29 files changed, 83 insertions(+), 88 deletions(-) diff --git a/mgmt/Alarms.cc b/mgmt/Alarms.cc index 2240b025781..d7a3171f3b4 100644 --- a/mgmt/Alarms.cc +++ b/mgmt/Alarms.cc @@ -212,7 +212,7 @@ Alarms::signalAlarm(alarm_t a, const char *desc, const char *ip) } /* - * Exec alarm bin for priority alarms everytime, regardless if they are + * Exec alarm bin for priority alarms every time, regardless if they are * potentially duplicates. However, only exec this for you own alarms, * don't want every node in the cluster reporting the same alarm. */ @@ -223,7 +223,7 @@ Alarms::signalAlarm(alarm_t a, const char *desc, const char *ip) ink_mutex_acquire(&mutex); if (!ip) { // if an OEM alarm, then must create the unique key alarm type; - // this key is used to hash the new OEM alarm descritption in the hash table + // this key is used to hash the new OEM alarm description in the hash table if (a == MGMT_ALARM_ADD_ALARM) { a = (alarmOEMcount - minOEMkey) % (maxOEMkey - minOEMkey) + minOEMkey; alarmOEMcount++; @@ -289,7 +289,7 @@ Alarms::signalAlarm(alarm_t a, const char *desc, const char *ip) (*(func))(a, ip, desc); } - /* Priority 2 alarms get signalled if they are the first unsolved occurrence. */ + /* Priority 2 alarms get signaled if they are the first unsolved occurrence. */ if (priority == 2 && !ip) { execAlarmBin(desc); } @@ -318,8 +318,8 @@ Alarms::resetSeenFlag(char *ip) /* * clearUnSeen(...) - * This function is a sweeper functionto clean up those alarms that have - * been taken care of through otehr local managers or at the peer itself. + * This function is a sweeper function to clean up those alarms that have + * been taken care of through other local managers or at the peer itself. */ void Alarms::clearUnSeen(char *ip) @@ -350,7 +350,7 @@ void Alarms::checkSystemNAlert() { return; -} /* End Alarms::checkSystenNAlert */ +} /* End Alarms::checkSystemNAlert */ void Alarms::execAlarmBin(const char *desc) diff --git a/mgmt/Alarms.h b/mgmt/Alarms.h index 2a0268bbedc..ce3317e6691 100644 --- a/mgmt/Alarms.h +++ b/mgmt/Alarms.h @@ -32,7 +32,7 @@ class AppVersionInfo; /*********************************************************************** * - * MODULARIZATTION: if you are adding new alarms, please ensure to add + * MODULARIZATION: if you are adding new alarms, please be sure to add * the corresponding alarms in lib/records/I_RecAlarms.h * ***********************************************************************/ diff --git a/mgmt/DerivativeMetrics.cc b/mgmt/DerivativeMetrics.cc index ca85c8afcb2..16572db439e 100644 --- a/mgmt/DerivativeMetrics.cc +++ b/mgmt/DerivativeMetrics.cc @@ -50,7 +50,7 @@ static const std::vector sum_metrics = { RECD_INT, {"proxy.process.http.origin_server_response_document_total_size", "proxy.process.http.origin_server_response_header_total_size"}}, - // Total byates of client request and response (total traffic to and from clients) + // Total bytes of client request and response (total traffic to and from clients) {"proxy.process.user_agent_total_bytes", RECD_INT, {"proxy.process.http.user_agent_total_request_bytes", "proxy.process.http.user_agent_total_response_bytes"}}, @@ -69,7 +69,7 @@ static const std::vector sum_metrics = { RECD_COUNTER, {"proxy.process.http.cache_miss_cold", "proxy.process.http.cache_miss_changed", "proxy.process.http.cache_miss_client_no_cache", "proxy.process.http.cache_miss_ims", "proxy.process.http.cache_miss_client_not_cacheable"}}, - // Total requests, both hits and misses (this is slightly superflous, but assures correct percentage calculations) + // Total requests, both hits and misses (this is slightly superfluous, but assures correct percentage calculations) {"proxy.process.cache_total_requests", RECD_COUNTER, {"proxy.process.cache_total_hits", "proxy.process.cache_total_misses"}}, // Total cache requests bytes which are cache hits {"proxy.process.cache_total_hits_bytes", diff --git a/mgmt/FileManager.cc b/mgmt/FileManager.cc index 6e9edfecf9c..9288d92e740 100644 --- a/mgmt/FileManager.cc +++ b/mgmt/FileManager.cc @@ -118,7 +118,7 @@ FileManager::addFileHelper(const char *fileName, const char *configName, bool ro // Sets rbPtr to the rollback object associated // with the passed in fileName. // -// If there is no binding, falseis returned +// If there is no binding, false is returned // bool FileManager::getRollbackObj(const char *fileName, Rollback **rbPtr) @@ -134,7 +134,7 @@ FileManager::getRollbackObj(const char *fileName, Rollback **rbPtr) // bool FileManager::fileChanged(const char* fileName) // -// Called by the Rollback class whenever a a config has changed +// Called by the Rollback class whenever a config has changed // Initiates callbacks // // @@ -148,7 +148,7 @@ FileManager::fileChanged(const char *fileName, const char *configName, bool incV for (cb = cblist.head; cb != nullptr; cb = cb->link.next) { // Dup the string for each callback to be - // defensive incase it modified when it is not supposed to be + // defensive in case it's modified when it's not supposed to be confignameCopy = ats_strdup(configName); filenameCopy = ats_strdup(fileName); (*cb->func)(filenameCopy, confignameCopy, incVersion); @@ -160,7 +160,7 @@ FileManager::fileChanged(const char *fileName, const char *configName, bool incV // void FileManger::rereadConfig() // -// Interates through the list of managed files and +// Iterates through the list of managed files and // calls Rollback::checkForUserUpdate on them // // although it is tempting, DO NOT CALL FROM SIGNAL HANDLERS diff --git a/mgmt/FileManager.h b/mgmt/FileManager.h index 94cc7cc731e..a667504d418 100644 --- a/mgmt/FileManager.h +++ b/mgmt/FileManager.h @@ -62,7 +62,7 @@ enum lockAction_t { // a binding and false otherwise // // registerCallback(FileCallbackFunc) - registers a callback function -// which will get called everytime a managed file changes. The +// which will get called every time a managed file changes. The // callback function should NOT use the calling thread to // access any Rollback objects or block for a long time // diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index fae81d336e5..15ec02ebb2f 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -151,8 +151,8 @@ LocalManager::clearStats(const char *name) // before the proxy clears them, but this should be rare. // // Doing things in the opposite order prevents that race - // but excerbates the race between the node and cluster - // stats getting cleared by progation of clearing the + // but exacerbates the race between the node and cluster + // stats getting cleared by propagation of clearing the // cluster stats // if (name && *name) { @@ -756,10 +756,10 @@ LocalManager::signalEvent(int msg_id, const char *data_raw, int data_len) #if HAVE_EVENTFD // we don't care about the actual value of wakeup_fd, so just keep adding 1. just need to - // wakeup the fd. also, note that wakeup_fd was initalized to non-blocking so we can + // wakeup the fd. also, note that wakeup_fd was initialized to non-blocking so we can // directly write to it without any timeout checking. // - // don't tigger if MGMT_EVENT_LIBRECORD because they happen all the time + // don't trigger if MGMT_EVENT_LIBRECORD because they happen all the time // and don't require a quick response. for MGMT_EVENT_LIBRECORD, rely on timeouts so // traffic_server can spend more time doing other things uint64_t one = 1; @@ -834,7 +834,7 @@ LocalManager::startProxy(const char *onetime_options) pid_t pid; // Before we do anything lets check for the existence of - // the traffic server binary along with it's execute permmissions + // the traffic server binary along with it's execute permissions if (access(absolute_proxy_binary, F_OK) < 0) { // Error can't find traffic_server mgmt_elog(errno, "[LocalManager::startProxy] Unable to find traffic server at %s\n", absolute_proxy_binary); @@ -843,7 +843,7 @@ LocalManager::startProxy(const char *onetime_options) // traffic server binary exists, check permissions else if (access(absolute_proxy_binary, R_OK | X_OK) < 0) { // Error don't have proper permissions - mgmt_elog(errno, "[LocalManager::startProxy] Unable to access %s due to bad permisssions \n", absolute_proxy_binary); + mgmt_elog(errno, "[LocalManager::startProxy] Unable to access %s due to bad permissions \n", absolute_proxy_binary); return false; } @@ -981,7 +981,7 @@ LocalManager::listenForProxy() this->bindProxyPort(p); } - // read backlong configuration value and overwrite the default value if found + // read backlog configuration value and overwrite the default value if found bool found; std::string_view fam{ats_ip_family_name(p.m_family)}; RecInt backlog = REC_readInteger("proxy.config.net.listen_backlog", &found); diff --git a/mgmt/MgmtDefs.h b/mgmt/MgmtDefs.h index 868ec25b86d..87dadf65f7c 100644 --- a/mgmt/MgmtDefs.h +++ b/mgmt/MgmtDefs.h @@ -56,7 +56,7 @@ using MgmtCallback = std::function; //------------------------------------------------------------------------- // API conversion functions. //------------------------------------------------------------------------- -/** Conversion functions to and from an aribrary type and Management types. +/** Conversion functions to and from an arbitrary type and Management types. * * A type that wants to support conversion in the TS API should create a static instance of this * class and fill in the appropriate members. The TS API set/get functions can then check for a diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc index abfcb9e48c2..b492c31d1cc 100644 --- a/mgmt/ProcessManager.cc +++ b/mgmt/ProcessManager.cc @@ -272,10 +272,10 @@ ProcessManager::signalManager(MgmtMessageHdr *mh) #if HAVE_EVENTFD // we don't care about the actual value of wakeup_fd, so just keep adding 1. just need to - // wakeup the fd. also, note that wakeup_fd was initalized to non-blocking so we can + // wakeup the fd. also, note that wakeup_fd was initialized to non-blocking so we can // directly write to it without any timeout checking. // - // don't tigger if MGMT_EVENT_LIBRECORD because they happen all the time + // don't trigger if MGMT_EVENT_LIBRECORD because they happen all the time // and don't require a quick response. for MGMT_EVENT_LIBRECORD, rely on timeouts so // traffic_server can spend more time doing other things/ uint64_t one = 1; diff --git a/mgmt/ProxyConfig.cc b/mgmt/ProxyConfig.cc index 9c189ca3878..67199ee4def 100644 --- a/mgmt/ProxyConfig.cc +++ b/mgmt/ProxyConfig.cc @@ -145,7 +145,7 @@ ConfigProcessor::set(unsigned int id, ConfigInfo *info, unsigned timeout_secs) if (old_info) { // The ConfigInfoReleaser now takes our refcount, but - // someother thread might also have one ... + // some other thread might also have one ... ink_assert(old_info->refcount() > 0); eventProcessor.schedule_in(new ConfigInfoReleaser(id, old_info), HRTIME_SECONDS(timeout_secs)); } diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 8e8b5a0b8ea..b382acc7826 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1189,7 +1189,7 @@ static const RecordElement RecordsConfig[] = //############################################################################## //# - //# Congifuration for TLSv1.3 and above + //# Configuration for TLSv1.3 and above //# //############################################################################## // The default value (nullptr) means the default value of TLS stack will be used. @@ -1219,7 +1219,7 @@ static const RecordElement RecordsConfig[] = , // Interim configuration setting for obeying keepalive requests on internal - // (PLuginVC) sessions. See TS-4960 and friends. + // (PluginVC) sessions. See TS-4960 and friends. {RECT_LOCAL, "proxy.config.http.keepalive_internal_vc", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}, //############################################################################## diff --git a/mgmt/Rollback.cc b/mgmt/Rollback.cc index 05e4c427f7e..0f4af405d55 100644 --- a/mgmt/Rollback.cc +++ b/mgmt/Rollback.cc @@ -63,7 +63,7 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc numberBackups(0) { version_t highestSeen; // the highest backup version - ExpandingArray existVer(25, true); // Exsisting versions + ExpandingArray existVer(25, true); // Existing versions struct stat fileInfo; MgmtInt numBak; char *alarmMsg; @@ -123,7 +123,7 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc // if (statFile(ACTIVE_VERSION, &fileInfo) < 0) { // If we can't find an active version because there is not - // one, attempt to rollback to a previous verision if one exists + // one, attempt to rollback to a previous version if one exists // // If it does not, create a zero length file to prevent total havoc // @@ -143,7 +143,7 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc needZeroLength = false; highestSeen--; // Since we've made the highestVersion active - // remove it from the backup verision q + // remove it from the backup version q versionQ.remove(versionQ.tail); } ats_free(highestSeenStr); @@ -383,7 +383,7 @@ Rollback::internalUpdate(TextBuffer *buf, version_t newVersion, bool notifyChang // Check to see if the callee has specified a newVersion number // If the newVersion argument is less than zero, the callee - // is telling us to use the next version in squence + // is telling us to use the next version in sequence if (newVersion < 0) { newVersion = this->currentVersion + 1; if (incVersion) { @@ -469,7 +469,7 @@ Rollback::internalUpdate(TextBuffer *buf, version_t newVersion, bool notifyChang newBak->modTime = 0; versionQ.enqueue(newBak); } - // Update instance varibles + // Update instance variables this->numVersions++; this->currentVersion = newVersion; @@ -731,7 +731,7 @@ Rollback::extractVersionInfo(ExpandingArray *listNames, const char *testFileName // Add wrapper to // version_t Rollback::findVersions_ml(ExpandingArray* listNames) // -// Puts the data in a queue rather than an ExpaningArray +// Puts the data in a queue rather than an ExpandingArray // version_t Rollback::findVersions_ml(Queue &q) @@ -860,7 +860,7 @@ Rollback::setLastModifiedTime() fileLastModified = TS_ARCHIVE_STAT_MTIME(fileInfo); return true; } else { - // We really shoudn't fail to stat the file since we just + // We really shouldn't fail to stat the file since we just // created it. If we do, just punt and just use the current // time. fileLastModified = (time(nullptr) - ink_timezone()) * 1000000000; @@ -872,14 +872,14 @@ Rollback::setLastModifiedTime() // // Called to check if the file has been changed // by the user. Timestamps are compared to see if a -// change occured +// change occurred // // If the file has been changed, a new version is rolled. -// The new current version and its predicessor will +// The new current version and its predecessor will // be the same in this case. While this is pointless, // for Rolling backward, we need to the version number // to up'ed so that WebFileEdit knows that the file has -// changed. Rolling a new verion also has the effect +// changed. Rolling a new version also has the effect // of creating a new timestamp // bool diff --git a/mgmt/Rollback.h b/mgmt/Rollback.h index 2ea74e4adf7..bdcaebde673 100644 --- a/mgmt/Rollback.h +++ b/mgmt/Rollback.h @@ -94,7 +94,7 @@ struct versionInfo { // // checkForUserUpdate() - compares the last known modification time // of the active version of the file with that files current modification -// time. Returns true if the file has been chaged manually or false +// time. Returns true if the file has been changed manually or false // if it hasn't // // versionTimeStamp(version_t) - returns the modification time (mtime) @@ -250,6 +250,6 @@ class Rollback Queue versionQ; // stores the backup version info }; -// qSort comptable function to sort versionInfo* +// qSort compatible function to sort versionInfo* // based on version number int versionCmp(const void *i1, const void *i2); diff --git a/mgmt/WebMgmtUtils.cc b/mgmt/WebMgmtUtils.cc index fa7fd9e9576..81b17e07d57 100644 --- a/mgmt/WebMgmtUtils.cc +++ b/mgmt/WebMgmtUtils.cc @@ -100,11 +100,11 @@ varSetFromStr(const char *varName, const char *value) // bool varSetFloat(const char* varName, RecFloat value) // -// Sets the variable specifed by varName to value. varName +// Sets the variable specified by varName to value. varName // must be a RecFloat variable. No conversion is done for // other types unless convert is set to ture. In the case // of convert is ture, type conversion is perform if applicable. -// By default, convert is set to be false and can be overrided +// By default, convert is set to be false and can be overridden // when the function is called. // bool @@ -151,11 +151,11 @@ varSetFloat(const char *varName, RecFloat value, bool convert) // bool varSetCounter(const char* varName, RecCounter value) // -// Sets the variable specifed by varName to value. varName +// Sets the variable specified by varName to value. varName // must be an RecCounter variable. No conversion is done for // other types unless convert is set to ture. In the case // of convert is ture, type conversion is perform if applicable. -// By default, convert is set to be false and can be overrided +// By default, convert is set to be false and can be overridden // when the function is called. // bool @@ -201,11 +201,11 @@ varSetCounter(const char *varName, RecCounter value, bool convert) // bool varSetInt(const char* varName, RecInt value) // -// Sets the variable specifed by varName to value. varName +// Sets the variable specified by varName to value. varName // must be an RecInt variable. No conversion is done for // other types unless convert is set to ture. In the case // of convert is ture, type conversion is perform if applicable. -// By default, convert is set to be false and can be overrided +// By default, convert is set to be false and can be overridden // when the function is called. // bool @@ -251,7 +251,7 @@ varSetInt(const char *varName, RecInt value, bool convert) // bool varSetData(RecDataT varType, const char *varName, RecData value) // -// Sets the variable specifed by varName to value. value and varName +// Sets the variable specified by varName to value. value and varName // must be varType variables. // bool @@ -279,7 +279,7 @@ varSetData(RecDataT varType, const char *varName, RecData value) // // Sets the *value to value of the varName according varType. // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // bool @@ -296,7 +296,7 @@ varDataFromName(RecDataT varType, const char *varName, RecData *value) // // Sets the *value to value of the varName. // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // bool @@ -345,7 +345,7 @@ varCounterFromName(const char *varName, RecCounter *value) // // Sets the *value to value of the varName. // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // bool @@ -395,7 +395,7 @@ varFloatFromName(const char *varName, RecFloat *value) // // Sets the *value to value of the varName. // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // bool @@ -548,12 +548,12 @@ bytesFromInt(RecInt bytes, char *bufVal) // Sets the bufVal string to the value of the local manager // named by varName. bufLen is size of bufVal // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // // EVIL ALERT: overviewRecord::varStrFromName is extremely // similar to this function except in how it gets it's -// data. Changes to this fuction must be propogated +// data. Changes to this function must be propagated // to its twin. Cut and Paste sucks but there is not // an easy way to merge the functions // @@ -933,7 +933,7 @@ setHostnameVar() return 0; } -// void appendDefautDomain(char* hostname, int bufLength) +// void appendDefaultDomain(char* hostname, int bufLength) // // Appends the pasted in hostname with the default // domain if the hostname is an unqualified name @@ -949,7 +949,7 @@ appendDefaultDomain(char *hostname, int bufLength) { int len = strlen(hostname); const char msg[] = "Nodes will be know by their unqualified host name"; - static int error_before = 0; // Race ok since effect is multple error msg + static int error_before = 0; // Race ok since effect is multiple error msg ink_assert(len < bufLength); ink_assert(bufLength >= 64); diff --git a/mgmt/WebMgmtUtils.h b/mgmt/WebMgmtUtils.h index 9112785f8d5..26cb01d915a 100644 --- a/mgmt/WebMgmtUtils.h +++ b/mgmt/WebMgmtUtils.h @@ -62,9 +62,9 @@ bool varFloatFromName(const char *varName, RecFloat *value); bool varCounterFromName(const char *varName, RecCounter *value); bool varDataFromName(RecDataT varType, const char *varName, RecData *value); -// No conversion done. varName must represnt a value of the appropriate +// No conversion done. varName must represent a value of the appropriate // type -// Default arguement "convert" added to allow great flexiblity in type checking +// Default argument "convert" added to allow great flexibility in type checking bool varSetInt(const char *varName, RecInt value, bool convert = false); bool varSetCounter(const char *varName, RecCounter value, bool convert = false); bool varSetFloat(const char *varName, RecFloat value, bool convert = false); diff --git a/mgmt/api/CoreAPI.cc b/mgmt/api/CoreAPI.cc index 56da4d4f044..4959513415d 100644 --- a/mgmt/api/CoreAPI.cc +++ b/mgmt/api/CoreAPI.cc @@ -57,7 +57,7 @@ extern FileManager *configFiles; // global in traffic_manager /*------------------------------------------------------------------------- * Init *------------------------------------------------------------------------- - * performs any necesary initializations for the local API client, + * performs any necessary initializations for the local API client, * eg. set up global structures; called by the TSMgmtAPI::TSInit() */ TSMgmtError @@ -79,7 +79,7 @@ Init(const char * /* socket_path ATS_UNUSED */, TSInitOptionT options) /*------------------------------------------------------------------------- * Terminate *------------------------------------------------------------------------- - * performs any necesary cleanup of global structures, etc, + * performs any necessary cleanup of global structures, etc, * for the local API client, */ TSMgmtError @@ -112,7 +112,7 @@ ProxyShutdown() lmgmt->processShutdown(false /* only shut down the proxy*/); - // Wait for awhile for shtudown to happen + // Wait for awhile for shutdown to happen do { mgmt_sleep_sec(1); i++; @@ -559,7 +559,7 @@ MgmtRecordGet(const char *rec_name, TSRecordEle *rec_ele) Debug("RecOp", "[MgmtRecordGet] Get String Var %s = %s", rec_ele->rec_name, rec_ele->valueT.string_val); break; - default: // UNKOWN TYPE + default: // UNKNOWN TYPE Debug("RecOp", "[MgmtRecordGet] Get Failed : %d is Unknown Var type %s", rec_type, rec_name); return TS_ERR_FAIL; } @@ -917,7 +917,3 @@ StatsReset(const char *name) lmgmt->clearStats(name); return TS_ERR_OKAY; } - -/*------------------------------------------------------------- - * rmserver.cfg - *-------------------------------------------------------------*/ diff --git a/mgmt/api/CoreAPIRemote.cc b/mgmt/api/CoreAPIRemote.cc index 8406d78b588..fbcdc42a43e 100644 --- a/mgmt/api/CoreAPIRemote.cc +++ b/mgmt/api/CoreAPIRemote.cc @@ -263,7 +263,7 @@ Terminate() // cancel the listening socket thread // it's important to call this before setting paths to NULL because the - // socket_test_thread actually will try to reconnect() and this funntion + // socket_test_thread actually will try to reconnect() and this function // will seg fault if the socket paths are NULL while it is connecting; // the thread will be cancelled at a cancellation point in the // socket_test_thread, eg. sleep @@ -484,7 +484,7 @@ StorageDeviceCmdOffline(const char *dev) } /*------------------------------------------------------------------------- - * LIfecycle Alert + * Lifecycle Alert *------------------------------------------------------------------------- * Send alert to plugins */ diff --git a/mgmt/api/CoreAPIShared.cc b/mgmt/api/CoreAPIShared.cc index 84462583d92..5ee8fd8f9c8 100644 --- a/mgmt/api/CoreAPIShared.cc +++ b/mgmt/api/CoreAPIShared.cc @@ -77,7 +77,7 @@ parseHTTPResponse(char *buffer, char **header, int *hdr_size, char **body, int * } /* readHTTPResponse - * - read from an openned socket to memory-allocated buffer and close the + * - read from an opened socket to memory-allocated buffer and close the * socket regardless success or failure. * INPUT: sock -- the socket to read the response from * buffer -- the buffer to be filled with the HTTP response @@ -132,7 +132,7 @@ readHTTPResponse(int sock, char *buffer, int bufsize, uint64_t timeout) } /* sendHTTPRequest - * - Compose a HTTP GET request and sent it via an openned socket. + * - Compose a HTTP GET request and sent it via an opened socket. * INPUT: sock -- the socket to send the message to * req -- the request to send * OUTPUT: bool -- true if everything went well. false otherwise (and sock is @@ -182,7 +182,6 @@ sendHTTPRequest(int sock, char *req, uint64_t timeout) return TS_ERR_NET_WRITE; } -/* Modified from TrafficCop.cc (open_socket) */ int connectDirect(const char *host, int port, uint64_t /* timeout ATS_UNUSED */) { @@ -241,7 +240,6 @@ connectDirect(const char *host, int port, uint64_t /* timeout ATS_UNUSED */) return -1; } /* connectDirect */ -/* COPIED direclty form TrafficCop.cc */ static int poll_read(int fd, int timeout) { @@ -336,7 +334,7 @@ get_event_id(const char *event_name) /********************************************************************** * get_event_id * - * Purpose: based on alarm_id, determine the corresonding alarm name + * Purpose: based on alarm_id, determine the corresponding alarm name * Note: allocates memory for the name returned *********************************************************************/ char * diff --git a/mgmt/api/CoreAPIShared.h b/mgmt/api/CoreAPIShared.h index c5b6c0d4895..eaa86820768 100644 --- a/mgmt/api/CoreAPIShared.h +++ b/mgmt/api/CoreAPIShared.h @@ -65,7 +65,7 @@ // used by TSReadFromUrl #define HTTP_DIVIDER "\r\n\r\n" -#define URL_BUFSIZE 65536 // the max. lenght of URL obtainable (in bytes) +#define URL_BUFSIZE 65536 // the max. length of URL obtainable (in bytes) #define URL_TIMEOUT 5000 // the timeout value for send/recv HTTP in ms #define HTTP_PORT 80 #define BUFSIZE 1024 diff --git a/mgmt/api/INKMgmtAPI.cc b/mgmt/api/INKMgmtAPI.cc index e332764423c..7b0626c4cd6 100644 --- a/mgmt/api/INKMgmtAPI.cc +++ b/mgmt/api/INKMgmtAPI.cc @@ -512,7 +512,7 @@ TSRecordGetString(const char *rec_name, TSString *string_val) *------------------------------------------------------------------------- * Purpose: Retrieves list of record values specified in the rec_names list * Input: rec_names - list of record names to retrieve - * rec_vals - queue of TSRecordEle* that correspons to rec_names + * rec_vals - queue of TSRecordEle* that corresponds to rec_names * Output: If at any point, while retrieving one of the records there's a * a failure then the entire process is aborted, all the allocated * TSRecordEle's are deallocated and TS_ERR_FAIL is returned. @@ -658,7 +658,7 @@ TSRecordSetMlt(TSList rec_list, TSActionNeedT *action_need) } // keep track of most severe action; reset if needed - // the TSACtionNeedT should be listed such that most severe actions have + // the TSActionNeedT should be listed such that most severe actions have // a lower number (so most severe action == 0) if (*action_need < top_action_req) { // a more severe action top_action_req = *action_need; diff --git a/mgmt/api/NetworkMessage.cc b/mgmt/api/NetworkMessage.cc index de2e7fa7c66..a9ba89b36ee 100644 --- a/mgmt/api/NetworkMessage.cc +++ b/mgmt/api/NetworkMessage.cc @@ -36,7 +36,7 @@ struct NetCmdOperation { const MgmtMarshallType fields[MAX_OPERATION_FIELDS]; }; -// Requests always begin with a OpType, followed by aditional fields. +// Requests always begin with a OpType, followed by additional fields. static const struct NetCmdOperation requests[] = { /* RECORD_SET */ {3, {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_STRING}}, /* RECORD_GET */ {2, {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING}}, diff --git a/mgmt/api/NetworkMessage.h b/mgmt/api/NetworkMessage.h index 99c979b007b..70f15e72a11 100644 --- a/mgmt/api/NetworkMessage.h +++ b/mgmt/api/NetworkMessage.h @@ -28,7 +28,7 @@ #define REMOTE_DELIM ':' #define REMOTE_DELIM_STR ":" -#define MAX_CONN_TRIES 10 // maximum number of attemps to reconnect to TM +#define MAX_CONN_TRIES 10 // maximum number of attempts to reconnect to TM // the possible operations or msg types sent from remote client to TM enum class OpType : MgmtMarshallInt { @@ -75,7 +75,7 @@ struct mgmt_message_sender { TSMgmtError send_mgmt_request(const mgmt_message_sender &snd, OpType optype, ...); TSMgmtError send_mgmt_request(int fd, OpType optype, ...); -// Marshall and send an error respose for this operation type. +// Marshall and send an error response for this operation type. TSMgmtError send_mgmt_error(int fd, OpType op, TSMgmtError error); // Parse a request message from a buffer. diff --git a/mgmt/api/NetworkUtilsRemote.cc b/mgmt/api/NetworkUtilsRemote.cc index 35e731bb089..e5c49841425 100644 --- a/mgmt/api/NetworkUtilsRemote.cc +++ b/mgmt/api/NetworkUtilsRemote.cc @@ -96,7 +96,7 @@ socket_test(int fd) * requests & issues out responses and alerts * 1) create and set the client socket_fd; connect to TM * 2) create and set the client's event_socket_fd; connect to TM - * output: TS_ERR_OKAY - if both sockets sucessfully connect to TM + * output: TS_ERR_OKAY - if both sockets successfully connect to TM * TS_ERR_NET_ESTABLISH - at least one unsuccessful connection * notes: If connection breaks it is responsibility of client to reconnect * otherwise traffic server will assume mgmt stopped request and @@ -209,11 +209,11 @@ disconnect() /*************************************************************************** * reconnect * - * purpose: reconnects to TM (eg. when TM restarts); does all the necesarry + * purpose: reconnects to TM (eg. when TM restarts); does all the necessary * set up for reconnection * input: None * output: TS_ERR_FAIL, TS_ERR_OKAY - * notes: necessarry events for a new client-TM connection: + * notes: necessary events for a new client-TM connection: * 1) get new socket_fd using old socket_path by calling connect() * 2) relaunch event_poll_thread_main with new socket_fd * 3) re-notify TM of all the client's registered callbacks by send msg @@ -239,7 +239,7 @@ reconnect() // relaunch a new event thread since socket_fd changed if (0 == (ts_init_options & TS_MGMT_OPT_NO_EVENTS)) { ink_thread_create(&ts_event_thread, event_poll_thread_main, &event_socket_fd, 0, 0, nullptr); - // reregister the callbacks on the TM side for this new client connection + // re-register the callbacks on the TM side for this new client connection if (remote_event_callbacks) { err = send_register_all_callbacks(event_socket_fd, remote_event_callbacks); if (err != TS_ERR_OKAY) { // problem establishing connection diff --git a/mgmt/api/NetworkUtilsRemote.h b/mgmt/api/NetworkUtilsRemote.h index 30f1a485c22..f8413dd7c21 100644 --- a/mgmt/api/NetworkUtilsRemote.h +++ b/mgmt/api/NetworkUtilsRemote.h @@ -56,7 +56,8 @@ void set_socket_paths(const char *path); * the client connection information stored in the variables in * NetworkUtilsRemote.cc */ -TSMgmtError ts_connect(); /* TODO: update documenation, Renamed due to conflict with connect() in on some platforms*/ +TSMgmtError +ts_connect(); /* TODO: update documentation, Renamed due to conflict with connect() in on some platforms*/ TSMgmtError disconnect(); TSMgmtError reconnect(); TSMgmtError reconnect_loop(int num_attempts); diff --git a/mgmt/api/TSControlMain.cc b/mgmt/api/TSControlMain.cc index 57a676115ee..b9f49a13dff 100644 --- a/mgmt/api/TSControlMain.cc +++ b/mgmt/api/TSControlMain.cc @@ -828,7 +828,7 @@ handle_host_status_down(int fd, void *req, size_t reqlen) /************************************************************************** * handle_api_ping * - * purpose: handles the API_PING messaghat is sent by API clients to keep + * purpose: handles the API_PING message that is sent by API clients to keep * the management socket alive * output: TS_ERR_xx. There is no response message. *************************************************************************/ diff --git a/mgmt/api/include/mgmtapi.h b/mgmt/api/include/mgmtapi.h index 588592b7611..c88c0dff295 100644 --- a/mgmt/api/include/mgmtapi.h +++ b/mgmt/api/include/mgmtapi.h @@ -432,7 +432,7 @@ tsapi TSMgmtError TSHostStatusSetUp(const char *host_name, int down_time, const tsapi TSMgmtError TSHostStatusSetDown(const char *host_name, int down_time, const char *reason); /*--- statistics operations -----------------------------------------------*/ /* TSStatsReset: sets all the statistics variables to their default values - * Outpue: TSErrr + * Output: TSMgmtError */ tsapi TSMgmtError TSStatsReset(const char *name); diff --git a/mgmt/utils/MgmtMarshall.cc b/mgmt/utils/MgmtMarshall.cc index 35263c34331..5152da9cc41 100644 --- a/mgmt/utils/MgmtMarshall.cc +++ b/mgmt/utils/MgmtMarshall.cc @@ -1,6 +1,6 @@ /** @file - Managment packet marshalling. + Management packet marshalling. @section license License diff --git a/mgmt/utils/MgmtMarshall.h b/mgmt/utils/MgmtMarshall.h index 11c3d9222d6..91bc257bd69 100644 --- a/mgmt/utils/MgmtMarshall.h +++ b/mgmt/utils/MgmtMarshall.h @@ -1,6 +1,6 @@ /** @file - Managment packet marshalling. + Management packet marshalling. @section license License diff --git a/mgmt/utils/MgmtSocket.cc b/mgmt/utils/MgmtSocket.cc index 5bd6bb014a1..2ad3fc6c6c8 100644 --- a/mgmt/utils/MgmtSocket.cc +++ b/mgmt/utils/MgmtSocket.cc @@ -183,7 +183,7 @@ mgmt_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *errorfds, struc // Note: Linux select() has slight different semantics. From the // man page: "On Linux, timeout is modified to reflect the amount of // time not slept; most other implementations do not do this." -// Linux select() can also return ENOMEM, so we espeically need to +// Linux select() can also return ENOMEM, so we especially need to // protect the call with the transient error retry loop. // Fortunately, because of the Linux timeout handling, our // mgmt_select call will still timeout correctly, rather than diff --git a/mgmt/utils/MgmtUtils.cc b/mgmt/utils/MgmtUtils.cc index 71e5fd6601e..3ac6642e629 100644 --- a/mgmt/utils/MgmtUtils.cc +++ b/mgmt/utils/MgmtUtils.cc @@ -334,7 +334,7 @@ mgmt_getAddrForIntr(char *intrName, sockaddr *addr, int *mtu) int fakeSocket; // a temporary socket to pass to ioctl struct ifconf ifc; // ifconf information char *ifbuf; // ifconf buffer - struct ifreq *ifr, *ifend; // pointer to individual inferface info + struct ifreq *ifr, *ifend; // pointer to individual interface info int lastlen; int len; From 494a1ead9bc711f3baa8657b90ad7d3caa3f18c4 Mon Sep 17 00:00:00 2001 From: Bill Chen Date: Thu, 25 Apr 2019 08:49:49 -0700 Subject: [PATCH 509/526] cppcheck: Fix various issues found in iocore/eventsystem --- iocore/eventsystem/IOBuffer.cc | 4 +--- iocore/eventsystem/I_Continuation.h | 4 ++-- iocore/eventsystem/I_EThread.h | 1 + iocore/eventsystem/I_EventProcessor.h | 2 +- iocore/eventsystem/I_Lock.h | 2 +- iocore/eventsystem/I_MIOBufferWriter.h | 2 +- iocore/eventsystem/I_VConnection.h | 10 +++++----- iocore/eventsystem/I_VIO.h | 2 +- iocore/eventsystem/P_Freer.h | 11 +++++++---- iocore/eventsystem/P_UnixSocketManager.h | 8 +++----- iocore/eventsystem/ProtectedQueue.cc | 2 +- iocore/eventsystem/SocketManager.cc | 4 +--- iocore/eventsystem/UnixEThread.cc | 8 ++++---- 13 files changed, 29 insertions(+), 31 deletions(-) diff --git a/iocore/eventsystem/IOBuffer.cc b/iocore/eventsystem/IOBuffer.cc index 1fcd758de89..844a5af16e5 100644 --- a/iocore/eventsystem/IOBuffer.cc +++ b/iocore/eventsystem/IOBuffer.cc @@ -46,8 +46,6 @@ int64_t max_iobuffer_size = DEFAULT_BUFFER_SIZES - 1; void init_buffer_allocators(int iobuffer_advice) { - char *name; - for (int i = 0; i < DEFAULT_BUFFER_SIZES; i++) { int64_t s = DEFAULT_BUFFER_BASE_SIZE * (((int64_t)1) << i); int64_t a = DEFAULT_BUFFER_ALIGNMENT; @@ -56,7 +54,7 @@ init_buffer_allocators(int iobuffer_advice) a = s; } - name = new char[64]; + auto name = new char[64]; snprintf(name, 64, "ioBufAllocator[%d]", i); ioBufAllocator[i].re_init(name, s, n, a, iobuffer_advice); } diff --git a/iocore/eventsystem/I_Continuation.h b/iocore/eventsystem/I_Continuation.h index b4531a95415..b34f78d2525 100644 --- a/iocore/eventsystem/I_Continuation.h +++ b/iocore/eventsystem/I_Continuation.h @@ -198,8 +198,8 @@ class Continuation : private force_VFPT_to_top @param amutex Lock to be set for this Continuation. */ - Continuation(ProxyMutex *amutex = nullptr); - Continuation(Ptr &amutex); + explicit Continuation(ProxyMutex *amutex = nullptr); + explicit Continuation(Ptr &amutex); }; /** diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index 7e86c8e0649..c70fb04050f 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -358,6 +358,7 @@ class EThread : public Thread */ class DefaultTailHandler : public LoopTailHandler { + // cppcheck-suppress noExplicitConstructor; allow implicit conversion DefaultTailHandler(ProtectedQueue &q) : _q(q) {} int diff --git a/iocore/eventsystem/I_EventProcessor.h b/iocore/eventsystem/I_EventProcessor.h index 031ca71d2ac..a15d406f96c 100644 --- a/iocore/eventsystem/I_EventProcessor.h +++ b/iocore/eventsystem/I_EventProcessor.h @@ -391,7 +391,7 @@ class EventProcessor : public Processor EventProcessor *_evp; public: - ThreadInit(EventProcessor *evp) : _evp(evp) { SET_HANDLER(&self::init); } + explicit ThreadInit(EventProcessor *evp) : _evp(evp) { SET_HANDLER(&self::init); } int init(int /* event ATS_UNUSED */, Event *ev) diff --git a/iocore/eventsystem/I_Lock.h b/iocore/eventsystem/I_Lock.h index c061d266b16..b5bef2c40e0 100644 --- a/iocore/eventsystem/I_Lock.h +++ b/iocore/eventsystem/I_Lock.h @@ -245,7 +245,7 @@ Mutex_trylock( Ptr &m, EThread *t) { ink_assert(t != nullptr); - ink_assert(t == (EThread *)this_thread()); + ink_assert(t == reinterpret_cast(this_thread())); if (m->thread_holding != t) { if (!ink_mutex_try_acquire(&m->the_mutex)) { #ifdef DEBUG diff --git a/iocore/eventsystem/I_MIOBufferWriter.h b/iocore/eventsystem/I_MIOBufferWriter.h index 639f4615482..856881626a6 100644 --- a/iocore/eventsystem/I_MIOBufferWriter.h +++ b/iocore/eventsystem/I_MIOBufferWriter.h @@ -42,7 +42,7 @@ class MIOBufferWriter : public ts::BufferWriter using self_type = MIOBufferWriter; ///< Self reference type. public: - MIOBufferWriter(MIOBuffer *miob) : _miob(miob) {} + explicit MIOBufferWriter(MIOBuffer *miob) : _miob(miob) {} self_type &write(const void *data_, size_t length) override; diff --git a/iocore/eventsystem/I_VConnection.h b/iocore/eventsystem/I_VConnection.h index 193c62bd3d4..80a7218f08e 100644 --- a/iocore/eventsystem/I_VConnection.h +++ b/iocore/eventsystem/I_VConnection.h @@ -313,8 +313,8 @@ class VConnection : public Continuation */ virtual void do_io_shutdown(ShutdownHowTo_t howto) = 0; - VConnection(ProxyMutex *aMutex); - VConnection(Ptr &aMutex); + explicit VConnection(ProxyMutex *aMutex); + explicit VConnection(Ptr &aMutex); // Private // Set continuation on a given vio. The public interface @@ -389,8 +389,8 @@ class AnnotatedVConnection : public VConnection using super_type = VConnection; public: - AnnotatedVConnection(ProxyMutex *aMutex) : super_type(aMutex){}; - AnnotatedVConnection(Ptr &aMutex) : super_type(aMutex){}; + explicit AnnotatedVConnection(ProxyMutex *aMutex) : super_type(aMutex){}; + explicit AnnotatedVConnection(Ptr &aMutex) : super_type(aMutex){}; void * get_user_arg(unsigned ix) const @@ -440,5 +440,5 @@ struct DummyVConnection : public AnnotatedVConnection { "cannot use default implementation"); } - DummyVConnection(ProxyMutex *m) : AnnotatedVConnection(m) {} + explicit DummyVConnection(ProxyMutex *m) : AnnotatedVConnection(m) {} }; diff --git a/iocore/eventsystem/I_VIO.h b/iocore/eventsystem/I_VIO.h index 40d8fa444a6..e31c5ddb247 100644 --- a/iocore/eventsystem/I_VIO.h +++ b/iocore/eventsystem/I_VIO.h @@ -139,7 +139,7 @@ class VIO */ inkcoreapi void reenable_re(); - VIO(int aop); + explicit VIO(int aop); VIO(); enum { diff --git a/iocore/eventsystem/P_Freer.h b/iocore/eventsystem/P_Freer.h index 374af33622b..deb916d4e8f 100644 --- a/iocore/eventsystem/P_Freer.h +++ b/iocore/eventsystem/P_Freer.h @@ -44,7 +44,7 @@ template struct DeleterContinuation : public Continuation { delete this; return EVENT_DONE; } - DeleterContinuation(C *ap) : Continuation(new_ProxyMutex()), p(ap) { SET_HANDLER(&DeleterContinuation::dieEvent); } + explicit DeleterContinuation(C *ap) : Continuation(new_ProxyMutex()), p(ap) { SET_HANDLER(&DeleterContinuation::dieEvent); } }; // This can be useful for two things (or both): @@ -73,7 +73,7 @@ template struct FreeCallContinuation : public Continuation { delete this; return EVENT_DONE; } - FreeCallContinuation(C *ap) : Continuation(nullptr), p(ap) { SET_HANDLER(&FreeCallContinuation::dieEvent); } + explicit FreeCallContinuation(C *ap) : Continuation(nullptr), p(ap) { SET_HANDLER(&FreeCallContinuation::dieEvent); } }; template @@ -99,7 +99,10 @@ struct FreerContinuation : public Continuation { return EVENT_DONE; } - FreerContinuation(void *ap) : Continuation(nullptr), p(ap) { SET_HANDLER((FreerContHandler)&FreerContinuation::dieEvent); } + explicit FreerContinuation(void *ap) : Continuation(nullptr), p(ap) + { + SET_HANDLER((FreerContHandler)&FreerContinuation::dieEvent); + } }; TS_INLINE void @@ -123,7 +126,7 @@ template struct DereferContinuation : public Continuation { return EVENT_DONE; } - DereferContinuation(C *ap) : Continuation(nullptr), p(ap) { SET_HANDLER(&DereferContinuation::dieEvent); } + explicit DereferContinuation(C *ap) : Continuation(nullptr), p(ap) { SET_HANDLER(&DereferContinuation::dieEvent); } }; template diff --git a/iocore/eventsystem/P_UnixSocketManager.h b/iocore/eventsystem/P_UnixSocketManager.h index b90b2b1a13b..dc5c0c911ec 100644 --- a/iocore/eventsystem/P_UnixSocketManager.h +++ b/iocore/eventsystem/P_UnixSocketManager.h @@ -114,14 +114,12 @@ SocketManager::vector_io(int fd, struct iovec *vector, size_t count, int read_re { const int max_iovecs_per_request = 16; int n; - int64_t r = 0; + int64_t r; int n_vec; int64_t bytes_xfered = 0; - int current_count; - int64_t current_request_bytes; for (n_vec = 0; n_vec < (int)count; n_vec += max_iovecs_per_request) { - current_count = std::min(max_iovecs_per_request, ((int)(count - n_vec))); + int current_count = std::min(max_iovecs_per_request, ((int)(count - n_vec))); do { // coverity[tainted_data_argument] r = read_request ? ::readv(fd, &vector[n_vec], current_count) : ::writev(fd, &vector[n_vec], current_count); @@ -141,7 +139,7 @@ SocketManager::vector_io(int fd, struct iovec *vector, size_t count, int read_re } // Compute bytes in current vector - current_request_bytes = 0; + int64_t current_request_bytes = 0; for (n = n_vec; n < (n_vec + current_count); ++n) { current_request_bytes += vector[n].iov_len; } diff --git a/iocore/eventsystem/ProtectedQueue.cc b/iocore/eventsystem/ProtectedQueue.cc index bd434e77325..e8741c79a89 100644 --- a/iocore/eventsystem/ProtectedQueue.cc +++ b/iocore/eventsystem/ProtectedQueue.cc @@ -93,7 +93,7 @@ ProtectedQueue::dequeue_timed(ink_hrtime cur_time, ink_hrtime timeout, bool slee void ProtectedQueue::dequeue_external() { - Event *e = (Event *)ink_atomiclist_popall(&al); + Event *e = static_cast(ink_atomiclist_popall(&al)); // invert the list, to preserve order SLL l, t; t.head = e; diff --git a/iocore/eventsystem/SocketManager.cc b/iocore/eventsystem/SocketManager.cc index 24bc38bc20a..6dffbe11e42 100644 --- a/iocore/eventsystem/SocketManager.cc +++ b/iocore/eventsystem/SocketManager.cc @@ -66,10 +66,8 @@ accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags) int SocketManager::accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags) { - int fd; - do { - fd = ::accept4(s, addr, addrlen, flags); + int fd = ::accept4(s, addr, addrlen, flags); if (likely(fd >= 0)) { return fd; } diff --git a/iocore/eventsystem/UnixEThread.cc b/iocore/eventsystem/UnixEThread.cc index a57932295a5..03b31c3eeb3 100644 --- a/iocore/eventsystem/UnixEThread.cc +++ b/iocore/eventsystem/UnixEThread.cc @@ -193,16 +193,16 @@ EThread::execute_regular() { Event *e; Que(Event, link) NegativeQueue; - ink_hrtime next_time = 0; - ink_hrtime delta = 0; // time spent in the event loop + ink_hrtime next_time; + ink_hrtime delta; // time spent in the event loop ink_hrtime loop_start_time; // Time the loop started. ink_hrtime loop_finish_time; // Time at the end of the loop. // Track this so we can update on boundary crossing. EventMetrics *prev_metric = this->prev(metrics + (ink_get_hrtime_internal() / HRTIME_SECOND) % N_EVENT_METRICS); - int nq_count = 0; - int ev_count = 0; + int nq_count; + int ev_count; // A statically initialized instance we can use as a prototype for initializing other instances. static EventMetrics METRIC_INIT; From 69b5748e8b8fba8d9638fcef839118d2036c3bdd Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Mon, 22 Apr 2019 19:50:06 -0700 Subject: [PATCH 510/526] correctly handle return value 0 from recv() --- iocore/dns/DNS.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 509ae2970e9..b2629b7cd30 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -808,10 +808,10 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) // see if TS gets a two-byte size uint16_t tmp = 0; res = socketManager.recv(dnsc->fd, &tmp, sizeof(tmp), MSG_PEEK); - if (res == -EAGAIN || res == 0 || res == 1) { + if (res == -EAGAIN || res == 1) { break; } - if (res < 0) { + if (res <= 0) { goto Lerror; } // reading total size @@ -830,10 +830,10 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) // continue reading data void *buf_start = (char *)dnsc->tcp_data.buf_ptr->buf + dnsc->tcp_data.done_reading; res = socketManager.recv(dnsc->fd, buf_start, dnsc->tcp_data.total_length - dnsc->tcp_data.done_reading, 0); - if (res == -EAGAIN || res == 0) { + if (res == -EAGAIN) { break; } - if (res < 0) { + if (res <= 0) { goto Lerror; } Debug("dns", "received packet size = %d over TCP", res); From d611b54aadd399641e009fde77017fed4d2da01f Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 23 Apr 2019 16:42:31 +0000 Subject: [PATCH 511/526] Off by 1 error in memcpy of redirect url --- proxy/http/HttpSM.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 533db3f4c87..a01a12039f1 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -7733,7 +7733,9 @@ HttpSM::redirect_request(const char *arg_redirect_url, const int arg_redirect_le // Prepend a slash and parse again. char redirect_url_leading_slash[arg_redirect_len + 1]; redirect_url_leading_slash[0] = '/'; - memcpy(redirect_url_leading_slash + 1, arg_redirect_url, arg_redirect_len + 1); + if (arg_redirect_len > 0) { + memcpy(redirect_url_leading_slash + 1, arg_redirect_url, arg_redirect_len); + } url_nuke_proxy_stuff(redirectUrl.m_url_impl); redirectUrl.parse(redirect_url_leading_slash, arg_redirect_len + 1); } From 5be013418a71a85d0b5dba2c956be75a9942526b Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Thu, 18 Apr 2019 18:02:26 +0000 Subject: [PATCH 512/526] MIMEScanner: only clear m_line buffer if at MIME_PARSE_BEFORE state --- proxy/hdrs/MIME.cc | 4 ++-- proxy/hdrs/unit_tests/test_Hdrs.cc | 31 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/proxy/hdrs/MIME.cc b/proxy/hdrs/MIME.cc index c23e2646f82..73d5b2bec74 100644 --- a/proxy/hdrs/MIME.cc +++ b/proxy/hdrs/MIME.cc @@ -2334,6 +2334,7 @@ MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, b while (PARSE_RESULT_CONT == zret && !text.empty()) { switch (m_state) { case MIME_PARSE_BEFORE: // waiting to find a field. + m_line.resize(0); // any caller should already be done with the buffer if (ParseRules::is_cr(*text)) { ++text; if (!text.empty() && ParseRules::is_lf(*text)) { @@ -2445,8 +2446,7 @@ MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, b output_shares_input = true; if (PARSE_RESULT_CONT != zret) { if (!m_line.empty()) { - output = m_line; - m_line.resize(0); // depending resize(0) not deallocating internal string memory. + output = m_line; // cleared when called with state MIME_PARSE_BEFORE output_shares_input = false; } else { output = parsed_text; diff --git a/proxy/hdrs/unit_tests/test_Hdrs.cc b/proxy/hdrs/unit_tests/test_Hdrs.cc index 790418c45c4..a471339f8ad 100644 --- a/proxy/hdrs/unit_tests/test_Hdrs.cc +++ b/proxy/hdrs/unit_tests/test_Hdrs.cc @@ -84,3 +84,34 @@ TEST_CASE("HdrTest", "[proxy][hdrtest]") req_hdr.destroy(); } } + +TEST_CASE("MIMEScanner_fragments", "[proxy][mimescanner_fragments]") +{ + constexpr ts::TextView const message = "GET /index.html HTTP/1.0\r\n"; + + struct Fragment { + ts::TextView msg; + bool shares_input; + int expected_result; + }; + constexpr std::array const fragments = {{ + {message.substr(0, 11), true, PARSE_RESULT_CONT}, + {message.substr(11, 11), true, PARSE_RESULT_CONT}, + {message.substr(22), false, PARSE_RESULT_OK}, + }}; + + MIMEScanner scanner; + ts::TextView output; // only set on last call + + for (auto const &frag : fragments) { + ts::TextView input = frag.msg; + bool got_shares_input = !frag.shares_input; + constexpr bool const is_eof = false; + ParseResult const got_res = scanner.get(input, output, got_shares_input, is_eof, MIMEScanner::LINE); + + REQUIRE(frag.expected_result == got_res); + REQUIRE(frag.shares_input == got_shares_input); + } + + REQUIRE(message == output); +} From 0ee898d63e65ee8ccafc3bd55b85b2f25aa4ef2d Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Thu, 25 Apr 2019 04:50:25 -0500 Subject: [PATCH 513/526] Clang Analyzer: Fix IpMap.cc false positives. --- src/tscore/IpMap.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/tscore/IpMap.cc b/src/tscore/IpMap.cc index 2fd01f8c937..89f11434986 100644 --- a/src/tscore/IpMap.cc +++ b/src/tscore/IpMap.cc @@ -382,8 +382,11 @@ namespace detail if (n->_data == payload) { if (x) { if (n->_max <= max) { - // next range is covered, so we can remove and continue. - this->remove(n); +// next range is covered, so we can remove and continue. +#if defined(__clang_analyzer__) + ink_assert(x != n) +#endif + this->remove(n); n = next(x); } else if (n->_min <= max_plus1) { // Overlap or adjacent with larger max - absorb and finish. @@ -642,10 +645,14 @@ namespace detail void IpMapBase::insert_before(N *spot, N *n) { - N *c = left(spot); - if (!c) { + if (left(spot) == nullptr) { spot->setChild(n, N::LEFT); } else { +// If there's a left child, there's a previous node, therefore spot->_prev is valid. +// Clang analyzer doesn't realize this so it generates a false positive. +#if defined(__clang_analyzer__) + ink_assert(spot->_prev != nullptr); +#endif spot->_prev->setChild(n, N::RIGHT); } From 8246a4e178293831a814a71752d02f9954ef0f09 Mon Sep 17 00:00:00 2001 From: Steven Feltner Date: Tue, 23 Apr 2019 15:58:17 +0800 Subject: [PATCH 514/526] Added user defined conversion operator in ConstBuffer for string_view. --- include/tscore/TsBuffer.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/tscore/TsBuffer.h b/include/tscore/TsBuffer.h index 39e55b8a98b..7837c64352b 100644 --- a/include/tscore/TsBuffer.h +++ b/include/tscore/TsBuffer.h @@ -33,6 +33,7 @@ // For memcmp() #include +#include /// Apache Traffic Server commons. namespace ts @@ -194,6 +195,8 @@ struct ConstBuffer { /// @return @c true if the buffer has a non-zero pointer @b and size. operator pseudo_bool() const; + operator std::string_view() const { return {_ptr, _size}; } + /// @name Accessors. //@{ /// Get the data in the buffer. From 0fe89372bf4e9881df1b37ddf0c6979c8b4f2feb Mon Sep 17 00:00:00 2001 From: hankai17 <867614535@qq.com> Date: Tue, 30 Apr 2019 22:30:47 +0800 Subject: [PATCH 515/526] Update jtest.cc --- tools/jtest/jtest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jtest/jtest.cc b/tools/jtest/jtest.cc index da22fe1202a..4253d04bbc7 100644 --- a/tools/jtest/jtest.cc +++ b/tools/jtest/jtest.cc @@ -1806,7 +1806,7 @@ poll_loop() if (ip >= POLL_GROUP_SIZE || i == last_fd) { int n = poll(pfd, ip, POLL_TIMEOUT); if (n > 0) { - for (int j = 0; j < ip; j++) { + for (int j = 0; j < n; j++) { if (pfd[j].revents & (POLLIN | POLLERR | POLLHUP | POLLNVAL)) { if (verbose) { printf("poll read %d %X\n", pfd[j].fd, pfd[j].revents); From e68b191d04e0df1d2be51196acb8779431d7e9cc Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 30 Apr 2019 10:19:15 -0600 Subject: [PATCH 516/526] Tries to make builds on older clang happy --- .../slice/unit-tests/test_config.cc | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/plugins/experimental/slice/unit-tests/test_config.cc b/plugins/experimental/slice/unit-tests/test_config.cc index fcd1b8e051c..b85f381e959 100644 --- a/plugins/experimental/slice/unit-tests/test_config.cc +++ b/plugins/experimental/slice/unit-tests/test_config.cc @@ -36,13 +36,23 @@ TEST_CASE("config default", "[AWS][slice][utility]") TEST_CASE("config bytesfrom valid parsing", "[AWS][slice][utility]") { - static std::array const teststrings = { - "1000", "1m", "5g", "2k", "3kb", "1z", - }; + static std::array const teststrings = {{ + "1000", + "1m", + "5g", + "2k", + "3kb", + "1z", + }}; - constexpr std::array const expvals = { - 1000, 1024 * 1024, int64_t(1024) * 1024 * 1024 * 5, 1024 * 2, 1024 * 3, 1, - }; + constexpr std::array const expvals = {{ + 1000, + 1024 * 1024, + int64_t(1024) * 1024 * 1024 * 5, + 1024 * 2, + 1024 * 3, + 1, + }}; for (size_t index = 0; index < teststrings.size(); ++index) { std::string const &teststr = teststrings[index]; @@ -58,13 +68,13 @@ TEST_CASE("config bytesfrom valid parsing", "[AWS][slice][utility]") TEST_CASE("config bytesfrom invalid parsing", "[AWS][slice][utility]") { - static std::array const badstrings = { + static std::array const badstrings = {{ "abc", // alpha "g00", // giga "M00", // mega "k00", // kilo "-500", // negative - }; + }}; for (std::string const &badstr : badstrings) { int64_t const val = Config::bytesFrom(badstr.c_str()); From 05398aa981f37a2b67a85d962b56b8563c2ce689 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Mon, 29 Apr 2019 18:21:50 -0500 Subject: [PATCH 517/526] ssl_session_resuse: operator for redis endpoint compare functor must be const for STL compatability. --- plugins/experimental/ssl_session_reuse/src/redis_endpoint.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h b/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h index 77584e4f2ac..2762d9b80fc 100644 --- a/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h +++ b/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h @@ -38,7 +38,7 @@ typedef struct redis_endpoint { typedef struct redis_endpoint_compare { bool - operator()(const RedisEndpoint &lhs, const RedisEndpoint &rhs) + operator()(const RedisEndpoint &lhs, const RedisEndpoint &rhs) const { return lhs.m_hostname < rhs.m_hostname || (lhs.m_hostname == rhs.m_hostname && lhs.m_port < rhs.m_port); } From a34a1227cea1d66f28eba34c1165205f108d0371 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Tue, 30 Apr 2019 07:30:22 -0600 Subject: [PATCH 518/526] Removes code related to removed configuration --- tests/README.md | 1 - tests/gold_tests/autest-site/trafficserver.test.ext | 3 --- 2 files changed, 4 deletions(-) diff --git a/tests/README.md b/tests/README.md index 84fb30f5c30..e9ff4424faa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -67,7 +67,6 @@ These are the current variables that are defined dynamically for Trafficserver port - the ipv4 port to listen on portv6 - the ipv6 port to listen on -manager_port - the manager port used. This is set even is select_port is False admin_port - the admin port used. This is set even is select_port is False #### File objects diff --git a/tests/gold_tests/autest-site/trafficserver.test.ext b/tests/gold_tests/autest-site/trafficserver.test.ext index 64ca9221900..7ae7626440a 100755 --- a/tests/gold_tests/autest-site/trafficserver.test.ext +++ b/tests/gold_tests/autest-site/trafficserver.test.ext @@ -254,15 +254,12 @@ def MakeATSProcess(obj, name, command='traffic_server', select_ports=True): else: p.Variables.port = 8080 p.Variables.portv6 = 8080 - get_port(p, "manager_port") get_port(p, "admin_port") # set the ports if select_ports: p.Env['PROXY_CONFIG_HTTP_SERVER_PORTS'] = "{0} {1}:ipv6".format( p.Variables.port, p.Variables.portv6) # your own listen port - p.Env['PROXY_CONFIG_PROCESS_MANAGER_MGMT_PORT'] = str( - p.Variables.manager_port) p.Env['PROXY_CONFIG_ADMIN_SYNTHETIC_PORT'] = str(p.Variables.admin_port) p.Env['PROXY_CONFIG_ADMIN_AUTOCONF_PORT'] = str( p.Variables.admin_port) # support pre ATS 6.x From c3517301215b29d24489a96b219d78f933bbcaa1 Mon Sep 17 00:00:00 2001 From: Ben Williams Date: Wed, 1 May 2019 14:48:36 -0500 Subject: [PATCH 519/526] Update README.md Updated all headers test to not use netstat. Fixed README.md --- tests/README.md | 22 +++++++++++++------- tests/gold_tests/logging/all_headers.test.py | 8 ++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/README.md b/tests/README.md index e9ff4424faa..0e092665de1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,13 +1,15 @@ # Getting Started -This directory contains different tests for Apache Trafficserver. It is recommended that all test move to this common area under the correct location based on the type of test being added. +This directory contains different tests for Apache Trafficserver. It is recommended that all tests move to this common area under the correct location based on the type of test being added. ## Layout The current layout is: **gold_tests/** - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest) + **tools/** - contains programs used to help with testing. + **include/** - contains headers used for unit testing. ## Scripts @@ -15,24 +17,28 @@ The current layout is: To help with easy running of the tests, there is autest.sh and bootstrap.py. ### autest.sh -This file is a simple wrapper that will call the Reusable Gold Testing System (Autest) program in a python virtualenv. If the virtualenv is not setup, the script will try to install it on the system. That will set up the Autest on most systems in a Python virtual environment. The wrapper add some basic options to the command to point to the location of the tests. Use --help for more details on options for running Autest. +This file is a simple wrapper that will call the Reusable Gold Testing System (Autest) program in a python virtualenv. If the virtualenv is not setup, the script will try to install it on the system. That will set up the Autest on most systems in a Python virtual environment. The wrapper adds some basic options to the command to point to the location of the tests. Use --help for more details on options for running Autest. ### bootstrap.py This script will try to install python35 or better on the system, and the needed python packages for running the tests. +# Basic setup + +AuTest can be run using the script file autest.sh listed above. Run the file from the tests/ directory followed by --ats-bin and the bin name. (ie ~/ats/bin) This will run the wrapper for the tests. See documentation for more details. + # Advanced setup -AuTest and the relevant tools can be install manually instead of using the wrapper script. The advange of this is that it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done. -1. run the bootstrap script then source the path with a "source ./env-test/bin/activate" command. At this point autest command should run without the wrapper script -2. The other way is to make sure you install python 3.5 or better on your system. From there install these python packages ( ie pip install ): +AuTest and the relevant tools can be install manually instead of using the wrapper script. By doing this, it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done. +1. Run the bootstrap script then source the path with a "source ./env-test/bin/activate" command. At this point autest command should run without the wrapper script +2. Make sure you install python 3.5 or better on your system. From there install these python packages ( ie pip install ): - hyper - git+https://bitbucket.org/autestsuite/reusable-gold-testing-system.git - [traffic-replay](https://bitbucket.org/autestsuite/trafficreplay/src/master/) (This will automatically install [MicroDNS](https://bitbucket.org/autestsuite/microdns/src/master/), [MicroServer](https://bitbucket.org/autestsuite/microserver/src/master/), [TrafficReplayLibrary](https://bitbucket.org/autestsuite/trafficreplaylibrary/src/master/), and dnslib as part of the dependencies.) -# Writting tests for AuTest -When writting for the AuTest system please refer to the current documenation on the [online wiki](https://bitbucket.org/dragon512/reusable-gold-testing-system/wiki/Home) for general use of the system. +# Writing tests for AuTest +When writing for the AuTest system please refer to the current [Online Documentation](https://autestsuite.bitbucket.io/) for general use of the system. -## Documenation of AuTest extension for ATS. +## Documentation of AuTest extension for ATS. Autest allows for the creation of extensions to help specialize and simplify test writing for a given application domain. Minus API addition the extension code will check that python 3.5 or better is used. There is also a new command line argumented added specifically for Trafficserver: --ats-bin < path to bin directory > diff --git a/tests/gold_tests/logging/all_headers.test.py b/tests/gold_tests/logging/all_headers.test.py index 8279ae36b78..cdb62758c52 100644 --- a/tests/gold_tests/logging/all_headers.test.py +++ b/tests/gold_tests/logging/all_headers.test.py @@ -69,10 +69,6 @@ Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'test_all_headers.log.san'), exists=True, content='gold/test_all_headers.gold') -# Ask the OS if the port is ready for connect() -# -def CheckPort(Port): - return lambda: 0 == subprocess.call('netstat --listen --tcp -n | grep -q :{}'.format(Port), shell=True) def reallyLong(): value = 'abcdefghijklmnop' @@ -85,8 +81,8 @@ def reallyLong(): return retval tr = Test.AddTestRun() -tr.Processes.Default.StartBefore(server, ready=CheckPort(server.Variables.Port)) -tr.Processes.Default.StartBefore(Test.Processes.ts, ready=CheckPort(ts.Variables.port)) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts) tr.Processes.Default.Command = ( 'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() ) From 92ecbcbe194621a0f2f2d1fddc5e143f8390e9c2 Mon Sep 17 00:00:00 2001 From: Walter Karas Date: Wed, 1 May 2019 15:30:28 -0500 Subject: [PATCH 520/526] TS C++ API: add member function TSSslConnection InterceptPlugin::getSslConnection() --- include/tscpp/api/InterceptPlugin.h | 6 ++++++ src/tscpp/api/InterceptPlugin.cc | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/include/tscpp/api/InterceptPlugin.h b/include/tscpp/api/InterceptPlugin.h index 62eb2340115..d7d6597b4e5 100644 --- a/include/tscpp/api/InterceptPlugin.h +++ b/include/tscpp/api/InterceptPlugin.h @@ -68,6 +68,12 @@ class InterceptPlugin : public TransactionPlugin /** Should be called only after request header has completely been consumed */ Headers &getRequestHeaders(); + /** + * The returned TSSslConnection would be nullptr if the connection is not a TLS connection + * or Intercept plugin is not ready. + */ + TSSslConnection getSslConnection(); + ~InterceptPlugin() override; struct State; /** Internal use only */ diff --git a/src/tscpp/api/InterceptPlugin.cc b/src/tscpp/api/InterceptPlugin.cc index db0bd00a700..b3ad83bb1b0 100644 --- a/src/tscpp/api/InterceptPlugin.cc +++ b/src/tscpp/api/InterceptPlugin.cc @@ -181,6 +181,17 @@ InterceptPlugin::getRequestHeaders() return state_->request_headers_; } +TSSslConnection +InterceptPlugin::getSslConnection() +{ + if (!state_->net_vc_) { + LOG_ERROR("Intercept Plugin is not ready to provide SSL Connection"); + return nullptr; + } + + return TSVConnSSLConnectionGet(state_->net_vc_); +} + bool InterceptPlugin::doRead() { From 37661eb4bd512a1ba7278979140d963dd864cd7c Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Tue, 23 Apr 2019 16:22:59 +0800 Subject: [PATCH 521/526] Fixed clang-analyzer issues in cookie_remap --- .../experimental/cookie_remap/Makefile.inc | 4 +- .../cookie_remap/{strip.c => strip.cc} | 113 +++++++++--------- 2 files changed, 56 insertions(+), 61 deletions(-) rename plugins/experimental/cookie_remap/{strip.c => strip.cc} (71%) diff --git a/plugins/experimental/cookie_remap/Makefile.inc b/plugins/experimental/cookie_remap/Makefile.inc index 4af577dd3d4..266cc9e4072 100644 --- a/plugins/experimental/cookie_remap/Makefile.inc +++ b/plugins/experimental/cookie_remap/Makefile.inc @@ -19,7 +19,7 @@ pkglib_LTLIBRARIES += experimental/cookie_remap/cookie_remap.la experimental_cookie_remap_cookie_remap_la_SOURCES = \ experimental/cookie_remap/cookie_remap.cc \ experimental/cookie_remap/hash.c \ - experimental/cookie_remap/strip.c \ + experimental/cookie_remap/strip.cc \ experimental/cookie_remap/cookiejar.cc experimental_cookie_remap_cookie_remap_la_LDFLAGS = \ @@ -33,7 +33,7 @@ check_PROGRAMS += \ experimental_cookie_remap_test_cookiejar_CPPFLAGS = $(AM_CPPFLAGS) -Iexperimental/cookie_remap -I$(abs_top_srcdir)/tests/include experimental_cookie_remap_test_cookiejar_SOURCES = \ experimental/cookie_remap/test_cookiejar.cc \ - experimental/cookie_remap/strip.c \ + experimental/cookie_remap/strip.cc \ experimental/cookie_remap/cookiejar.cc \ experimental/cookie_remap/cookiejar.h diff --git a/plugins/experimental/cookie_remap/strip.c b/plugins/experimental/cookie_remap/strip.cc similarity index 71% rename from plugins/experimental/cookie_remap/strip.c rename to plugins/experimental/cookie_remap/strip.cc index d5965730c8a..0302f536b2b 100644 --- a/plugins/experimental/cookie_remap/strip.c +++ b/plugins/experimental/cookie_remap/strip.cc @@ -31,58 +31,52 @@ static int copy_whitespace(const char **r, const char *in_end, char **w, const c static int strip_whitespace(const char **r, const char **in_end); -/* Determine if there is room to store len bytes starting at p for an - * object that ends at maxp. This is not as simple as a less-than - * comparison, because our code may increment p well beyond the end of - * the object it originally pointed to (in complete violation of what - * ANSI C says is legitimate). The result is that p may wrap around. - * This has been observed with using stack buffers as arguments - * from 32 bit programs running on 64-bit RHEL. - */ -#define ROOM(p, len, maxp) (((maxp) - ((p) + (len))) >= 0) +// Determine if there is room to store len bytes starting at p for an +// object that ends at maxp. This is not as simple as a less-than +// comparison, because our code may increment p well beyond the end of +// the object it originally pointed to (in complete violation of what +// ANSI C says is legitimate). The result is that p may wrap around. +// This has been observed with using stack buffers as arguments +// from 32 bit programs running on 64-bit RHEL. +static bool +room(const char *p, const int len, const char *maxp) +{ + return ((maxp - (p + len)) >= 0); +} -/* write c into *p if there's room, always incrementing *p. This - * implementation uses a do-loop to avoid several syntactic issues - * when this macro is expanded in the context of if-then-else constructs. - * It is expected that the compiler will optimize away the "while (0)" - */ -#define WRITE_C_IF_ROOM(p, maxp, c) \ - do { \ - if (ROOM(*(p), 1, (maxp))) \ - **(p) = (c); \ - (*(p))++; \ - } while (0) - -/* write s into *p if there's room, always adding slen to *p . This - * implementation uses a do-loop to avoid several syntactic issues - * when this macro is expanded in the context of if-then-else constructs. - * It is expected that the compiler will optimize away the "while 0" - */ -#define WRITE_STR_IF_ROOM(p, maxp, s, slen) \ - do { \ - if (ROOM(*(p), (slen), (maxp))) \ - memcpy(*(p), (s), (slen)); \ - *(p) += (slen); \ - } while (0) - -/* Write count spaces into *p if there's room, always adding count to *p. - * The count argument is set to zero at the end of execution. This - * implementation uses a do-loop to avoid several syntactic issues when - * this macro is expanded in the context of if-then-else constructs. - * It is expected that the compiler will optimize away the "while 0" - */ -#define WRITE_SPACES_IF_ROOM(p, maxp, slen) \ - do { \ - if (ROOM(*(p), (slen), (maxp))) \ - memset(*(p), ' ', (slen)); \ - *(p) += (slen); \ - (slen) = 0; \ - } while (0) +// write c into *p if there's room, always incrementing *p. +static void +write_char_if_room(char **p, const char *maxp, const char c) +{ + if (p == nullptr || *p != nullptr) { + return; + } -/* - * File-scope data - */ + if (room(*p, 1, maxp)) { + **p = c; + } + (*p)++; +} + +/// Write count spaces into *p if there's room, always adding count to *p. +// The count argument is set to zero at the end of execution. +static void +write_spaces_if_room(char **p, const char *maxp, int &slen) +{ + if (p == nullptr || *p != nullptr) { + return; + } + + if (room(*p, slen, maxp)) { + memset(*p, ' ', slen); + } + + *p += slen; + slen = 0; +} + +// File-scope data static const unsigned int allowed_flags = (STRIP_FLAG_LEAVE_WHITESP | STRIP_FLAG_STRIP_LOW | STRIP_FLAG_STRIP_HIGH | STRIP_FLAG_UNSAFE_QUOTES | STRIP_FLAG_UNSAFE_SLASHES | STRIP_FLAG_UNSAFE_SPACES); @@ -98,7 +92,7 @@ stripped_core(const char *r, const char *in_end, char **w, const char *out_end, /* parse the string, stripping risky characters/sequences */ for (/* already established */; r < in_end; r++) { - unsigned char c = *r; + unsigned char c = static_cast(*r); if (in_tag) { switch (c) { case '>': @@ -171,17 +165,18 @@ stripped_core(const char *r, const char *in_end, char **w, const char *out_end, space = 1; /* replace stripped sequence with space */ } stripped = 0; /* reset until next stripped sequence */ - WRITE_SPACES_IF_ROOM(w, out_end, space); + write_spaces_if_room(w, out_end, space); /* Process as single character. */ - WRITE_C_IF_ROOM(w, out_end, c); + write_char_if_room(w, out_end, c); } } } /* Restore trailing whitespace if asked */ - if (flags & STRIP_FLAG_LEAVE_WHITESP) - WRITE_SPACES_IF_ROOM(w, out_end, space); + if (flags & STRIP_FLAG_LEAVE_WHITESP) { + write_spaces_if_room(w, out_end, space); + } return STRIP_RESULT_OK; } @@ -223,7 +218,7 @@ get_stripped(const char *in, ssize_t in_len, char *out, int *out_len, unsigned i /* handle empty input case (null terminated or not) */ if ((!(flags & STRIP_FLAG_LEAVE_WHITESP) && r >= in_end) || ((flags & STRIP_FLAG_LEAVE_WHITESP) && in_len == 0)) { - WRITE_C_IF_ROOM(&w, out_end, '\0'); /* make out empty string */ + write_char_if_room(&w, out_end, '\0'); /* make out empty string */ *out_len = 1; return STRIP_RESULT_EMPTY_IN; /* input is empty string */ } @@ -232,8 +227,8 @@ get_stripped(const char *in, ssize_t in_len, char *out, int *out_len, unsigned i retval = stripped_core(r, in_end, &w, out_end, flags); /* null terminate */ - out_end += out_end ? 1 : 0; /* undo decrement at start */ - WRITE_C_IF_ROOM(&w, out_end, '\0'); /* try to term at end of output */ + out_end += out_end ? 1 : 0; /* undo decrement at start */ + write_char_if_room(&w, out_end, '\0'); /* try to term at end of output */ /* report the required/used length */ *out_len = w - out; @@ -245,7 +240,7 @@ get_stripped(const char *in, ssize_t in_len, char *out, int *out_len, unsigned i if (retval != STRIP_RESULT_OK) { /* return the empty string on all errors */ - WRITE_C_IF_ROOM(&out, out_end, '\0'); /* make out the empty string */ + write_char_if_room(&out, out_end, '\0'); /* make out the empty string */ if (retval != STRIP_RESULT_OUTLEN_SMALL) { *out_len = 1; /* even if retried, we won't use more than 1 byte */ } @@ -262,7 +257,7 @@ copy_whitespace(const char **r, const char *in_end, char **w, const char *out_en { char c; while (*r < in_end && (c = **r) && (c == ' ' || c == '\t' || c == '\r' || c == '\n')) { - WRITE_C_IF_ROOM(w, out_end, c); + write_char_if_room(w, out_end, c); (*r)++; } return 0; From d2386a5027c186f6c4b5751c858fdb97264cfd23 Mon Sep 17 00:00:00 2001 From: Valentin Gutierrez Date: Wed, 3 Apr 2019 15:38:12 +0200 Subject: [PATCH 522/526] Implement prefetched OCSP stapling responses --- doc/admin-guide/files/records.config.en.rst | 8 ++ .../files/ssl_multicert.config.en.rst | 8 ++ doc/admin-guide/security/index.en.rst | 8 ++ include/ts/ts.h | 2 +- iocore/net/OCSPStapling.cc | 109 ++++++++++++------ iocore/net/P_OCSPStapling.h | 2 +- iocore/net/P_SSLConfig.h | 1 + iocore/net/P_SSLUtils.h | 17 +-- iocore/net/SSLConfig.cc | 4 + iocore/net/SSLUtils.cc | 61 ++++++---- mgmt/RecordsConfig.cc | 4 +- src/traffic_server/InkAPI.cc | 4 +- src/traffic_server/InkAPITest.cc | 2 +- .../autest-site/conditions.test.ext | 4 + tests/gold_tests/tls/ssl/ca.ocsp.key | 27 +++++ tests/gold_tests/tls/ssl/ca.ocsp.pem | 20 ++++ tests/gold_tests/tls/ssl/ocsp_response.der | Bin 0 -> 2266 bytes tests/gold_tests/tls/ssl/responder.ocsp.key | 27 +++++ tests/gold_tests/tls/ssl/responder.ocsp.pem | 22 ++++ tests/gold_tests/tls/ssl/server.ocsp.key | 27 +++++ tests/gold_tests/tls/ssl/server.ocsp.pem | 23 ++++ tests/gold_tests/tls/tls_ocsp.test.py | 73 ++++++++++++ 22 files changed, 383 insertions(+), 70 deletions(-) create mode 100644 tests/gold_tests/tls/ssl/ca.ocsp.key create mode 100644 tests/gold_tests/tls/ssl/ca.ocsp.pem create mode 100644 tests/gold_tests/tls/ssl/ocsp_response.der create mode 100644 tests/gold_tests/tls/ssl/responder.ocsp.key create mode 100644 tests/gold_tests/tls/ssl/responder.ocsp.pem create mode 100644 tests/gold_tests/tls/ssl/server.ocsp.key create mode 100644 tests/gold_tests/tls/ssl/server.ocsp.pem create mode 100644 tests/gold_tests/tls/tls_ocsp.test.py diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 080a99bff17..b0d3c974dfb 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3516,6 +3516,14 @@ OCSP Stapling Configuration Update period (in seconds) for stapling caches. +.. ts:cv:: CONFIG proxy.config.ssl.ocsp.response.path STRING NULL + + The directory path of the prefetched OCSP stapling responses. Change this + variable only if you intend to use and administratively maintain + prefetched OCSP stapling responses. All stapling responses listed in + :file:`ssl_multicert.config` will be loaded relative to this + path. + HTTP/2 Configuration ==================== diff --git a/doc/admin-guide/files/ssl_multicert.config.en.rst b/doc/admin-guide/files/ssl_multicert.config.en.rst index e79ae467965..54a391d3d50 100644 --- a/doc/admin-guide/files/ssl_multicert.config.en.rst +++ b/doc/admin-guide/files/ssl_multicert.config.en.rst @@ -93,6 +93,14 @@ ssl_ca_name=FILENAME (optional) the certificate chain. *FILENAME* is resolved relative to the :ts:cv:`proxy.config.ssl.CA.cert.path` configuration variable. +ssl_ocsp_name=FILENAME (optional) + The name of the file containing the prefetched OCSP stapling response + for this certificate. This field can be omitted to let trafficserver + fetch OCSP responses dynamically. Otherwise, when included, the administrator is + responsible for updating the file's content. *FILENAME* is resolved + relative to the :ts:cv:`proxy.config.ssl.ocsp.response.path` + configuration variable. + ssl_ticket_enabled=1|0 (optional) Enable RFC 5077 stateless TLS session tickets. To support this, OpenSSL should be upgraded to version 0.9.8f or higher. This diff --git a/doc/admin-guide/security/index.en.rst b/doc/admin-guide/security/index.en.rst index 37ed09660cb..84586c2ebf1 100644 --- a/doc/admin-guide/security/index.en.rst +++ b/doc/admin-guide/security/index.en.rst @@ -281,6 +281,13 @@ Authority Information Access field of the signed certificate. For example:: OCSP - URI:http://ocsp.digicert.com CA Issuers - URI:http://cacerts.digicert.com/DigiCertSHA2SecureServerCA.crt +Traffic Server can also use prefetched OCSP stapling responses if ssl_ocsp_name parameter +is used in :file:`ssl_multicert.config`. Take into account that when using prefetched +OCSP stapling responses traffic server will not refresh them and it should be done +externally. This can be done using openssl: + $ openssl ocsp -issuer ca.crt -cert cert.crt -host ocsp.digicert.com:80 \ + -header "Host=ocsp.digicert.com" -respout /var/cache/ocsp/cert.ocsp + Support for OCSP Stapling can be tested using the -status option of the OpenSSL client:: $ openssl s_client -connect mozillalabs.com:443 -status @@ -301,6 +308,7 @@ in :file:`records.config` file: * :ts:cv:`proxy.config.ssl.ocsp.cache_timeout` * :ts:cv:`proxy.config.ssl.ocsp.request_timeout` * :ts:cv:`proxy.config.ssl.ocsp.update_period` +* :ts:cv:`proxy.config.ssl.ocsp.response.path` .. _admin-split-dns: diff --git a/include/ts/ts.h b/include/ts/ts.h index 55bae84c822..cd4f46a670f 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -1234,7 +1234,7 @@ tsapi TSSslConnection TSVConnSSLConnectionGet(TSVConn sslp); tsapi TSSslContext TSSslContextFindByName(const char *name); tsapi TSSslContext TSSslContextFindByAddr(struct sockaddr const *); /* Create a new SSL context based on the settings in records.config */ -tsapi TSSslContext TSSslServerContextCreate(TSSslX509 cert, const char *certname); +tsapi TSSslContext TSSslServerContextCreate(TSSslX509 cert, const char *certname, const char *rsp_file); tsapi void TSSslContextDestroy(TSSslContext ctx); tsapi void TSSslTicketKeyUpdate(char *ticketData, int ticketDataLen); tsapi TSNextProtocolSet TSUnregisterProtocol(TSNextProtocolSet protoset, const char *protocol); diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index 5a3121eeb8a..e6635ed9b5a 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -22,6 +22,7 @@ #include "P_OCSPStapling.h" #if TS_USE_TLS_OCSP +#include #include #include #include "P_Net.h" @@ -43,6 +44,7 @@ struct certinfo { ink_mutex stapling_mutex; unsigned char resp_der[MAX_STAPLING_DER]; unsigned int resp_derlen; + bool is_prefetched; bool is_expire; time_t expire_time; }; @@ -141,11 +143,44 @@ stapling_get_issuer(SSL_CTX *ssl_ctx, X509 *x) return issuer; } +static bool +stapling_cache_response(OCSP_RESPONSE *rsp, certinfo *cinf) +{ + unsigned char resp_der[MAX_STAPLING_DER]; + unsigned char *p; + unsigned int resp_derlen; + + p = resp_der; + resp_derlen = i2d_OCSP_RESPONSE(rsp, &p); + + if (resp_derlen == 0) { + Error("stapling_cache_response: cannot decode OCSP response for %s", cinf->certname); + return false; + } + + if (resp_derlen > MAX_STAPLING_DER) { + Error("stapling_cache_response: OCSP response too big (%u bytes) for %s", resp_derlen, cinf->certname); + return false; + } + + ink_mutex_acquire(&cinf->stapling_mutex); + memcpy(cinf->resp_der, resp_der, resp_derlen); + cinf->resp_derlen = resp_derlen; + cinf->is_expire = false; + cinf->expire_time = time(nullptr) + SSLConfigParams::ssl_ocsp_cache_timeout; + ink_mutex_release(&cinf->stapling_mutex); + + Debug("ssl_ocsp", "stapling_cache_response: success to cache response"); + return true; +} + bool -ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) +ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname, const char *rsp_file) { scoped_X509 issuer; STACK_OF(OPENSSL_STRING) *aia = nullptr; + BIO *rsp_bio = nullptr; + OCSP_RESPONSE *rsp = nullptr; if (!cert) { Error("null cert passed in for %s", certname); @@ -174,8 +209,33 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) cinf->certname = ats_strdup(certname); cinf->resp_derlen = 0; ink_mutex_init(&cinf->stapling_mutex); - cinf->is_expire = true; - cinf->expire_time = 0; + cinf->is_prefetched = rsp_file ? true : false; + cinf->is_expire = true; + cinf->expire_time = 0; + + if (cinf->is_prefetched) { + Debug("ssl_ocsp", "using OCSP prefetched response file %s", rsp_file); + rsp_bio = BIO_new_file(rsp_file, "r"); + if (rsp_bio) { + rsp = d2i_OCSP_RESPONSE_bio(rsp_bio, nullptr); + } + + if (!rsp_bio || !rsp) { + Note("cannot get prefetched response for %s from %s", certname, rsp_file); + goto err; + } + + if (!stapling_cache_response(rsp, cinf)) { + Error("stapling_refresh_response: can not cache response"); + goto err; + } else { + Debug("ssl_ocsp", "stapling_refresh_response: successful refresh OCSP response"); + OCSP_RESPONSE_free(rsp); + rsp = nullptr; + BIO_free(rsp_bio); + rsp_bio = nullptr; + } + } issuer = stapling_get_issuer(ctx, cert); if (issuer == nullptr) { @@ -222,6 +282,14 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) if (map) { delete map; } + + if (rsp) { + OCSP_RESPONSE_free(rsp); + } + if (rsp_bio) { + BIO_free(rsp_bio); + } + return false; } @@ -239,37 +307,6 @@ stapling_get_cert_info(SSL_CTX *ctx) return nullptr; } -static bool -stapling_cache_response(OCSP_RESPONSE *rsp, certinfo *cinf) -{ - unsigned char resp_der[MAX_STAPLING_DER]; - unsigned char *p; - unsigned int resp_derlen; - - p = resp_der; - resp_derlen = i2d_OCSP_RESPONSE(rsp, &p); - - if (resp_derlen == 0) { - Error("stapling_cache_response: cannot decode OCSP response for %s", cinf->certname); - return false; - } - - if (resp_derlen > MAX_STAPLING_DER) { - Error("stapling_cache_response: OCSP response too big (%u bytes) for %s", resp_derlen, cinf->certname); - return false; - } - - ink_mutex_acquire(&cinf->stapling_mutex); - memcpy(cinf->resp_der, resp_der, resp_derlen); - cinf->resp_derlen = resp_derlen; - cinf->is_expire = false; - cinf->expire_time = time(nullptr) + SSLConfigParams::ssl_ocsp_cache_timeout; - ink_mutex_release(&cinf->stapling_mutex); - - Debug("ssl_ocsp", "stapling_cache_response: success to cache response"); - return true; -} - static int stapling_check_response(certinfo *cinf, OCSP_RESPONSE *rsp) { @@ -461,7 +498,7 @@ ocsp_update() cinf = iter->second; ink_mutex_acquire(&cinf->stapling_mutex); current_time = time(nullptr); - if (cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) { + if ((cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) && !cinf->is_prefetched) { ink_mutex_release(&cinf->stapling_mutex); if (stapling_refresh_response(cinf, &resp)) { Debug("Successfully refreshed OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); @@ -506,7 +543,7 @@ ssl_callback_ocsp_stapling(SSL *ssl) ink_mutex_acquire(&cinf->stapling_mutex); time_t current_time = time(nullptr); - if (cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) { + if ((cinf->resp_derlen == 0 || cinf->is_expire) || (cinf->expire_time < current_time && !cinf->is_prefetched)) { ink_mutex_release(&cinf->stapling_mutex); Debug("ssl_ocsp", "ssl_callback_ocsp_stapling: failed to get certificate status for %s", cinf->certname); return SSL_TLSEXT_ERR_NOACK; diff --git a/iocore/net/P_OCSPStapling.h b/iocore/net/P_OCSPStapling.h index e3d5a6f63dc..fbee1103322 100644 --- a/iocore/net/P_OCSPStapling.h +++ b/iocore/net/P_OCSPStapling.h @@ -28,7 +28,7 @@ #include void ssl_stapling_ex_init(); -bool ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname); +bool ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname, const char *rsp_file); void ocsp_update(); int ssl_callback_ocsp_stapling(SSL *); #endif diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 87a4a33e0ad..098d9230f52 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -107,6 +107,7 @@ struct SSLConfigParams : public ConfigInfo { static int ssl_ocsp_request_timeout; static int ssl_ocsp_update_period; static int ssl_handshake_timeout_in; + char *ssl_ocsp_response_path_only; static size_t session_cache_number_buckets; static size_t session_cache_max_bucket_size; diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index b154dc1e9c9..932a8c3303d 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -45,14 +45,15 @@ typedef int ssl_error_t; struct SSLMultiCertConfigParams { SSLMultiCertConfigParams() { REC_ReadConfigInt32(session_ticket_enabled, "proxy.config.ssl.server.session_ticket.enable"); } - int session_ticket_enabled; ///< session ticket enabled - ats_scoped_str addr; ///< IPv[64] address to match - ats_scoped_str cert; ///< certificate - ats_scoped_str first_cert; ///< the first certificate name when multiple cert files are in 'ssl_cert_name' - ats_scoped_str ca; ///< CA public certificate - ats_scoped_str key; ///< Private key - ats_scoped_str dialog; ///< Private key dialog - ats_scoped_str servername; ///< Destination server + int session_ticket_enabled; ///< session ticket enabled + ats_scoped_str addr; ///< IPv[64] address to match + ats_scoped_str cert; ///< certificate + ats_scoped_str first_cert; ///< the first certificate name when multiple cert files are in 'ssl_cert_name' + ats_scoped_str ca; ///< CA public certificate + ats_scoped_str key; ///< Private key + ats_scoped_str ocsp_response; ///< prefetched OCSP response + ats_scoped_str dialog; ///< Private key dialog + ats_scoped_str servername; ///< Destination server SSLCertContext::Option opt = SSLCertContext::OPT_NONE; ///< SSLCertContext special handling option }; diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index aa82eb08c44..eada98d17d8 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -183,6 +183,7 @@ SSLConfigParams::initialize() char *clientCACertRelativePath = nullptr; char *ssl_server_ca_cert_filename = nullptr; char *ssl_client_ca_cert_filename = nullptr; + char *ssl_ocsp_response_path = nullptr; cleanup(); @@ -327,6 +328,9 @@ SSLConfigParams::initialize() REC_EstablishStaticConfigInt32(ssl_ocsp_cache_timeout, "proxy.config.ssl.ocsp.cache_timeout"); REC_EstablishStaticConfigInt32(ssl_ocsp_request_timeout, "proxy.config.ssl.ocsp.request_timeout"); REC_EstablishStaticConfigInt32(ssl_ocsp_update_period, "proxy.config.ssl.ocsp.update_period"); + REC_ReadConfigStringAlloc(ssl_ocsp_response_path, "proxy.config.ssl.ocsp.response.path"); + set_paths_helper(ssl_ocsp_response_path, nullptr, &ssl_ocsp_response_path_only, nullptr); + ats_free(ssl_ocsp_response_path); REC_ReadConfigInt32(async_handshake_enabled, "proxy.config.ssl.async.handshake.enabled"); REC_ReadConfigStringAlloc(engine_conf_file, "proxy.config.ssl.engine.conf_file"); diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 1e390fda57a..efc3f3e428b 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -69,6 +69,7 @@ using namespace std::literals; static constexpr std::string_view SSL_IP_TAG("dest_ip"sv); static constexpr std::string_view SSL_CERT_TAG("ssl_cert_name"sv); static constexpr std::string_view SSL_PRIVATE_KEY_TAG("ssl_key_name"sv); +static constexpr std::string_view SSL_OCSP_RESPONSE_TAG("ssl_ocsp_name"sv); static constexpr std::string_view SSL_CA_TAG("ssl_ca_name"sv); static constexpr std::string_view SSL_ACTION_TAG("action"sv); static constexpr std::string_view SSL_ACTION_TUNNEL_TAG("tunnel"sv); @@ -1360,26 +1361,6 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co SSL_CTX_set_next_protos_advertised_cb(ctx, SSLNetVConnection::advertise_next_protocol, nullptr); SSL_CTX_set_alpn_select_cb(ctx, SSLNetVConnection::select_next_protocol, nullptr); -#if TS_USE_TLS_OCSP - if (SSLConfigParams::ssl_ocsp_enabled) { - Debug("ssl", "SSL OCSP Stapling is enabled"); - SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); - const char *cert_name = sslMultCertSettings ? sslMultCertSettings->cert.get() : nullptr; - - for (auto cert : cert_list) { - if (!ssl_stapling_init_cert(ctx, cert, cert_name)) { - Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", cert_name); - } - } - } else { - Debug("ssl", "SSL OCSP Stapling is disabled"); - } -#else - if (SSLConfigParams::ssl_ocsp_enabled) { - Warning("failed to enable SSL OCSP Stapling; this version of OpenSSL does not support it"); - } -#endif /* TS_USE_TLS_OCSP */ - if (SSLConfigParams::init_ssl_ctx_cb) { SSLConfigParams::init_ssl_ctx_cb(ctx, true); } @@ -1528,6 +1509,10 @@ ssl_extract_certificate(const matcher_line *line_info, SSLMultiCertConfigParams sslMultCertSettings.key = ats_strdup(value); } + if (strcasecmp(label, SSL_OCSP_RESPONSE_TAG) == 0) { + sslMultCertSettings.ocsp_response = ats_strdup(value); + } + if (strcasecmp(label, SSL_SESSION_TICKET_ENABLED) == 0) { sslMultCertSettings.session_ticket_enabled = atoi(value); } @@ -1785,6 +1770,27 @@ SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_lis } } +#if TS_USE_TLS_OCSP + if (SSLConfigParams::ssl_ocsp_enabled) { + Debug("ssl", "SSL OCSP Stapling is enabled"); + SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); + } else { + Debug("ssl", "SSL OCSP Stapling is disabled"); + } + SimpleTokenizer ocsp_tok("", SSL_CERT_SEPARATE_DELIM); + if (sslMultCertSettings->ocsp_response) { + ocsp_tok.setString(sslMultCertSettings->ocsp_response); + if (cert_tok.getNumTokensRemaining() != ocsp_tok.getNumTokensRemaining()) { + Error("the number of certificates in ssl_cert_name and ssl_ocsp_name doesn't match"); + return false; + } + } +#else + if (SSLConfigParams::ssl_ocsp_enabled) { + Warning("failed to enable SSL OCSP Stapling; this version of OpenSSL does not support it"); + } +#endif /* TS_USE_TLS_OCSP */ + for (const char *certname = cert_tok.getNext(); certname; certname = cert_tok.getNext()) { std::string completeServerCertPath = Layout::relative_to(params->serverCertPathOnly, certname); scoped_BIO bio(BIO_new_file(completeServerCertPath.c_str(), "r")); @@ -1845,6 +1851,21 @@ SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_lis } } } +#if TS_USE_TLS_OCSP + if (SSLConfigParams::ssl_ocsp_enabled) { + if (sslMultCertSettings->ocsp_response) { + const char *ocsp_response_name = ocsp_tok.getNext(); + ats_scoped_str completeOCSPResponsePath(Layout::relative_to(params->ssl_ocsp_response_path_only, ocsp_response_name)); + if (!ssl_stapling_init_cert(ctx, cert, certname, (const char *)completeOCSPResponsePath)) { + Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", certname); + } + } else { + if (!ssl_stapling_init_cert(ctx, cert, certname, nullptr)) { + Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", certname); + } + } + } +#endif /* TS_USE_TLS_OCSP */ } return true; diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index b382acc7826..9b21975bcb1 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1186,7 +1186,9 @@ static const RecordElement RecordsConfig[] = // # Update period for stapling caches. 60s (1 min) by default. {RECT_CONFIG, "proxy.config.ssl.ocsp.update_period", RECD_INT, "60", RECU_DYNAMIC, RR_NULL, RECC_NULL, "^[0-9]+$", RECA_NULL} , - + // # Base path for OCSP prefetched responses + {RECT_CONFIG, "proxy.config.ssl.ocsp.response.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , //############################################################################## //# //# Configuration for TLSv1.3 and above diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 2111fa0c265..f2a2bd59e7b 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -8998,7 +8998,7 @@ TSSslContextFindByAddr(struct sockaddr const *addr) } tsapi TSSslContext -TSSslServerContextCreate(TSSslX509 cert, const char *certname) +TSSslServerContextCreate(TSSslX509 cert, const char *certname, const char *rsp_file) { TSSslContext ret = nullptr; SSLConfigParams *config = SSLConfig::acquire(); @@ -9007,7 +9007,7 @@ TSSslServerContextCreate(TSSslX509 cert, const char *certname) #if TS_USE_TLS_OCSP if (ret && SSLConfigParams::ssl_ocsp_enabled && cert && certname) { if (SSL_CTX_set_tlsext_status_cb(reinterpret_cast(ret), ssl_callback_ocsp_stapling)) { - if (!ssl_stapling_init_cert(reinterpret_cast(ret), reinterpret_cast(cert), certname)) { + if (!ssl_stapling_init_cert(reinterpret_cast(ret), reinterpret_cast(cert), certname, rsp_file)) { Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", (const char *)certname); } } diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 50e0c99364c..00daa08250f 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -9175,7 +9175,7 @@ REGRESSION_TEST(SDK_API_TSSslServerContextCreate)(RegressionTest *test, int leve TSSslContext ctx; // See TS-4769: TSSslServerContextCreate always returns null. - ctx = TSSslServerContextCreate(nullptr, nullptr); + ctx = TSSslServerContextCreate(nullptr, nullptr, nullptr); *pstatus = ctx ? REGRESSION_TEST_PASSED : REGRESSION_TEST_FAILED; TSSslContextDestroy(ctx); diff --git a/tests/gold_tests/autest-site/conditions.test.ext b/tests/gold_tests/autest-site/conditions.test.ext index 9cac02f78b8..e60c06ca52c 100644 --- a/tests/gold_tests/autest-site/conditions.test.ext +++ b/tests/gold_tests/autest-site/conditions.test.ext @@ -19,6 +19,9 @@ def HasOpenSSLVersion(self, version): return self.EnsureVersion(["openssl","version"],min_version=version) +def HasCurlVersion(self, version): + return self.EnsureVersion(["curl","--version"],min_version=version) + def HasCurlFeature(self, feature): def default(output): @@ -60,6 +63,7 @@ def PluginExists(self, pluginname): ExtendCondition(HasOpenSSLVersion) ExtendCondition(HasATSFeature) +ExtendCondition(HasCurlVersion) ExtendCondition(HasCurlFeature) ExtendCondition(PluginExists) diff --git a/tests/gold_tests/tls/ssl/ca.ocsp.key b/tests/gold_tests/tls/ssl/ca.ocsp.key new file mode 100644 index 00000000000..ccae9f121a4 --- /dev/null +++ b/tests/gold_tests/tls/ssl/ca.ocsp.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAsg7JDZ8J/yG/bBxl/Td9urR78e0E/IwSfuujWbewn2fulEkF +jrvN8G4I8nmEG/Fg+2lMR2wKqUYYwbxKaq0tlprbZp/bB9QJqGdEqmX9HBKIPB6L +n0GFMwj5yG4etx2sx3scdHPaJBZ12IPljxJXPKsHurIY/tXa9WS6IhEwHgcNef+p +uVt8pNvmw35jyPMZhcXnN/Fs+bHX908SCrMgiwf4oKyPHFxh7WEqLVZj4Z2At7lH +On0wRHnYjFB7wKSMgM44eUjhjN6zOe4gWLGdfbtXgZibxgKLYfYL8YBzcPPChdCY +Evshn3RTbJ3IiwDHTAlBTchoDoBjV/gKlM8LbQIDAQABAoIBAQCMMeoJqIxFbrv6 +ko2XB2ceZ1cjz/xaIKu7dStDy8bsa/fEl44hqStoQCsZR6ZGHhK/QVRG9AGc8E0z +1V4+iiZX64wOTJU3n7MO/mhpRi45OTo9I8vJU9xdp5aMQnA6u8m7supfooxCV9Dn +7koEEWvQn9VRIUNe+uEQ0ANiKWhaauC4aODV3amNgmGTQVF1NKvcqdXbeQGCGHDy +HKoehDVlbaoKvzZ03dt0kL1sa+vqQmC6pNHlWQ9pchML6E+KBdaQrNCeRFHbLLiO +ULQg2KS0IF+kqbBrpBRd3lW4ZV2rglWZQ1998CJP0et3WXET14nqCTNfty3iTcUL +tXPGnXAdAoGBAM3BcnSWeEkTtY8Sxu3y/M/zB9wF7f+mHNE88ISWrdx/hJni4imV +diyzcnfyxTQc8uuett1UHA85SZuuDwdvVz7CD+UglmploFsaUvwl6Ytl05a2oFqT +n5dmnsGgLCpnpSNAZWpT6gyddQR3lyzCw+k5fkQVikPzcGJrd7mk27YfAoGBAN2J +1uYw3E7uiywjYlQXqrQ7L0Ouccw3wV0jcsSgDJGsffhTkPhn/BwZ2QQcldGR8TCB +Y6Er02Kqzk1mGFG94TPQy5C+xJzZv2KKq9LSrOVQszh+wozEruh7bKARQ/uPDLPw ++F/DWFe6/0mDb5gZf40pyk+njFh1UT7yGE2poVTzAoGBAMaDQ2hU3Iy05VC6rw9Y +hq4jLowLdIpYvCjsAKoLroa0yTynd8jjGPcb0u8DXVxgKcdGg+uagM/3V5tKHdnw +hF5aYXeRL05L6qC7DyGTenYxsikQ3jlFgI5URgtN/A6VnPAb6zzg5UlyiTncIBDh +gJ7+B2Ks3Y+dyepLAWItOoXFAoGBAMgahR2O7K/vD45it6I1bl81Rk/f9bH8eo/i +QPwRhMjgATiYYs29Px8yya7JExoktLKXbKJbr6fjmEyY90Z+ODhRVE39wiHbHN+p +WeInoTvQVNGmzZvQ3BvpwAglED7cyyCNfAsjq1wy7/w62EWOYoPjR3YDZOVRsn0k +t4cOvUa5AoGAL2LxuYaU5GESEv3MAtM4oQUsc0sHApGyy/hJzGBF/RXpy+U4kYBF +apAonFCQZ2VBr/t0XH0MW4GrRZoJsa1xWWhlXhARfOSWCRRS8tdTdir2TpLMmJJN +QakM9/QgfIQQ+J25XeLe16dRl2i4acG1BA2b1S1isC8HmV/wy4sGopA= +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/ca.ocsp.pem b/tests/gold_tests/tls/ssl/ca.ocsp.pem new file mode 100644 index 00000000000..cd8114fb258 --- /dev/null +++ b/tests/gold_tests/tls/ssl/ca.ocsp.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDWDCCAkCgAwIBAgIUUCa7m/ftLhb69KUqFfGBKwnw+lcwDQYJKoZIhvcNAQEL +BQAwRDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZBcGFjaGUx +FzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMB4XDTE5MDUwMjEyNDAwMFoXDTQ5MDQy +NDEyNDAwMFowRDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZB +cGFjaGUxFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAsg7JDZ8J/yG/bBxl/Td9urR78e0E/IwSfuujWbewn2fu +lEkFjrvN8G4I8nmEG/Fg+2lMR2wKqUYYwbxKaq0tlprbZp/bB9QJqGdEqmX9HBKI +PB6Ln0GFMwj5yG4etx2sx3scdHPaJBZ12IPljxJXPKsHurIY/tXa9WS6IhEwHgcN +ef+puVt8pNvmw35jyPMZhcXnN/Fs+bHX908SCrMgiwf4oKyPHFxh7WEqLVZj4Z2A +t7lHOn0wRHnYjFB7wKSMgM44eUjhjN6zOe4gWLGdfbtXgZibxgKLYfYL8YBzcPPC +hdCYEvshn3RTbJ3IiwDHTAlBTchoDoBjV/gKlM8LbQIDAQABo0IwQDAOBgNVHQ8B +Af8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUmAVXj2UeCyJRWe73 +dBL672OFRKIwDQYJKoZIhvcNAQELBQADggEBAEf1Jntnfg+4W4/u7FGGFgkjiQ9e +zS38xRsSyKhQ5pBipUHbkphbgwu0xXDa9jQkVJuwsk7xrMdvbKmbVx8DBg4OZKu+ +50eqw8/NTg401TVLKC+rJ5kcgUEGuZobegBhKtUqzuswQQF4F4KMTUreJBXWPeET +CxFyy04LAxeewEYLzJ/Ylw4jgosC+MlTlXmN41SxtQ5URjOOIPmHOZF9CzSHoJcz +qN84JT1cgcfd/gYgQD+r1pCPRjJrLU+5X3PCHHehNphIuU9cQ6KuJaCq1FABpovx +iIMiW56fzZ4Wbon3Sh0zuCxzvtBhtM5qOd+eP3pbIB6Ps+J9Dfkrpa/wVe0= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/ocsp_response.der b/tests/gold_tests/tls/ssl/ocsp_response.der new file mode 100644 index 0000000000000000000000000000000000000000..fd6d9a631f6d6663795a7a73170675355ac68267 GIT binary patch literal 2266 zcmchYdpOkj9>?c5H#3Hrm~qLiam#i18e>H~D2ybRa+lp0vs%pb9uh+=ll75UY`#TU$00fvWG}Z(Sp-?gaf{+4C%c`pp@XN}SktEe${5qAqXB_D2)3uz*OV`o{hu*6a zsKCMoSOKbqCDq!}A{ZP5abO3)qERMz1Y8XTLz1i}QzT^n%R4vU)|Rld2{+F+#MvN8 z`KSO9Qwyi#6ErJ5p!n_uBh9%CPKqSES$UYxJxFqGeI@g^Yu!mTB*26Hi;7rb%|){T zG%D?rusj-TEOZk}bfX6CQrNPz0Lod8t`P4-N4mteshb;ZSGhvpjQtCSkbc&hisr zdaB=OQ_HE$`c^5>O7aK_p7c3VM~ytl7&hYRwBI@3Vc_60_q0(gO9pzI?+GKK3-F1d zt%($CVMbuLRDUZ+Jb;y4y{%R}oIBEHE>(|MD$h8qmQDIG+pE)RSw9;aL0=SS5A~LY zB&zx$$PK%D%pWQ-Af~BUfL;Uwv=j&sc|ZU^B1XU=IGjXZmo~*uIhloVdcIj>x6L{& zhT#DCvV6eV0=L>kgdY-3g!}FKWMGK!Z5|ec)#TNPED`M#6UNxfv`_&`YYqev&j>SP z9tvZ}u$X2H4jX8x$bXy;Dh&V-tRnxhXCF-9{psQVMnxBBe^625Ft{<4SSB}y6CJ^f z{X$Uy>3seb=(kBAz<>afg|CgoHL$ew_e;4RY~!?n^y4F(YO|V+#wS{OJlRzbonw;Y ze{w{X9z4uJ>^0n(VLGf8{-tCu62`T!SMzW5QeFmfmz3U*?^`}bG$-q<%pXAsZS+%nAKCDDECp9y0a z?%1a{yYe}GKjf@4`qp2m*r*7OfMC$MIKTz?)naNAAZP`NM8IWufVgJsBjtcB8uMWV zAh1Ag%}HEKf&&=PT(j`Vf3pj~hPALRQWYrk{uJ-e8}kX#a^94+3(|+W4DXHE!!jtd zDE#a8F}HIkrJ6~%O`;r5^MXS!WmGe{n29Y(b-nek51NHKNw^J07L{%Wzq@ z?u7NHWb}gX7zg<8fxj#UZ6>fCZ}kwk^7J|p$zn`thCA_y`SQv%lrnxaw9soE*?C%> zvLD^!Zp{CyVD_y&y%am>QPZS zFYCg4iHo*>Deoy5zAs0)B{FqMv4d=3tcCr0?ZIT!Dw=_-XgYpG(|cocfpo@9`Q5^K zW2Je%33l#Xz#q!?e?s$rvjF}Rj5> zt&9?|H^DZhvm1MI3~nrc;-2S#2+H0dY+EgAXvmCEZO zN#&yxw+}NUzphWaJ83=7dV6I^>ZK@KN6wIWBfcoo*c~<#W=!#Cj2H15uG4Lkfph%h zOz*_D@=RXOrg*pU%&|I~S#sc&qU7cPety9{ct+T3+&qsP^J{xrZ@%K9cFBQVtRhJU ztiuEAv_rBN&tn9^E-Z8Mw1;Kss^(~uGD=Luj}7%+cgfBewFWb-D~&OvKoEyg-71d_Zjm? wQ*=U#OS+4dqmN15H7y%Wxi@;l>U*MWo)kMIgpjpFbw4M|zco33Y0huvpEm_mqyPW_ literal 0 HcmV?d00001 diff --git a/tests/gold_tests/tls/ssl/responder.ocsp.key b/tests/gold_tests/tls/ssl/responder.ocsp.key new file mode 100644 index 00000000000..c71e4e55fda --- /dev/null +++ b/tests/gold_tests/tls/ssl/responder.ocsp.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAtjD9/eCvlUxtKoXVh43ab60uriYqkbzOTm2r0URwfXndQAWh +d35vA2goUosnNNKcssvh8SZFpNNtYvD/5r+4LZL09nlCOdJ/+JEqKDxR0WfXNCxd +UB2RwFnf1rMSAfyj5sT7bBYlPy2EX8LvbruGlzI00MBS1zLsYKNFWq2V1gu8rl0z +pFkJhhOJqXUUbINlfCH+NvLaO9NByjV1esYfHe4lYtnHxno9Q0vuxT1dobuE4pUV +6v+SnSg8VP0ZdFQSuuss0E412NNna3naBoZn14cBWkFjAtNxqNC+a9dHYxOZrsEo +joB8R1E96YZwY0oJ47/LmG/Sp5IQKWZWFZ4pnQIDAQABAoIBAD32gZujm/PJ72LD +67BThVPv8W1XG6k/LmcsE4Bzp1J1bNMGVzj9riHZfcU9AFONwa9peel7G7qIEa7R +yiafU7NkRJ3C9cwWlGFkdZMDmMwAZgefgwjpVZW2u1MYyeoVE2U730qOaZKIF3o+ +IRJnAspPT/kjP8liz1O6k67YVJpcBcp9nJruDJh8BZtgneb2z+di8aV6gi2d2Cfq +rPrAxlwDc1XiMgG+Auuq0OQs++K1GFAcy6pMutjIpXMDl3eAUddPjEPBuKlUiRO1 +GqR9d3YRWR+/KZV++RzdwPbZVjiK2STSrimsDfyufibD6xoQ3TuKtdL8UsSVdOkr +OORojd0CgYEAwfJ1EnJXhXmJhZj6qXE2Es61c2/JLXFbk0ryE7nIdmalU1rT7h0J +jPp+uzl1mtSRrx/aTesLOqHFefJqMOiTVjFO/7+J+fe7gsnHAvB9cHXLkSY6FN37 +p7PQ6yFlb+k8Biug+6tyuGv78MxADZMZ9pWc/AD4VQZlDkAygIFbVVcCgYEA8Hup +70Xv7O2lxnaPAhbqm1Ijm+ABDgObh8s1CaTO7F4upwmOSZ+EOrMdP6+vdPq0Hrm/ +KObaqRM5Vwi74hEoxKCoLulBslEnq2KbU0t22TnqCF+UBYledfFKgYTJmfvqGCxg +uVPQ1qZp//9yQxcrIhr4AwQDr3dX1Mb8VfH3TCsCgYEAkDHIMsfKJFVhFm/PZSzj +jAYdR88DnoKaGB9vbZUB4m2cWyW7TVxPXn5avK4SruN90Nr4vleTCKt/m5PMucIg +0MNmPaTVW4CA69NC3/+W84bQq4DlS+BimqOJH1e8CAE6/Edxr8sfRtgZ/0SMFsuY +UQmZJo8+ElDnzzmRkpMaKY8CgYA/KHtM+BU3KILtSJ3uco5TFJN9kKs2PwRN+bSI +P9yIf4PJIt1XwKk9sWTxIPb3xhAgMbBe0aKD3SSmEwklKlSGr5r8Fw7GAkJk5JTe +n2crTeaFJHT/r0A7wY9LzNAVvO+SQbV1dunWNgaI0VH1BNSzNFoGkLtXDgTnQQts +lwvX2QKBgQCjMNbYh+7TSe/L/ys/TG+tuFkNaprvrHeRYigI4U43rWKiY2EteYi/ +V/eVY/t7hWDnwV9FDo5q/v4dneWWgl2kJA8NlP/p65M+YrTY1nVRrbJX4+GIlxxY +yaGI4Ab1WBFsYkpHC4BpBommJV/znaOcNS/EBY3G3ZmW+EzGmiTuvw== +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/responder.ocsp.pem b/tests/gold_tests/tls/ssl/responder.ocsp.pem new file mode 100644 index 00000000000..03753b8dd1f --- /dev/null +++ b/tests/gold_tests/tls/ssl/responder.ocsp.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlzCCAn+gAwIBAgIURxCF64GDk44IQeo7HFw5EB5jKNUwDQYJKoZIhvcNAQEL +BQAwRDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZBcGFjaGUx +FzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMB4XDTE5MDUwMjEyNDAwMFoXDTQ5MDQy +NDEyNDAwMFowUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZB +cGFjaGUxIzAhBgNVBAMTGm9jc3AtcmVzcG9uZGVyLmV4YW1wbGUuY29tMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtjD9/eCvlUxtKoXVh43ab60uriYq +kbzOTm2r0URwfXndQAWhd35vA2goUosnNNKcssvh8SZFpNNtYvD/5r+4LZL09nlC +OdJ/+JEqKDxR0WfXNCxdUB2RwFnf1rMSAfyj5sT7bBYlPy2EX8LvbruGlzI00MBS +1zLsYKNFWq2V1gu8rl0zpFkJhhOJqXUUbINlfCH+NvLaO9NByjV1esYfHe4lYtnH +xno9Q0vuxT1dobuE4pUV6v+SnSg8VP0ZdFQSuuss0E412NNna3naBoZn14cBWkFj +AtNxqNC+a9dHYxOZrsEojoB8R1E96YZwY0oJ47/LmG/Sp5IQKWZWFZ4pnQIDAQAB +o3UwczAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwkwDAYDVR0T +AQH/BAIwADAdBgNVHQ4EFgQUGOVVyRVQ7T9HG+yyb+evtoXW1/wwHwYDVR0jBBgw +FoAUmAVXj2UeCyJRWe73dBL672OFRKIwDQYJKoZIhvcNAQELBQADggEBAKSA6zdF +BFEy/gLQcGBsJ6IxnIpL3kijk/cdfMMraj+WgFpdr4utZXMI5kJ8s9C09nVqfYIk +os2XEUkpWOTbbXJDcgYjOtUy/3VPJ6BTELQYLR/IDISDiIwdehTtBOosHwuSoW1t +LnPQFtmZda6PvYfETKGBsAIKSM16GZQHJFrEKldWYALynjDAKwBN+SQDC6SHs48U +Md4V7XPm1GXysewBg+GIXZxPECBE7BstcQbOViiMY0ZvSbR8mMoxiSYLNA+2QrTu +QBFQPf2Bpyc10vSyM+8P+pZb++p9raK0BWqS8rNi0BP7OnakzpvZygwtwY8sRYM8 +AToAbPRxJMJaIAU= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/server.ocsp.key b/tests/gold_tests/tls/ssl/server.ocsp.key new file mode 100644 index 00000000000..03790391e89 --- /dev/null +++ b/tests/gold_tests/tls/ssl/server.ocsp.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA53sB/uEdZNJIFLiMJr+5KaB75lTCEBIg98FCCfxBPsL1XqNO +s3Tx1ihmCMeT6wnvxIi+RIIbf0ymSquImmkzAodZdW0MGs/G9Vo2HsjclfkOsZhN +M5k8/AXF/FaGOjGd/WOKZFGejhd6utzqOo+Bj+STm2u+bCNtI4IZF+icTri7rIIK +CMdZalN5Au33DqT+MuKhtTmRYs1Pw8Qf2inv311y7f5Ok/TvBnfbAU18Jo+Ga/M0 +94bVAMA8WH6RyABZZlr8VIn6e5YadFfHVt7GrMIO/2QV4mlCK7zEC2NamNNslf4c +n2NpQNQToV1UbOK3uH5SNtB6CqJDBR9sIPnTKwIDAQABAoIBADgrmjVeLQLNIB6f +FatFdMoMHmSrBphdvdBA/iRsKOzw5Be96xgS9agxD2lr/JHZTGxVfk4jgEaos+WE +sFY/1tfzPhsHhhtvdekNpfpcZWKjGBSyT3GI6sqBICT2XgX7Ckp1gByNzbrPKcH0 +X4YsUpU3MzZQs9mL0yz9odcyY9OZ/BMX2lHb/G/IzZqb2a5YCbew1eZyU3pMRs2X +POtKJgCfiKOGcI+Kv3gL8CWtMWkxZN6CeNF1GicndNLWBdW6KSPOjAWf7eBAu8cs +nC7gtpCcpqL2GVOx7rzuPPjTRY62Xqf1/2AjeJkAqkeBbqs9shlK4Nm76WvDsDp9 +V7ZxjpECgYEA8YX3lCy0Ba+VE9xmHrzIqBzl39s73du17hiWlNcI05fer3lKt9g6 +IINnH60nzYXYW3hKHoKBUJiop2/TI6/aaLTrlpJ/0lcFNLI2ZXOQ7fQ4s/iFyWrP +DkCE5Z1qlUNsPjCN1ZG4xMciGy83JtzjsQfoj3sHBUnrk8uqftp7hAMCgYEA9Vrw +j45Z9WKbBb/ilRv0UhkZ8uFYDut2udZZ2fp//Cb0C9ykdKeu6Q9p6MdFjdCPiNWi +DDx885arMmtJuRS4MGAt9qps4BIsOQAdsLW/5BB9hRR5nqYSaaOA/FZ/X2uFtmIk +5zjkHXktBBTqoVNVqGwnjeQyGqCpWxRXtNtTz7kCgYEA3IzfZnnj8oVB9x7+SfdO +rOWmrOMAKjpmSgQ+DbDHqKE4griaGIPloKcd1nlCrZUZ231fAble6QBekne1MRN2 +uMLtl1Q0URmR8WsD7WS45fJsjTvWv/U/Gt6j/SHgoGkvQSMJggtN1LObW4OkM2Lm +sVRtdAh+gr/b1dzX1nsg640CgYAWQbaavyYH7Xb0kZCDSDLkk6RX9Psg91kgyIIE +FQYxIHN48/3zGxbxy1UnKZR0pduvZPm7NG19R0imXTcl0+xVbxQcUR9pQBzE2u7W +jdYnYRuRy+awbo4zCQL1YP9S75UEk2iXlQCUb96WhTM3iTC3A4CfDXlCExrpyTGf +lVnH+QKBgQCGanhp+cL4gmbgFvu2KuLXGneK+gIRL3X9neLTq/0XoVE2pNiTZBne +bHVerfcogPXMSdFfCnjpWqx82UoXrQWghaOB9iLTPu+HH2SWWrvSle6JBYoF8iv8 +Y8/YKzlnZV62KpCPeKaal6RsPbNiPD5o6jONPKZmnxQeCa694ImLwg== +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/server.ocsp.pem b/tests/gold_tests/tls/ssl/server.ocsp.pem new file mode 100644 index 00000000000..a8b14c0c736 --- /dev/null +++ b/tests/gold_tests/tls/ssl/server.ocsp.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID6TCCAtGgAwIBAgIUZyBINUwv8M98Rr31B9y2zXyuFDAwDQYJKoZIhvcNAQEL +BQAwRDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZBcGFjaGUx +FzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMB4XDTE5MDUwMjEyNDAwMFoXDTQ5MDQy +NDEyNDAwMFowSDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZB +cGFjaGUxGzAZBgNVBAMTEnNlcnZlci5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAOd7Af7hHWTSSBS4jCa/uSmge+ZUwhASIPfBQgn8 +QT7C9V6jTrN08dYoZgjHk+sJ78SIvkSCG39MpkqriJppMwKHWXVtDBrPxvVaNh7I +3JX5DrGYTTOZPPwFxfxWhjoxnf1jimRRno4Xerrc6jqPgY/kk5trvmwjbSOCGRfo +nE64u6yCCgjHWWpTeQLt9w6k/jLiobU5kWLNT8PEH9op799dcu3+TpP07wZ32wFN +fCaPhmvzNPeG1QDAPFh+kcgAWWZa/FSJ+nuWGnRXx1bexqzCDv9kFeJpQiu8xAtj +WpjTbJX+HJ9jaUDUE6FdVGzit7h+UjbQegqiQwUfbCD50ysCAwEAAaOBzjCByzAO +BgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIw +ADAdBgNVHQ4EFgQUeLq8rzbtQim/wBI+OmgVScP+cWgwHwYDVR0jBBgwFoAUmAVX +j2UeCyJRWe73dBL672OFRKIwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzABhhVo +dHRwOi8vbG9jYWxob3N0Ojg4ODgwIwYDVR0RBBwwGoISc2VydmVyLmV4YW1wbGUu +Y29thwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCwLPIR8qzABd2OUvBOBXSWnXm2 +YiGY11805CISiDB+SGkRO+cd82wcjzI7bhPG9jtZ4ydrmOZ5FNmjtc0dgtmfnd3s +zGRQQ/cej5iWZbmWJOUKeCFZ1KC6c4DbwJe4iYekP0YIxcRK3hnRO1GBQ+zNwWs4 +74toxUp4dLUOuUaDZLsk0tsTqZO8alszpmIYNn1VzEguqE6FwBcFxNx110/dtH6P +chSOOG4m8ZJkZuV2PmdtOhqi4RdnUGIGS6kWjDYM6Qe+zworGzmNHv1iZMcaVvzm +YOQ54MTQ/V8o411Ckszzu1N0edC/WPBcE1ARrRMTN1hXzlGIwsLvGz51/lwd +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/tls_ocsp.test.py b/tests/gold_tests/tls/tls_ocsp.test.py new file mode 100644 index 00000000000..69f319e00e2 --- /dev/null +++ b/tests/gold_tests/tls/tls_ocsp.test.py @@ -0,0 +1,73 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server prefetched OCSP responses +''' + +# curl --cert-status option has been introduced in version 7.41.0 +Test.SkipUnless( + Condition.HasCurlVersion("7.41.0") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/ca.ocsp.pem") +ts.addSSLfile("ssl/server.ocsp.pem") +ts.addSSLfile("ssl/server.ocsp.key") +ts.addSSLfile("ssl/ocsp_response.der") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.ocsp.pem ssl_key_name=server.ocsp.key ssl_ocsp_name=ocsp_response.der' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.cert_chain.filename': 'ca.ocsp.pem', + # enable prefetched OCSP responses + 'proxy.config.ssl.ocsp.response.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.ocsp.enabled': 1, + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0 +}) + + +tr = Test.AddTestRun("Check OCSP response using curl") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Command = "curl -v --cacert {0} --cert-status -H \"host:example.com\" https://127.0.0.1:{1}".format(os.path.join(ts.Variables.SSLDir, "ca.ocsp.pem"), ts.Variables.ssl_port) +tr.ReturnCode = 0 From 00c11d0666ab3a03144d2c4cfab1772745353a64 Mon Sep 17 00:00:00 2001 From: scw00 Date: Thu, 2 May 2019 06:49:58 +0000 Subject: [PATCH 523/526] Move setsockopt from UnixNetProcessor to Server::setup_fd_for_listen --- iocore/net/Connection.cc | 19 ++++++++++++++++++- iocore/net/I_NetProcessor.h | 8 ++++++++ iocore/net/UnixNetProcessor.cc | 26 -------------------------- proxy/http/HttpProxyServerMain.cc | 4 ++++ 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/iocore/net/Connection.cc b/iocore/net/Connection.cc index a8370a65d3a..0b6d14db932 100644 --- a/iocore/net/Connection.cc +++ b/iocore/net/Connection.cc @@ -140,7 +140,8 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions ink_assert(fd != NO_FD); - if (http_accept_filter) { + if (opt.etype == ET_NET && opt.defer_accept > 0) { + http_accept_filter = true; add_http_filter(fd); } @@ -253,6 +254,22 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } #endif +#ifdef TCP_DEFER_ACCEPT + // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is + // data on the socket ready to be read + if (opt.defer_accept > 0 && (res = setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int))) < 0) { + // FIXME: should we go to the error + // goto error; + Error("[Server::listen] Defer accept is configured but set failed: %d", errno); + } +#endif + +#ifdef TCP_INIT_CWND + if (opt.init_cwnd > 0 && (res = setsockopt(fd, IPPROTO_TCP, TCP_INIT_CWND, &opt.init_cwnd, sizeof(int))) < 0) { + Error("[Server::listen] Cannot set initial congestion window to %d error: %d", tcp_init_cwnd, errno); + } +#endif + if (non_blocking) { if ((res = safe_nonblocking(fd)) < 0) { goto Lerror; diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index 5f94cd639b9..04dedf3795b 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -80,6 +80,14 @@ class NetProcessor : public Processor /// Socket transmit buffer size. /// 0 => OS default. int send_bufsize; + /// defer accpet for @c sockopt. + /// 0 => OS default. + int defer_accept; +#ifdef TCP_INIT_CWND + /// tcp init cwnd for @c sockopt + /// OS default + int init_cwnd; +#endif /// Socket options for @c sockopt. /// 0 => do not set options. uint32_t sockopt_flags; diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index 488f6db6178..c55b6187847 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -129,13 +129,6 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons Debug("http_tproxy", "Marked accept server %p on port %d for proxy protocol", na, opt.local_port); } - int should_filter_int = 0; - na->server.http_accept_filter = false; - REC_ReadConfigInteger(should_filter_int, "proxy.config.net.defer_accept"); - if (should_filter_int > 0 && opt.etype == ET_NET) { - na->server.http_accept_filter = true; - } - SessionAccept *sa = dynamic_cast(cont); na->proxyPort = sa ? sa->proxyPort : nullptr; na->snpa = dynamic_cast(cont); @@ -173,25 +166,6 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons naVec.push_back(na); } -#ifdef TCP_DEFER_ACCEPT - // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is - // data on the socket ready to be read - if (should_filter_int > 0) { - setsockopt(na->server.fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &should_filter_int, sizeof(int)); - } -#endif - -#ifdef TCP_INIT_CWND - int tcp_init_cwnd = 0; - REC_ReadConfigInteger(tcp_init_cwnd, "proxy.config.http.server_tcp_init_cwnd"); - if (tcp_init_cwnd > 0) { - Debug("net", "Setting initial congestion window to %d", tcp_init_cwnd); - if (setsockopt(na->server.fd, IPPROTO_TCP, TCP_INIT_CWND, &tcp_init_cwnd, sizeof(int)) != 0) { - Error("Cannot set initial congestion window to %d", tcp_init_cwnd); - } - } -#endif - return na->action_.get(); } diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index 11abf906584..631d54201d5 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -151,6 +151,10 @@ make_net_accept_options(const HttpProxyPort *port, unsigned nthreads) REC_ReadConfigInteger(net.recv_bufsize, "proxy.config.net.sock_recv_buffer_size_in"); REC_ReadConfigInteger(net.send_bufsize, "proxy.config.net.sock_send_buffer_size_in"); REC_ReadConfigInteger(net.sockopt_flags, "proxy.config.net.sock_option_flag_in"); + REC_ReadConfigInteger(net.defer_accept, "proxy.config.net.defer_accept"); +#ifdef TCP_INIT_CWND + REC_ReadConfigInteger(net.init_cwnd, "proxy.config.http.server_tcp_init_cwnd"); +#endif #ifdef TCP_FASTOPEN REC_ReadConfigInteger(net.tfo_queue_length, "proxy.config.net.sock_option_tfo_queue_size_in"); From c934b15aa817088bf18d724e01ba2bec3308f9cf Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Tue, 30 Apr 2019 14:17:03 +0800 Subject: [PATCH 524/526] Rewrite the assert statement to avoid Socks Proxy triggering the assertions. --- proxy/http/HttpSM.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index a01a12039f1..05630849e54 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1728,7 +1728,10 @@ HttpSM::state_http_server_open(int event, void *data) netvc = static_cast(data); session->attach_hostname(t_state.current.server->name); UnixNetVConnection *vc = static_cast(data); - ink_release_assert(pending_action == nullptr || pending_action == vc->get_action()); + // Since the UnixNetVConnection::action_ or SocksEntry::action_ may be returned from netProcessor.connect_re, and the + // SocksEntry::action_ will be copied into UnixNetVConnection::action_ before call back NET_EVENT_OPEN from SocksEntry::free(), + // so we just compare the Continuation between pending_action and VC's action_. + ink_release_assert(pending_action == nullptr || pending_action->continuation == vc->get_action()->continuation); pending_action = nullptr; session->new_connection(vc); From 96c0c71fde39c4c64b0c4dacbd44920093c6a3f2 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Tue, 30 Apr 2019 20:37:07 +0800 Subject: [PATCH 525/526] Load the Socks configuration after the host state persistence data is loaded. --- iocore/net/I_NetProcessor.h | 2 ++ iocore/net/P_UnixNetProcessor.h | 1 + iocore/net/UnixNetProcessor.cc | 21 ++++++++++++--------- proxy/shared/UglyLogStubs.cc | 6 ++++++ src/traffic_server/traffic_server.cc | 1 + 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index 04dedf3795b..6eb5a406157 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -198,6 +198,8 @@ class NetProcessor : public Processor */ virtual void init() = 0; + virtual void init_socks() = 0; + inkcoreapi virtual NetVConnection *allocate_vc(EThread *) = 0; /** Private constructor. */ diff --git a/iocore/net/P_UnixNetProcessor.h b/iocore/net/P_UnixNetProcessor.h index 7cd2208d452..87e16ba7540 100644 --- a/iocore/net/P_UnixNetProcessor.h +++ b/iocore/net/P_UnixNetProcessor.h @@ -43,6 +43,7 @@ struct UnixNetProcessor : public NetProcessor { NetVConnection *allocate_vc(EThread *t) override; void init() override; + void init_socks() override; Event *accept_thread_event; diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index c55b6187847..c18e6bdd04c 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -296,7 +296,18 @@ UnixNetProcessor::init() d.rec_int = 0; change_net_connections_throttle(nullptr, RECD_INT, d, nullptr); - // Socks + /* + * Stat pages + */ + extern Action *register_ShowNet(Continuation * c, HTTPHdr * h); + if (etype == ET_NET) { + statPagesManager.register_http("net", register_ShowNet); + } +} + +void +UnixNetProcessor::init_socks() +{ if (!netProcessor.socks_conf_stuff) { socks_conf_stuff = new socks_conf_struct; loadSocksConfiguration(socks_conf_stuff); @@ -309,14 +320,6 @@ UnixNetProcessor::init() socks_conf_stuff = netProcessor.socks_conf_stuff; } } - - /* - * Stat pages - */ - extern Action *register_ShowNet(Continuation * c, HTTPHdr * h); - if (etype == ET_NET) { - statPagesManager.register_http("net", register_ShowNet); - } } // Virtual function allows creation of an diff --git a/proxy/shared/UglyLogStubs.cc b/proxy/shared/UglyLogStubs.cc index 15ec22e53af..69523896125 100644 --- a/proxy/shared/UglyLogStubs.cc +++ b/proxy/shared/UglyLogStubs.cc @@ -88,6 +88,12 @@ UnixNetProcessor::init() ink_release_assert(false); } +void +UnixNetProcessor::init_socks() +{ + ink_release_assert(false); +} + // TODO: The following was necessary only for Solaris, should examine more. NetVCOptions const Connection::DEFAULT_OPTIONS; NetProcessor::AcceptOptions const NetProcessor::DEFAULT_ACCEPT_OPTIONS; diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 40e23026287..5674f7054df 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -1865,6 +1865,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) initCacheControl(); IpAllow::startup(); HostStatus::instance().loadHostStatusFromStats(); + netProcessor.init_socks(); ParentConfig::startup(); #ifdef SPLIT_DNS SplitDNSConfig::startup(); From 16ca8784f231ea1f38221df9bc9eafebf4bfedb5 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 17 Oct 2018 10:27:34 -0600 Subject: [PATCH 526/526] Adds voluspa - a configuration generator This generates configuration files for ATS from simpler high-level YAML specifications. It is a stand-alone command line utility and package implemented in Go. --- ci/rat-regex.txt | 5 + tools/voluspa/.gitignore | 4 + tools/voluspa/.gometalint.conf | 5 + tools/voluspa/Godeps/Godeps.json | 36 + tools/voluspa/Godeps/Readme | 5 + tools/voluspa/LICENSE | 202 ++ tools/voluspa/Makefile | 147 + tools/voluspa/README.md | 47 + tools/voluspa/adapter_config.go | 282 ++ tools/voluspa/adapters.go | 115 + tools/voluspa/adapters/access_approval.go | 83 + tools/voluspa/adapters/allow_ip.go | 86 + tools/voluspa/adapters/allow_ip_test.go | 85 + tools/voluspa/adapters/allow_methods.go | 65 + tools/voluspa/adapters/auth_proxy.go | 67 + tools/voluspa/adapters/cache_promote.go | 134 + tools/voluspa/adapters/cache_promote_test.go | 107 + tools/voluspa/adapters/cachekey.go | 125 + .../adapters/confremap/base_conf_remap.go | 63 + .../adapters/confremap/cache_version.go | 60 + tools/voluspa/adapters/confremap/common.go | 24 + .../voluspa/adapters/confremap/conf_remap.go | 107 + .../adapters/confremap/disable_cache.go | 51 + .../adapters/confremap/negative_caching.go | 68 + .../confremap/preserve_host_header.go | 82 + .../adapters/confremap/transaction_timeout.go | 68 + tools/voluspa/adapters/deny_methods.go | 69 + tools/voluspa/adapters/escalate.go | 150 + tools/voluspa/adapters/escalate_test.go | 86 + tools/voluspa/adapters/gzip.go | 198 ++ .../adapters/headerrewrite/add_header.go | 75 + .../headerrewrite/base_header_rewrite.go | 65 + .../voluspa/adapters/headerrewrite/common.go | 24 + .../headerrewrite/content_type_forge.go | 88 + .../adapters/headerrewrite/defaultttl.go | 83 + .../adapters/headerrewrite/defaultttl_test.go | 97 + tools/voluspa/adapters/headerrewrite/dscp.go | 92 + .../adapters/headerrewrite/forcettl.go | 82 + .../adapters/headerrewrite/forcettl_test.go | 95 + .../adapters/headerrewrite/header_rewrite.go | 77 + .../headerrewrite/log_cookie_header.go | 107 + .../adapters/headerrewrite/log_type.go | 74 + .../headerrewrite/proxy_cache_control.go | 77 + .../voluspa/adapters/headerrewrite/receipt.go | 101 + .../adapters/headerrewrite/remove_header.go | 74 + .../adapters/headerrewrite/rewrite_utils.go | 23 + .../adapters/headerrewrite/set_header.go | 75 + tools/voluspa/adapters/init.go | 24 + tools/voluspa/adapters/lua/base_lua.go | 65 + tools/voluspa/adapters/lua/common.go | 25 + tools/voluspa/adapters/lua/echo_cors.go | 83 + tools/voluspa/adapters/lua/lua.go | 78 + tools/voluspa/adapters/null.go | 51 + tools/voluspa/adapters/redirect.go | 87 + tools/voluspa/adapters/regex_remap.go | 76 + tools/voluspa/adapters/s3_auth.go | 139 + tools/voluspa/adapters/strip_query.go | 57 + tools/voluspa/adapters/util/ttl_utils.go | 62 + tools/voluspa/adapters/util/ttl_utils_test.go | 85 + tools/voluspa/adapters/util/util.go | 46 + .../adapters/video_background_fetch.go | 198 ++ .../adapters/video_background_fetch_test.go | 103 + tools/voluspa/attributes.go | 56 + tools/voluspa/cdncheck.go | 81 + tools/voluspa/cmd/voluspa/main.go | 172 ++ tools/voluspa/cmd/voluspa/main_test.go | 33 + tools/voluspa/cmd/voluspa/no_test.go | 20 + tools/voluspa/config_generator.go | 567 ++++ tools/voluspa/config_writer.go | 186 ++ tools/voluspa/consts.go | 30 + tools/voluspa/customer_config.go | 197 ++ tools/voluspa/doc.go | 100 + tools/voluspa/haproxy.go | 81 + tools/voluspa/hosting_config.go | 213 ++ tools/voluspa/internal/util/regex/regex.go | 24 + .../voluspa/internal/util/regex/regex_test.go | 45 + tools/voluspa/internal/util/regex/vregex.go | 33 + tools/voluspa/lifecycle.go | 29 + tools/voluspa/managed_file.go | 51 + tools/voluspa/options.go | 30 + tools/voluspa/parent_config.go | 393 +++ tools/voluspa/parent_config_test.go | 47 + tools/voluspa/property_config.go | 326 ++ tools/voluspa/property_remaps.go | 168 + tools/voluspa/receiptsd.go | 32 + tools/voluspa/registry.go | 132 + tools/voluspa/remap_options.go | 173 ++ tools/voluspa/remap_options_test.go | 66 + tools/voluspa/sample.conf-template | 178 ++ tools/voluspa/schema_v1.json | 1126 +++++++ tools/voluspa/specs/voluspa.spec | 53 + tools/voluspa/ssl_multicert.go | 199 ++ tools/voluspa/tests/.gitignore | 5 + tools/voluspa/tests/Makefile | 53 + tools/voluspa/tests/README.md | 16 + tools/voluspa/tests/other/all_bad/.gitignore | 1 + .../tests/other/all_bad/base/stderr.txt | 14 + .../all_bad/duplicated_property_name.conf | 13 + tools/voluspa/tests/other/all_bad/simple.conf | 13 + .../voluspa/tests/other/all_bad/simple2.conf | 25 + .../tests/other/cachekey_bad/base/stderr.txt | 3 + .../tests/other/cachekey_bad/test.conf | 28 + .../voluspa/tests/other/duped/base/stderr.txt | 2 + tools/voluspa/tests/other/duped/test.conf | 27 + tools/voluspa/tests/other/properties.sh | 19 + tools/voluspa/tests/other/regen_baseline.sh | 37 + .../tests/other/scheme/base/stderr.txt | 2 + tools/voluspa/tests/other/scheme/test.conf | 9 + tools/voluspa/tests/other/test.sh | 47 + .../tests/other/unicode/base/stderr.txt | 1 + .../voluspa/tests/other/unicode/httpbin.conf | 16 + .../tests/regress/all/base/all/all.config | 196 ++ .../tests/regress/all/base/all/gzip2.config | 10 + .../tests/regress/all/base/all/hdrs10.config | 4 + .../tests/regress/all/base/all/hdrs2.config | 52 + .../tests/regress/all/base/all/hdrs6.config | 4 + .../regress/all/base/all/redirect9.config | 3 + .../tests/regress/all/base/all/regex2.config | 3 + .../regress/all/base/all/strip_query2.config | 3 + .../regress/all/base/hosting.config_default | 7 + .../all/base/ssl_multicert.config_default | 9 + tools/voluspa/tests/regress/all/test.conf | 239 ++ tools/voluspa/tests/regress/properties.sh | 18 + tools/voluspa/tests/regress/regen_baseline.sh | 36 + tools/voluspa/tests/regress/test.sh | 52 + tools/voluspa/toplevel_remaps.go | 197 ++ tools/voluspa/util.go | 32 + tools/voluspa/validate.go | 312 ++ tools/voluspa/vendor/.gitignore | 9 + .../gojsonpointer/LICENSE-APACHE-2.0.txt | 202 ++ .../xeipuuv/gojsonpointer/pointer.go | 211 ++ .../gojsonreference/LICENSE-APACHE-2.0.txt | 202 ++ .../xeipuuv/gojsonreference/reference.go | 147 + .../xeipuuv/gojsonschema/.gitignore | 3 + .../gojsonschema/LICENSE-APACHE-2.0.txt | 202 ++ .../github.com/xeipuuv/gojsonschema/errors.go | 324 ++ .../xeipuuv/gojsonschema/format_checkers.go | 343 +++ .../xeipuuv/gojsonschema/internalLog.go | 37 + .../xeipuuv/gojsonschema/jsonContext.go | 72 + .../xeipuuv/gojsonschema/jsonLoader.go | 362 +++ .../xeipuuv/gojsonschema/locales.go | 313 ++ .../github.com/xeipuuv/gojsonschema/result.go | 195 ++ .../github.com/xeipuuv/gojsonschema/schema.go | 1000 ++++++ .../xeipuuv/gojsonschema/schemaLoader.go | 102 + .../xeipuuv/gojsonschema/schemaPool.go | 201 ++ .../gojsonschema/schemaReferencePool.go | 68 + .../xeipuuv/gojsonschema/schemaType.go | 83 + .../xeipuuv/gojsonschema/subSchema.go | 255 ++ .../github.com/xeipuuv/gojsonschema/types.go | 58 + .../github.com/xeipuuv/gojsonschema/utils.go | 226 ++ .../xeipuuv/gojsonschema/validation.go | 928 ++++++ tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE | 201 ++ .../vendor/gopkg.in/yaml.v2/LICENSE.libyaml | 31 + tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go | 739 +++++ .../voluspa/vendor/gopkg.in/yaml.v2/decode.go | 775 +++++ .../vendor/gopkg.in/yaml.v2/emitterc.go | 1685 +++++++++++ .../voluspa/vendor/gopkg.in/yaml.v2/encode.go | 390 +++ .../vendor/gopkg.in/yaml.v2/parserc.go | 1095 +++++++ .../vendor/gopkg.in/yaml.v2/readerc.go | 412 +++ .../vendor/gopkg.in/yaml.v2/resolve.go | 258 ++ .../vendor/gopkg.in/yaml.v2/scannerc.go | 2696 +++++++++++++++++ .../voluspa/vendor/gopkg.in/yaml.v2/sorter.go | 113 + .../vendor/gopkg.in/yaml.v2/writerc.go | 26 + tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go | 466 +++ .../voluspa/vendor/gopkg.in/yaml.v2/yamlh.go | 738 +++++ .../vendor/gopkg.in/yaml.v2/yamlprivateh.go | 173 ++ tools/voluspa/version.go | 28 + tools/voluspa/version/version.go | 39 + tools/voluspa/version/version_test.go | 20 + tools/voluspa/voluspa.go | 330 ++ 170 files changed, 27221 insertions(+) create mode 100644 tools/voluspa/.gitignore create mode 100644 tools/voluspa/.gometalint.conf create mode 100644 tools/voluspa/Godeps/Godeps.json create mode 100644 tools/voluspa/Godeps/Readme create mode 100644 tools/voluspa/LICENSE create mode 100644 tools/voluspa/Makefile create mode 100644 tools/voluspa/README.md create mode 100644 tools/voluspa/adapter_config.go create mode 100644 tools/voluspa/adapters.go create mode 100644 tools/voluspa/adapters/access_approval.go create mode 100644 tools/voluspa/adapters/allow_ip.go create mode 100644 tools/voluspa/adapters/allow_ip_test.go create mode 100644 tools/voluspa/adapters/allow_methods.go create mode 100644 tools/voluspa/adapters/auth_proxy.go create mode 100644 tools/voluspa/adapters/cache_promote.go create mode 100644 tools/voluspa/adapters/cache_promote_test.go create mode 100644 tools/voluspa/adapters/cachekey.go create mode 100644 tools/voluspa/adapters/confremap/base_conf_remap.go create mode 100644 tools/voluspa/adapters/confremap/cache_version.go create mode 100644 tools/voluspa/adapters/confremap/common.go create mode 100644 tools/voluspa/adapters/confremap/conf_remap.go create mode 100644 tools/voluspa/adapters/confremap/disable_cache.go create mode 100644 tools/voluspa/adapters/confremap/negative_caching.go create mode 100644 tools/voluspa/adapters/confremap/preserve_host_header.go create mode 100644 tools/voluspa/adapters/confremap/transaction_timeout.go create mode 100644 tools/voluspa/adapters/deny_methods.go create mode 100644 tools/voluspa/adapters/escalate.go create mode 100644 tools/voluspa/adapters/escalate_test.go create mode 100644 tools/voluspa/adapters/gzip.go create mode 100644 tools/voluspa/adapters/headerrewrite/add_header.go create mode 100644 tools/voluspa/adapters/headerrewrite/base_header_rewrite.go create mode 100644 tools/voluspa/adapters/headerrewrite/common.go create mode 100644 tools/voluspa/adapters/headerrewrite/content_type_forge.go create mode 100644 tools/voluspa/adapters/headerrewrite/defaultttl.go create mode 100644 tools/voluspa/adapters/headerrewrite/defaultttl_test.go create mode 100644 tools/voluspa/adapters/headerrewrite/dscp.go create mode 100644 tools/voluspa/adapters/headerrewrite/forcettl.go create mode 100644 tools/voluspa/adapters/headerrewrite/forcettl_test.go create mode 100644 tools/voluspa/adapters/headerrewrite/header_rewrite.go create mode 100644 tools/voluspa/adapters/headerrewrite/log_cookie_header.go create mode 100644 tools/voluspa/adapters/headerrewrite/log_type.go create mode 100644 tools/voluspa/adapters/headerrewrite/proxy_cache_control.go create mode 100644 tools/voluspa/adapters/headerrewrite/receipt.go create mode 100644 tools/voluspa/adapters/headerrewrite/remove_header.go create mode 100644 tools/voluspa/adapters/headerrewrite/rewrite_utils.go create mode 100644 tools/voluspa/adapters/headerrewrite/set_header.go create mode 100644 tools/voluspa/adapters/init.go create mode 100644 tools/voluspa/adapters/lua/base_lua.go create mode 100644 tools/voluspa/adapters/lua/common.go create mode 100644 tools/voluspa/adapters/lua/echo_cors.go create mode 100644 tools/voluspa/adapters/lua/lua.go create mode 100644 tools/voluspa/adapters/null.go create mode 100644 tools/voluspa/adapters/redirect.go create mode 100644 tools/voluspa/adapters/regex_remap.go create mode 100644 tools/voluspa/adapters/s3_auth.go create mode 100644 tools/voluspa/adapters/strip_query.go create mode 100644 tools/voluspa/adapters/util/ttl_utils.go create mode 100644 tools/voluspa/adapters/util/ttl_utils_test.go create mode 100644 tools/voluspa/adapters/util/util.go create mode 100644 tools/voluspa/adapters/video_background_fetch.go create mode 100644 tools/voluspa/adapters/video_background_fetch_test.go create mode 100644 tools/voluspa/attributes.go create mode 100644 tools/voluspa/cdncheck.go create mode 100644 tools/voluspa/cmd/voluspa/main.go create mode 100644 tools/voluspa/cmd/voluspa/main_test.go create mode 100644 tools/voluspa/cmd/voluspa/no_test.go create mode 100644 tools/voluspa/config_generator.go create mode 100644 tools/voluspa/config_writer.go create mode 100644 tools/voluspa/consts.go create mode 100644 tools/voluspa/customer_config.go create mode 100644 tools/voluspa/doc.go create mode 100644 tools/voluspa/haproxy.go create mode 100644 tools/voluspa/hosting_config.go create mode 100644 tools/voluspa/internal/util/regex/regex.go create mode 100644 tools/voluspa/internal/util/regex/regex_test.go create mode 100644 tools/voluspa/internal/util/regex/vregex.go create mode 100644 tools/voluspa/lifecycle.go create mode 100644 tools/voluspa/managed_file.go create mode 100644 tools/voluspa/options.go create mode 100644 tools/voluspa/parent_config.go create mode 100644 tools/voluspa/parent_config_test.go create mode 100644 tools/voluspa/property_config.go create mode 100644 tools/voluspa/property_remaps.go create mode 100644 tools/voluspa/receiptsd.go create mode 100644 tools/voluspa/registry.go create mode 100644 tools/voluspa/remap_options.go create mode 100644 tools/voluspa/remap_options_test.go create mode 100644 tools/voluspa/sample.conf-template create mode 100644 tools/voluspa/schema_v1.json create mode 100644 tools/voluspa/specs/voluspa.spec create mode 100644 tools/voluspa/ssl_multicert.go create mode 100644 tools/voluspa/tests/.gitignore create mode 100644 tools/voluspa/tests/Makefile create mode 100644 tools/voluspa/tests/README.md create mode 100644 tools/voluspa/tests/other/all_bad/.gitignore create mode 100644 tools/voluspa/tests/other/all_bad/base/stderr.txt create mode 100644 tools/voluspa/tests/other/all_bad/duplicated_property_name.conf create mode 100644 tools/voluspa/tests/other/all_bad/simple.conf create mode 100644 tools/voluspa/tests/other/all_bad/simple2.conf create mode 100644 tools/voluspa/tests/other/cachekey_bad/base/stderr.txt create mode 100644 tools/voluspa/tests/other/cachekey_bad/test.conf create mode 100644 tools/voluspa/tests/other/duped/base/stderr.txt create mode 100644 tools/voluspa/tests/other/duped/test.conf create mode 100644 tools/voluspa/tests/other/properties.sh create mode 100755 tools/voluspa/tests/other/regen_baseline.sh create mode 100644 tools/voluspa/tests/other/scheme/base/stderr.txt create mode 100644 tools/voluspa/tests/other/scheme/test.conf create mode 100755 tools/voluspa/tests/other/test.sh create mode 100644 tools/voluspa/tests/other/unicode/base/stderr.txt create mode 100644 tools/voluspa/tests/other/unicode/httpbin.conf create mode 100644 tools/voluspa/tests/regress/all/base/all/all.config create mode 100644 tools/voluspa/tests/regress/all/base/all/gzip2.config create mode 100644 tools/voluspa/tests/regress/all/base/all/hdrs10.config create mode 100644 tools/voluspa/tests/regress/all/base/all/hdrs2.config create mode 100644 tools/voluspa/tests/regress/all/base/all/hdrs6.config create mode 100644 tools/voluspa/tests/regress/all/base/all/redirect9.config create mode 100644 tools/voluspa/tests/regress/all/base/all/regex2.config create mode 100644 tools/voluspa/tests/regress/all/base/all/strip_query2.config create mode 100644 tools/voluspa/tests/regress/all/base/hosting.config_default create mode 100644 tools/voluspa/tests/regress/all/base/ssl_multicert.config_default create mode 100644 tools/voluspa/tests/regress/all/test.conf create mode 100644 tools/voluspa/tests/regress/properties.sh create mode 100755 tools/voluspa/tests/regress/regen_baseline.sh create mode 100755 tools/voluspa/tests/regress/test.sh create mode 100644 tools/voluspa/toplevel_remaps.go create mode 100644 tools/voluspa/util.go create mode 100644 tools/voluspa/validate.go create mode 100644 tools/voluspa/vendor/.gitignore create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go create mode 100644 tools/voluspa/version.go create mode 100644 tools/voluspa/version/version.go create mode 100644 tools/voluspa/version/version_test.go create mode 100644 tools/voluspa/voluspa.go diff --git a/ci/rat-regex.txt b/ci/rat-regex.txt index 18c5c827e59..a7aacf8029b 100644 --- a/ci/rat-regex.txt +++ b/ci/rat-regex.txt @@ -67,3 +67,8 @@ port\.h ^catch[.]hpp$ ^configuru.hpp$ ^yamlcpp$ +# tools/voluspa related +^.gometalint.conf$ +.*\.conf$ +.*config_default$ +gojsonschema(?: Apache 2.0. Properly documented in LICENSE-APACHE-2.0.txt ){0} diff --git a/tools/voluspa/.gitignore b/tools/voluspa/.gitignore new file mode 100644 index 00000000000..09a98f8e11d --- /dev/null +++ b/tools/voluspa/.gitignore @@ -0,0 +1,4 @@ +voluspa +voluspa-covered +/dist +/RPMS diff --git a/tools/voluspa/.gometalint.conf b/tools/voluspa/.gometalint.conf new file mode 100644 index 00000000000..c3719752a3d --- /dev/null +++ b/tools/voluspa/.gometalint.conf @@ -0,0 +1,5 @@ +{ + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.", + "EnableAll": true, + "Disable": ["gosec", "gas", "golint", "vetshadow", "gocyclo"] +} diff --git a/tools/voluspa/Godeps/Godeps.json b/tools/voluspa/Godeps/Godeps.json new file mode 100644 index 00000000000..3ef1bdf7daf --- /dev/null +++ b/tools/voluspa/Godeps/Godeps.json @@ -0,0 +1,36 @@ +{ + "ImportPath": "github.com/apache/trafficserver/tools/voluspa", + "GoVersion": "go1.12", + "GodepVersion": "v80", + "Packages": [ + ".", + "./adapters", + "./adapters/confremap", + "./adapters/headerrewrite", + "./adapters/lua", + "./adapters/util", + "./cmd/voluspa", + "./internal/util/regex", + "./version" + ], + "Deps": [ + { + "ImportPath": "github.com/xeipuuv/gojsonpointer", + "Rev": "4e3ac2762d5f479393488629ee9370b50873b3a6" + }, + { + "ImportPath": "github.com/xeipuuv/gojsonreference", + "Rev": "bd5ef7bd5415a7ac448318e64f11a24cd21e594b" + }, + { + "ImportPath": "github.com/xeipuuv/gojsonschema", + "Comment": "v1.1.0-14-g354ad34", + "Rev": "354ad34c2300dd6e84920ac70b41487336219e43" + }, + { + "ImportPath": "gopkg.in/yaml.v2", + "Comment": "v2.2.2-1-g7b8349a", + "Rev": "7b8349ac747c6a24702b762d2c4fd9266cf4f1d6" + } + ] +} diff --git a/tools/voluspa/Godeps/Readme b/tools/voluspa/Godeps/Readme new file mode 100644 index 00000000000..4cdaa53d56d --- /dev/null +++ b/tools/voluspa/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/tools/voluspa/LICENSE b/tools/voluspa/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/tools/voluspa/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/Makefile b/tools/voluspa/Makefile new file mode 100644 index 00000000000..c75fdd37967 --- /dev/null +++ b/tools/voluspa/Makefile @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +APP := voluspa +RELEASE ?= 0 +VERSION_BASE := 24A +BUILD_NUMBER ?= devel +VERSION := $(VERSION_BASE)$(BUILD_NUMBER) +GIT_VERSION := $(shell git describe --always --long) +PKG := github.com/apache/trafficserver/tools/voluspa +LDFLAGS := -ldflags "-w -X $(PKG).Version=$(VERSION) -X $(PKG).GitVersion=$(GIT_VERSION)" +DIST := dist +ARCHIVE := $(APP)-$(VERSION) +CURRENT_SCHEMA := schema_v1.json +DEST ?= . +GO ?= go + +COMMON_GO_FILES := *.go */*.go internal/util/*/*go */*/*.go + +GO_PKG := $(shell go list ./... | egrep -v 'vendor|cmd|other') +GO_PKG_DIRS := $(subst $(shell go list -e .),.,$(shell go list ./... | egrep -v 'vendor|other')) + +CMD_SOURCES := $(shell find cmd -name main.go) +DEV_TARGETS := $(patsubst cmd/%/main.go,%,$(CMD_SOURCES)) +DIST_TARGETS := voluspa +LINUX_TARGETS := $(patsubst %,dist/linux-x86_64/%,$(DIST_TARGETS)) +DARWIN_TARGETS := $(patsubst %,dist/darwin-x86_64/%,$(DIST_TARGETS)) + +all: default + +generate: + go generate ./... + +default: $(DEV_TARGETS) + +sanity_tests: $(APP) + @make -C tests test + +test: $(APP) sanity_tests + $(GO) test -cover $(GO_PKG_DIRS) + +lint: + -$(GO) vet ./... 2>&1 + -golint $(GO_PKG) | egrep -v 'vendor|unexport' + +metalint: + -gometalinter --config .gometalint.conf --deadline=60s -e 'vendor' ./... + +junit: + @mkdir -p .out/test-results/ # where rio junit support looks by default + $(GO) test -v $(GO_PKG_DIRS) 2>&1 | go-junit-report > .out/test-results/junit-report.xml + +cover: + # go get github.com/wadey/gocovmerge github.com/axw/gocov/... github.com/matm/gocov-html + # based on https://github.com/golang/go/issues/6909#issuecomment-233493644 + @echo Generating an overall coverage report + @rm -f *.coverprofile + go list -f '{{if or (len .TestGoFiles) (len .XTestGoFiles)}}go test -test.timeout=120s -covermode=count -coverprofile={{.Name}}_{{len .Imports}}_{{len .Deps}}.coverprofile -coverpkg ./... {{.ImportPath}}{{end}}' $(GO_PKG) | sh -x + gocovmerge `ls *.coverprofile` > cover.out + @rm -f *.coverprofile + @mkdir -p .out/coverage/ + gocov convert cover.out | gocov-html > .out/coverage/merged-coverage.html + @[[ $$(uname) == 'Darwin' ]] && open .out/coverage/merged-coverage.html || true + +voluspa-covered: cmd/voluspa/*.go $(COMMON_GO_FILES) + @# build a test version of the voluspa command with coverage enabled + @# e.g.: ./voluspa-covered -test.coverprofile cover.out .../*.conf + go test -o $(DEST)/$@ -c -tags testrunmain -coverpkg ./... $(PKG)/cmd/voluspa + +gocov: .out/coverage/coverage.html + +.out/coverage/coverage.json: voluspa-covered + @mkdir -p .out/coverage/ + gocov test $(GO_PKG) > .out/coverage/coverage.json + @make -C tests coverage + gocov convert tests/regress/profile*.out > tests/regress/system.json + gocov convert tests/other/profile*.out > tests/other/system.json + cat .out/coverage/coverage.json tests/other/system.json tests/regress/system.json | gocov-merge > .out/coverage/coverage.json + +.out/coverage/coverage.html: .out/coverage/coverage.json + gocov-html < .out/coverage/coverage.json > .out/coverage/coverage.html + +dist: artifactory_targets $(DIST)/artifactory/version.txt $(DIST)/artifactory/versions.sh + +clean: + @rm -rf $(DIST) out out_remap RPMS .dist .out + @rm -f $(DEV_TARGETS) $(TOOLS_TARGETS) $(OTHER_TARGETS) voluspa-covered + @make -C tests clean + +%: cmd/%/*.go $(COMMON_GO_FILES) + $(GO) build -o $(DEST)/$@ $(LDFLAGS) cmd/$@/*.go + +$(DIST)/linux-x86_64/%: + GOOS=linux GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/darwin-x86_64/%: + GOOS=darwin GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/freebsd-x86_64/%: + GOOS=freebsd GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/artifactory: + @mkdir -p $@ + +$(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64: $(DIST)/artifactory dist/darwin-x86_64/voluspa + cp dist/darwin-x86_64/voluspa $@ + +$(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64: $(DIST)/artifactory dist/linux-x86_64/voluspa + cp dist/linux-x86_64/voluspa $@ + +$(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION): $(DIST)/artifactory $(CURRENT_SCHEMA) + cp $(CURRENT_SCHEMA) $@ + +$(DIST)/artifactory/version.txt: + @mkdir -p dist/artifactory + echo $(VERSION) > $@ + +artifactory_targets: $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 $(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION) + +$(DIST)/artifactory/versions.sh: $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 + echo VOLUSPA_VERSION=$(VERSION) > $@ + echo >> $@ + sha1sum $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 | awk 'BEGIN { FS = " " } ; { printf "linux_SHA1=%s\n", $$1 }' >> $@ + sha1sum $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 | awk 'BEGIN { FS = " " } ; { printf "darwin_SHA1=%s\n", $$1 }' >> $@ + sha1sum $(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION) | awk 'BEGIN { FS = " " } ; { printf "schema_SHA1=%s\n", $$1 }' >> $@ + +update-deps: + -grep ImportPath Godeps/Godeps.json | cut -f 4 -d \" | grep -v voluspa | xargs go get -u + @rm -rf Godeps vendor/* + godep save $(GO_PKG_DIRS) + +.PHONY: all clean test linux dist sanity_tests diff --git a/tools/voluspa/README.md b/tools/voluspa/README.md new file mode 100644 index 00000000000..f712ccc34bb --- /dev/null +++ b/tools/voluspa/README.md @@ -0,0 +1,47 @@ +# Voluspa + +### Apache Traffic Server Configuration Tool (experiment) + +A tool to codify the CDN feature set and make it easier and more reliable to setup new CDN properties. + +Configuration files are well-formed YAML files that express the desired behaviour. + + +## Usage +``` +$ ./voluspa -dest=ourconfigs myproperty.conf + +$ find ourconfigs -type f +ourconfigs/ssl_multicert.config_tenant +ourconfigs/myproperty/myproperty.config +ourconfigs/myproperty/hdrs.config +ourconfigs/ssl_multicert.config_default +ourconfigs/voluspa.sls +``` + +The voluspa configuration language is defined by a +[JSON Schema](https://spacetelescope.github.io/understanding-json-schema/index.html) +stored in the [schema file](schema_v1.json). + +Tool will generally error out if something is wrong (missing required fields) or weirdly configured options. +Warnings will be generated for unknown configuration options (but will attempt to still process the file). +There are a number of options to control the output and output location (see -help) + +By default, all configs get a receipt header and propstats enabled unless disabled via the appropriate option. + +## Extending + +### General process includes: +- adding the shared library name (sharedLibraryNames) +- configuration option name (see pluginConfigNameMap) +- weighting the plugin (for sorting) (pluginWeights) +- most importantly, the class/struct/file for the plugin +- a bit more complicated if it's a "compound plugin" (eg HeaderRewrite) + + +## Tests +There are a couple of go-based unit tests (no where near comprehensive) and there's what I would call sanity tests in tests/. The script "test.sh" will use the current build of the tool to regenerate the remap configs for a given yaml file; and do a diff between base/ and the new output. Generally, there should be no changes between runs. If there are expected changes, the baselines would need to be regenerated. The configurations tested range from very simple to something exercising all current configuration options. + + +## TODO +- error on mutually exclusive options (eg allow\_ip and deny\_methods) diff --git a/tools/voluspa/adapter_config.go b/tools/voluspa/adapter_config.go new file mode 100644 index 00000000000..1ee9c03b4c1 --- /dev/null +++ b/tools/voluspa/adapter_config.go @@ -0,0 +1,282 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +type AdapterType string + +const ( + UnknownAdapter AdapterType = "unknown" + Receipt = "receipt" +) + +type ConfigLocation int + +const ( + UnknownLocation ConfigLocation = iota + ChildConfig + ParentConfig +) + +// validConfigrationOptions are options that are not plugin specific (which are managed by the plugin) +var validConfigurationOptions = map[string]bool{ + "parent_child": true, + Parent: true, + Child: true, +} + +type mappingRule struct { + Type AdapterType + Weight int + SubConfigFileName string + SubConfigContent *bytes.Buffer + ConfigContent *bytes.Buffer +} + +func newMappingRule(vp Adapter) *mappingRule { + return &mappingRule{Type: vp.Type(), Weight: getWeight(vp)} +} + +// hasContent returns true if there's configuration content, false otherwise +func (p *mappingRule) hasContent() bool { + return (p.SubConfigContent != nil && p.SubConfigContent.Len() > 0) || + (p.ConfigContent != nil && p.ConfigContent.Len() > 0) +} + +func (p *mappingRule) isCommandLineTemplated() bool { + return p.ConfigContent != nil && strings.Contains(p.ConfigContent.String(), "{%") +} + +// pluginWeights affects the ordering of the configuration output. +// Only add to this list if you have hard ordering requirements, otherwise just stick +// with the default weighting (5) +func getWeight(vp Adapter) int { + t, ok := vp.(Weighty) + if !ok { + return 5 + } + return t.Weight() +} + +// SubConfigFilename returns the subconfig filename given plugin and property name +func SubConfigFilename(p SubConfigAdapter, env *ConfigEnvironment) string { + extension := "config" + name := p.Name() + if strings.HasSuffix(p.Name(), ".lua") { + extension = "lua" + name = strings.TrimSuffix(name, ".lua") + } + if env.id == 1 { + if env.ConfigLocation == ParentConfig { + return fmt.Sprintf("%s_parent.%s", name, extension) + } + return fmt.Sprintf("%s.%s", name, extension) + } + + if env.ConfigLocation == ParentConfig { + return fmt.Sprintf("%s%d_parent.%s", name, env.id, extension) + } + return fmt.Sprintf("%s%d.%s", name, env.id, extension) +} + +// ConfigFileNameSpecification appends to buffer the sub config filename configuration specification +// nolint: interfacer +func ConfigFileNameSpecification(p Adapter, env *ConfigEnvironment, buffer *bytes.Buffer) (*bytes.Buffer, error) { + var configFileSwitch string + cfs, ok := p.(ConfigFileSwitch) + if ok { + configFileSwitch = cfs.ConfigFileSwitch() + } + + scp, ok := p.(SubConfigAdapter) + if ok { + fmt.Fprintf(buffer, " @pparam=%s%s/%s", configFileSwitch, strings.ToLower(env.Property), SubConfigFilename(scp, env)) + } + return buffer, nil +} + +func isParameterBasedAdapter(p Adapter) bool { + _, ok := p.(ParameterAdapter) + if ok { + return true + } + + _, ok = p.(RoleifiedParameterAdapter) + + return ok +} + +func ConfigContent(p Adapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + + var buffer bytes.Buffer + + var roleEnabled bool + rep, ok := p.(RoleEnabled) + if ok { + roleEnabled = true + fmt.Fprintf(&buffer, "{%% if salt.pillar.get(\"%s\") %%}%s", rep.Role(), remapPrefixBuffer) + } + + sla, ok := p.(SharedLibraryAdapter) + if ok { + fmt.Fprintf(&buffer, "@plugin=%s", sla.SharedLibraryName()) + } + + if !isParameterBasedAdapter(p) { + return ConfigFileNameSpecification(p, env, &buffer) + } + + var err error + var subfer *bytes.Buffer + switch vpp := p.(type) { + case ParameterAdapter: + subfer, err = processParameterAdapter(p, env, vpp) + case RoleifiedParameterAdapter: + subfer, err = processRoleifiedParameterAdapter(vpp, env) + default: + return nil, fmt.Errorf("unhandled adapter %s", vpp) + } + + if err != nil { + return nil, err + } + + // if sub-buffer is nil, pass back nil + if subfer == nil { + return nil, nil + } + + buffer.Write(subfer.Bytes()) + + if roleEnabled { + fmt.Fprintf(&buffer, " \\{%% else %%}%s\\{%% endif %%}", remapPrefixBuffer) + fmt.Fprintln(&buffer, "") + } + + return &buffer, nil +} + +func processParameterAdapter(p Adapter, env *ConfigEnvironment, vpp ParameterAdapter) (*bytes.Buffer, error) { + var buffer bytes.Buffer + + // throw away all work if we get an error (which can be a signal not to include this plugin (see propstats)) + // TODO: signal with appropriate error (like ErrSkipConfig) + pparams, err := vpp.PParams(env) + if err != nil { + return &buffer, err + } + + raw := false + rpparams, ok := vpp.(RawParameterAdapter) + if ok { + raw = rpparams.Raw() + } + + if len(pparams) == 0 { + if p.PluginType() == CompoundAdapter { + return &buffer, nil + } + // no buffer to append? + return nil, nil + } + + if raw { + for _, val := range pparams { + fmt.Fprintf(&buffer, " %s", val) + } + } else { + for _, val := range pparams { + fmt.Fprintf(&buffer, " @pparam=%s", val) + } + } + + return &buffer, nil +} + +func processRoleifiedParameterAdapter(vpp RoleifiedParameterAdapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + var buffer bytes.Buffer + pparams, err := vpp.PParams(env) + if err != nil { + return &buffer, err + } + + // extract non-Default role keys from parameter list + var roleKeys []string + for key := range pparams { + if key == DefaultRole { + continue + } + roleKeys = append(roleKeys, key) + } + sort.Strings(roleKeys) + + // 2 passes + // 1. shove the pparams and role-guards into a list + // 2. iterate over that listen, building up buffer with the \n and \\ and whitespace + + var args []string + pparam, found := pparams[DefaultRole] + if found { + for _, val := range pparam { + args = append(args, fmt.Sprintf("@pparam=%s", val)) + } + } + + for _, key := range roleKeys { + args = append(args, fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}%s", key, remapPrefixBuffer)) + + for _, val := range pparams[key] { + args = append(args, fmt.Sprintf("@pparam=%s", val)) + } + args = append(args, fmt.Sprintf(" \\{%% else %%}%s\\{%% endif %%}", remapPrefixBuffer)) + } + + // peel off last arg for special handling + lastArg := args[len(args)-1] + args = args[:len(args)-1] + + for _, arg := range args { + fmt.Fprintf(&buffer, " %s", arg) + } + + // TODO: assuming last element is {% endif %} block + fmt.Fprintf(&buffer, "%s\n", lastArg) + + return &buffer, nil +} + +func SubConfigContent(p Adapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + cba, ok := p.(ContentBasedAdapter) + if ok { + return cba.Content(env) + } + return nil, nil +} + +// IsValidConfigurationOption returns true if configuration key is valid +func IsValidConfigurationOption(option string) bool { + return validConfigurationOptions[option] +} diff --git a/tools/voluspa/adapters.go b/tools/voluspa/adapters.go new file mode 100644 index 00000000000..b705011a596 --- /dev/null +++ b/tools/voluspa/adapters.go @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "bytes" + +type PluginType int + +const ( + UnknownVoluspaAdapter PluginType = iota + GeneralAdapter + ActionAdapter + CompoundAdapter +) + +type ConfigEnvironment struct { + Property string + Options Options + RemapOptions RemapOptions + ConfigLocation ConfigLocation + id int +} + +func newConfigEnvironment(p *PropertyConfig, options Options, mappingID int, configLocation ConfigLocation) *ConfigEnvironment { + return &ConfigEnvironment{ + Property: p.Property, + Options: options, + ConfigLocation: configLocation, + id: mappingID, + } +} + +// Adapter needs to be implemented by all adapters +type Adapter interface { + PluginType() PluginType + Type() AdapterType + ConfigParameters() []string +} + +// ContentBasedAdapter should be implemented if a configuration is a blob of text (opposite of ParameterAdapter) +type ContentBasedAdapter interface { + Content(*ConfigEnvironment) (*bytes.Buffer, error) +} + +// SharedLibraryAdapter should be implemented if a configuration has an associated shared library +type SharedLibraryAdapter interface { + SharedLibraryName() string +} + +// CompoundTypeAdapter should be implemented if an adapter's output is shared with other like adapters +type CompoundTypeAdapter interface { + CompoundType() AdapterType +} + +// ParameterAdapter should be implemented if a configuration has @pparams +type ParameterAdapter interface { + PParams(*ConfigEnvironment) ([]string, error) +} + +// RawParameterAdapter should be implemented if a configuration has @pparams that should not be processed +type RawParameterAdapter interface { + Raw() bool +} + +// RolePParams is a map of role names to a list of PParam options +type RolePParams map[string][]string + +// RoleifiedParameterAdapter should be implemented in adapters +// that can vary their options based on host role/type +type RoleifiedParameterAdapter interface { + PParams(*ConfigEnvironment) (RolePParams, error) +} + +// SubConfigAdapter should be implemented if a sub config is involved with rule/plugin +type SubConfigAdapter interface { + Name() string +} + +// ConfigFileSwitch should be implemented if a sub config is specified with a plugin switch (eg --config=) +type ConfigFileSwitch interface { + ConfigFileSwitch() string +} + +// ParentChildAdapter should be implemented if plugin is conditionally placed based on parent/child context +type ParentChildAdapter interface { + ConfigLocations() []ConfigLocation +} + +// RoleEnabled should be implemented if command-line should be protected/enabled by a jinja role-block +type RoleEnabled interface { + Role() string +} + +// Weighty should be implemented if an Adapter's output needs to be placed in a certain +// order (in relation to other adapter's output) +type Weighty interface { + Weight() int +} diff --git a/tools/voluspa/adapters/access_approval.go b/tools/voluspa/adapters/access_approval.go new file mode 100644 index 00000000000..b49b000ecae --- /dev/null +++ b/tools/voluspa/adapters/access_approval.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + AccessApprovalAdapterParameter = "access_approval" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AccessApprovalAdapter{}) +} + +type AccessApprovalAdapter struct { +} + +func (*AccessApprovalAdapter) Weight() int { + return 1 +} + +func (p *AccessApprovalAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *AccessApprovalAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("access_approval") +} + +func (p *AccessApprovalAdapter) Name() string { + return "access_approval" +} + +func (p *AccessApprovalAdapter) ConfigParameters() []string { + return []string{AccessApprovalAdapterParameter} +} + +func (p *AccessApprovalAdapter) SharedLibraryName() string { + return "access_control.so" +} + +func (p *AccessApprovalAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(AccessApprovalAdapterParameter) + if err != nil { + return nil, util.FormatError(AccessApprovalAdapterParameter, err) + } + + val, ok := alt["raw"] + if !ok { + return nil, nil + } + + var pparams []string + pparams = append(pparams, strings.Split(val.(string), "\n")...) + + return pparams, nil +} + +func (p *AccessApprovalAdapter) Raw() bool { + return true +} diff --git a/tools/voluspa/adapters/allow_ip.go b/tools/voluspa/adapters/allow_ip.go new file mode 100644 index 00000000000..f7dbf5abdf3 --- /dev/null +++ b/tools/voluspa/adapters/allow_ip.go @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + allowIPAdapterParameter = "allow_ip" + allowIPAdapterParameterParent = "allow_ip_parent" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AllowIPAdapter{}) +} + +type AllowIPAdapter struct { +} + +func (*AllowIPAdapter) Weight() int { + return 120 +} +func (p *AllowIPAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *AllowIPAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("allow_ip") +} + +func (p *AllowIPAdapter) ConfigParameters() []string { + return []string{allowIPAdapterParameter, allowIPAdapterParameterParent} +} + +func (p *AllowIPAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + switch env.ConfigLocation { + case voluspa.UnknownLocation, voluspa.ChildConfig: + return p.configContent(env, allowIPAdapterParameter) + case voluspa.ParentConfig: + return p.configContent(env, allowIPAdapterParameterParent) + } + return nil, fmt.Errorf("Unhandled configLocation %q", env.ConfigLocation) +} + +func (p *AllowIPAdapter) configContent(env *voluspa.ConfigEnvironment, parameterName string) (*bytes.Buffer, error) { + ips, err := env.RemapOptions.ValueByNameAsSlice(parameterName) + if err != nil { + return nil, nil + } + + if len(ips) == 0 { + return nil, util.FormatError(parameterName, fmt.Errorf("empty IP range speclist")) + } + + // TODO other validation on actual content + + content := &bytes.Buffer{} + for _, ip := range ips { + fmt.Fprintf(content, "@src_ip=%s ", ip) + } + content.WriteString("@action=allow") + + return content, nil +} diff --git a/tools/voluspa/adapters/allow_ip_test.go b/tools/voluspa/adapters/allow_ip_test.go new file mode 100644 index 00000000000..736d81b8938 --- /dev/null +++ b/tools/voluspa/adapters/allow_ip_test.go @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +var ( + childIPRange = []string{"17.0.0.0-17.255.255.255", "10.0.0.0-10.255.255.255"} + parentIPRange = []string{"17.253.0.0-17.255.255.255", "10.253.0.0-10.255.255.255"} +) + +func TestIPAllowAdapter(t *testing.T) { + var plugin AllowIPAdapter + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + (*ro)[allowIPAdapterParameter] = "" + out, err := plugin.Content(env) + if err == nil && out != nil { + t.Fatalf("Expected error for invalid configuration.") + } + + (*ro)[allowIPAdapterParameter] = []string{} + _, err = plugin.Content(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + (*ro)[allowIPAdapterParameter] = childIPRange + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected := "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } + + // "child" config. should still return child version + (*ro)[allowIPAdapterParameterParent] = parentIPRange + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected = "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } + + ro = &voluspa.RemapOptions{} + env = &voluspa.ConfigEnvironment{RemapOptions: *ro, ConfigLocation: voluspa.ParentConfig} + (*ro)[allowIPAdapterParameterParent] = []string{"17.0.0.0-17.255.255.255", "10.0.0.0-10.255.255.255"} + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected = "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } +} diff --git a/tools/voluspa/adapters/allow_methods.go b/tools/voluspa/adapters/allow_methods.go new file mode 100644 index 00000000000..02ff33a92f6 --- /dev/null +++ b/tools/voluspa/adapters/allow_methods.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AllowMethodsAdapter{}) +} + +type AllowMethodsAdapter struct { +} + +func (p *AllowMethodsAdapter) Weight() int { + return 140 +} + +func (p *AllowMethodsAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *AllowMethodsAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("allow_methods") +} + +func (p *AllowMethodsAdapter) ConfigParameters() []string { + return []string{"allow_methods"} +} + +func (p *AllowMethodsAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + methods, err := env.RemapOptions.ValueByNameAsSlice("allow_methods") + if err != nil { + return nil, err + } + + content := &bytes.Buffer{} + content.WriteString("@action=allow") + for _, method := range methods { + fmt.Fprintf(content, " @method=%s", method) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/auth_proxy.go b/tools/voluspa/adapters/auth_proxy.go new file mode 100644 index 00000000000..3d9d4be4736 --- /dev/null +++ b/tools/voluspa/adapters/auth_proxy.go @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "errors" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AuthProxyAdapter{}) +} + +type AuthProxyAdapter struct { +} + +func (p *AuthProxyAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *AuthProxyAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("auth_proxy") +} + +func (p *AuthProxyAdapter) Name() string { + return "authproxy" +} + +func (p *AuthProxyAdapter) ConfigParameters() []string { + return []string{"authproxy"} +} + +func (p *AuthProxyAdapter) SharedLibraryName() string { + return "authproxy.so" +} + +func (p *AuthProxyAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *AuthProxyAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + value, err := env.RemapOptions.ValueByNameAsString(p.Name()) + if err != nil && len(value) == 0 { + return nil, errors.New("Need transform parameter") + } + + return []string{fmt.Sprintf("--auth-transform=%s", value)}, nil +} diff --git a/tools/voluspa/adapters/cache_promote.go b/tools/voluspa/adapters/cache_promote.go new file mode 100644 index 00000000000..16b38f05690 --- /dev/null +++ b/tools/voluspa/adapters/cache_promote.go @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + CachePromoteAdapterParameter = "cache_promote" + CachePromoteAdapterPolicyParameter = "policy" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CachePromoteAdapter{}) +} + +type CachePromoteAdapter struct { +} + +func (p *CachePromoteAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *CachePromoteAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_promote") +} + +func (p *CachePromoteAdapter) Name() string { + return "cache_promote" +} + +func (p *CachePromoteAdapter) SharedLibraryName() string { + return "cache_promote.so" +} + +func (p *CachePromoteAdapter) Role() string { + return "roles_trafficserver_cache_promote" +} + +func (p *CachePromoteAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *CachePromoteAdapter) ConfigParameters() []string { + return []string{CachePromoteAdapterParameter} +} + +func (p *CachePromoteAdapter) isValidPolicy(policy string) bool { + return policy == "lru" || policy == "chance" +} + +func (p *CachePromoteAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + + var alt map[string]interface{} + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(CachePromoteAdapterParameter) + if err != nil { + return nil, err + } + + if err = p.validateArgs(alt); err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *CachePromoteAdapter) validateArgs(alt map[string]interface{}) error { + val := util.ParamToValue(alt["policy"]) + if len(val) == 0 || !p.isValidPolicy(val) { + return util.FormatError(CachePromoteAdapterPolicyParameter, fmt.Errorf("invalid policy specified")) + } + + return nil +} + +func (p *CachePromoteAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["policy"], "policy", args) + args = p.maybeAddArgs(alt["sample"], "sample", args) + + policy := util.ParamToValue(alt["policy"]) + if policy != "lru" { + return args, nil + } + args = p.maybeAddArgs(alt["lru_hits"], "hits", args) + args = p.maybeAddArgs(alt["lru_buckets"], "buckets", args) + + return args, nil +} + +func (p *CachePromoteAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := util.ParamToValue(option) + switch param { + case "policy": + if p.isValidPolicy(value) { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + case "sample": + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s%%", param, value)) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/cache_promote_test.go b/tools/voluspa/adapters/cache_promote_test.go new file mode 100644 index 00000000000..99d3e322596 --- /dev/null +++ b/tools/voluspa/adapters/cache_promote_test.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestCachePromoteAdapter(t *testing.T) { + var plugin CachePromoteAdapter + ro := &voluspa.RemapOptions{} + subOptions := make(map[interface{}]interface{}) + (*ro)["cache_promote"] = subOptions + + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + _, err := plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration") + } + + subOptions["policy"] = "notapolicy" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration") + } + + subOptions["policy"] = "lru" + out, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected := "--policy=lru" + if out[0] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[0], expected) + } + + subOptions["sample"] = 66 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined := strings.Join(out, " ") + expected = "--policy=lru --sample=66%" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } + + subOptions["lru_hits"] = 1000 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=lru --sample=66% --hits=1000" + if joined != expected { + t.Fatalf("Error: got '%s'; expected '%s'", joined, expected) + } + + subOptions["lru_buckets"] = 10000 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=lru --sample=66% --hits=1000 --buckets=10000" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } + + // Non-LRU policy's should drop hits and buckets from command-line + subOptions["policy"] = "chance" + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=chance --sample=66%" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } +} diff --git a/tools/voluspa/adapters/cachekey.go b/tools/voluspa/adapters/cachekey.go new file mode 100644 index 00000000000..a349dcd5e19 --- /dev/null +++ b/tools/voluspa/adapters/cachekey.go @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + CacheKeyAdapterParameter = "cachekey" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CacheKeyAdapter{}) +} + +type CacheKeyAdapter struct { +} + +func (p *CacheKeyAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *CacheKeyAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_key") +} + +func (p *CacheKeyAdapter) SharedLibraryName() string { + return "cachekey.so" +} + +func (p *CacheKeyAdapter) ConfigParameters() []string { + return []string{CacheKeyAdapterParameter} +} + +func (p *CacheKeyAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(CacheKeyAdapterParameter) + if err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *CacheKeyAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["include_params"], "include-params", args) + args = p.maybeAddArgs(alt["include_cookies"], "include-cookies", args) + args = p.maybeAddArgs(alt["include_headers"], "include-headers", args) + args = p.maybeAddArgs(alt["exclude_params"], "exclude-params", args) + args = p.maybeAddArgs(alt["static_prefix"], "static-prefix", args) + args = p.maybeAddArgs(alt["sort_query"], "sort-params", args) + args = p.maybeAddArgs(alt["remove_all_params"], "remove-all-params", args) + args = p.maybeAddArgs(alt["regex_replace_path"], "capture-path", args) + args = p.maybeAddArgs(alt["regex_replace_path_uri"], "capture-path-uri", args) + args = p.maybeAddArgs(alt["capture_header"], "capture-header", args) + + return args, nil +} + +func (p *CacheKeyAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := v1.(string) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} + +func (p *CacheKeyAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + + // capture-header needs a separate "--capture-header=" flag for each one when there are multiple + if param == "capture-header" && option != nil { + for _, o := range option.([]interface{}) { + args = append(args, "--capture-header="+o.(string)) + } + return args + } + + value := p.paramToValue(option) + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + + return args +} diff --git a/tools/voluspa/adapters/confremap/base_conf_remap.go b/tools/voluspa/adapters/confremap/base_conf_remap.go new file mode 100644 index 00000000000..4f5a61cac55 --- /dev/null +++ b/tools/voluspa/adapters/confremap/base_conf_remap.go @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseConfRemapAdapter{}) +} + +type BaseConfRemapAdapter struct { +} + +func (*BaseConfRemapAdapter) Weight() int { + return 17 +} + +func (p *BaseConfRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseConfRemapAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseConfRemapAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseConfRemapAdapter) Name() string { + return "base_conf_remap" +} + +func (p *BaseConfRemapAdapter) SharedLibraryName() string { + return "conf_remap.so" +} + +func (p *BaseConfRemapAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseConfRemapAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return nil, nil +} diff --git a/tools/voluspa/adapters/confremap/cache_version.go b/tools/voluspa/adapters/confremap/cache_version.go new file mode 100644 index 00000000000..4f0802490ca --- /dev/null +++ b/tools/voluspa/adapters/confremap/cache_version.go @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CacheVersionAdapter{}) +} + +type CacheVersionAdapter struct { +} + +func (s *CacheVersionAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (s *CacheVersionAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_version") +} + +func (s *CacheVersionAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (s *CacheVersionAdapter) ConfigParameters() []string { + return []string{"cache_version"} +} + +func (s *CacheVersionAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + cacheVersion, err := env.RemapOptions.ValueByNameAsInt("cache_version") + if err != nil { + return nil, err + } + + return []string{ + fmt.Sprintf("proxy.config.http.cache.generation=%d", cacheVersion), + }, nil +} diff --git a/tools/voluspa/adapters/confremap/common.go b/tools/voluspa/adapters/confremap/common.go new file mode 100644 index 00000000000..6f64559ac12 --- /dev/null +++ b/tools/voluspa/adapters/confremap/common.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("conf_remap") diff --git a/tools/voluspa/adapters/confremap/conf_remap.go b/tools/voluspa/adapters/confremap/conf_remap.go new file mode 100644 index 00000000000..a687141e5c2 --- /dev/null +++ b/tools/voluspa/adapters/confremap/conf_remap.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformConfRemapAdapter{}) +} + +type FreeformConfRemapAdapter struct { +} + +func (*FreeformConfRemapAdapter) Weight() int { + return 17 +} + +func (p *FreeformConfRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformConfRemapAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_conf_remap") +} + +func (p *FreeformConfRemapAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformConfRemapAdapter) ConfigParameters() []string { + return []string{"conf_remap"} +} + +// CONFIG proxy.config.http.connect_attempts_timeout INT 600 +var confRegexp = regexp.MustCompile(`CONFIG (.*) ([A-Z]+) (.*)`) + +func (p *FreeformConfRemapAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + val, err := env.RemapOptions.ValueByNameAsString("conf_remap") + if err != nil { + return nil, err + } + + seen := make(map[string]bool) + + var params []string + + lines := strings.Split(val, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // eg CONFIG proxy.config.http.connect_attempts_timeout INT 600 + vals := confRegexp.FindStringSubmatch(line) + if vals == nil { + return nil, fmt.Errorf("%s is not properly formed", line) + } + + if _, ok := seen[line]; ok { + return nil, fmt.Errorf("Already seen key '%s'", vals[1]) + } + seen[line] = true + + switch vals[2] { + case "INT": + _, err := strconv.Atoi(vals[3]) + if err != nil { + return nil, fmt.Errorf("Value is not of appropriate type; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + case "FLOAT": + _, err := strconv.ParseFloat(vals[3], 64) + if err != nil { + return nil, fmt.Errorf("Value is not of appropriate type; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + case "STRING": + default: + return nil, fmt.Errorf("Type is unknown/unhandled; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + params = append(params, fmt.Sprintf("%s=%s", vals[1], vals[3])) + } + return params, nil +} diff --git a/tools/voluspa/adapters/confremap/disable_cache.go b/tools/voluspa/adapters/confremap/disable_cache.go new file mode 100644 index 00000000000..06398e78476 --- /dev/null +++ b/tools/voluspa/adapters/confremap/disable_cache.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DisableCacheAdapter{}) +} + +type DisableCacheAdapter struct { +} + +func (p *DisableCacheAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DisableCacheAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("disable_cache") +} + +func (p *DisableCacheAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DisableCacheAdapter) ConfigParameters() []string { + return []string{"disable_cache"} +} + +func (p *DisableCacheAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return []string{"proxy.config.http.cache.http=0"}, nil +} diff --git a/tools/voluspa/adapters/confremap/negative_caching.go b/tools/voluspa/adapters/confremap/negative_caching.go new file mode 100644 index 00000000000..9c9af2a1c49 --- /dev/null +++ b/tools/voluspa/adapters/confremap/negative_caching.go @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + NegativeCachingAdapterParameterTTL = "negative_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&NegativeCachingAdapter{}) +} + +type NegativeCachingAdapter struct { +} + +func (p *NegativeCachingAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *NegativeCachingAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("negative_caching") +} + +func (p *NegativeCachingAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *NegativeCachingAdapter) ConfigParameters() []string { + return []string{NegativeCachingAdapterParameterTTL} +} + +func (p *NegativeCachingAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + duration, err := env.RemapOptions.ValueByNameAsString(NegativeCachingAdapterParameterTTL) + if err != nil { + return nil, err + } + + seconds, err := util.DurationToSeconds(duration) + if err != nil { + return nil, err + } + + return []string{fmt.Sprintf("proxy.config.http.negative_caching_lifetime=%s", seconds)}, nil +} diff --git a/tools/voluspa/adapters/confremap/preserve_host_header.go b/tools/voluspa/adapters/confremap/preserve_host_header.go new file mode 100644 index 00000000000..a2d35f0d7c3 --- /dev/null +++ b/tools/voluspa/adapters/confremap/preserve_host_header.go @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + OriginHostHeaderAdapterParameter = "origin_host_header" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&PreserveHostHeaderAdapter{}) +} + +type PreserveHostHeaderAdapter struct { +} + +func (*PreserveHostHeaderAdapter) Weight() int { + return 25 +} + +func (p *PreserveHostHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *PreserveHostHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("preserve_host_header") +} + +func (p *PreserveHostHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *PreserveHostHeaderAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *PreserveHostHeaderAdapter) ConfigParameters() []string { + return []string{OriginHostHeaderAdapterParameter} +} + +func (p *PreserveHostHeaderAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + intval := 0 + if env.RemapOptions.HasOption(OriginHostHeaderAdapterParameter) { + ohhValue, err := env.RemapOptions.ValueByNameAsString(OriginHostHeaderAdapterParameter) + if err != nil { + return nil, util.FormatError(OriginHostHeaderAdapterParameter, err) + } + switch ohhValue { + case "alias": + intval = 1 + case "origin": + intval = 0 + default: + return nil, util.FormatError(OriginHostHeaderAdapterParameter, fmt.Errorf("unknown value %q", ohhValue)) + } + } + + return []string{fmt.Sprintf("proxy.config.url_remap.pristine_host_hdr=%d", intval)}, nil +} diff --git a/tools/voluspa/adapters/confremap/transaction_timeout.go b/tools/voluspa/adapters/confremap/transaction_timeout.go new file mode 100644 index 00000000000..db462cf17ae --- /dev/null +++ b/tools/voluspa/adapters/confremap/transaction_timeout.go @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&TransactionTimeoutAdapter{}) +} + +type TransactionTimeoutAdapter struct { +} + +func (s *TransactionTimeoutAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (s *TransactionTimeoutAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("transaction_timeout") +} + +func (s *TransactionTimeoutAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (s *TransactionTimeoutAdapter) ConfigParameters() []string { + return []string{"transaction_timeout"} +} + +func (s *TransactionTimeoutAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + duration, err := env.RemapOptions.ValueByNameAsString("transaction_timeout") + if err != nil { + return nil, err + } + + seconds, err := util.DurationToSeconds(duration) + if err != nil { + return nil, err + } + + return []string{ + fmt.Sprintf("proxy.config.http.connect_attempts_timeout=%s", seconds), + fmt.Sprintf("proxy.config.http.transaction_no_activity_timeout_in=%s", seconds), + fmt.Sprintf("proxy.config.http.transaction_no_activity_timeout_out=%s", seconds), + }, nil +} diff --git a/tools/voluspa/adapters/deny_methods.go b/tools/voluspa/adapters/deny_methods.go new file mode 100644 index 00000000000..580447dd67f --- /dev/null +++ b/tools/voluspa/adapters/deny_methods.go @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DenyMethodsAdapter{}) +} + +type DenyMethodsAdapter struct { +} + +func (*DenyMethodsAdapter) Weight() int { + return 150 +} + +func (p *DenyMethodsAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *DenyMethodsAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("deny_methods") +} + +func (p *DenyMethodsAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *DenyMethodsAdapter) ConfigParameters() []string { + return []string{"deny_methods"} +} + +func (p *DenyMethodsAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + methods, err := env.RemapOptions.ValueByNameAsSlice("deny_methods") + if err != nil { + return nil, err + } + + content := &bytes.Buffer{} + content.WriteString("@action=deny") + for _, method := range methods { + fmt.Fprintf(content, " @method=%s", method) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/escalate.go b/tools/voluspa/adapters/escalate.go new file mode 100644 index 00000000000..d4e0997d07a --- /dev/null +++ b/tools/voluspa/adapters/escalate.go @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + EscalateAdapterFailoverParameter = "failover" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&EscalateAdapter{}) +} + +type EscalateAdapter struct { +} + +func (p *EscalateAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *EscalateAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("escalate") +} + +func (p *EscalateAdapter) Name() string { + return "escalate" +} + +func (p *EscalateAdapter) SharedLibraryName() string { + return "escalate.so" +} + +func (p *EscalateAdapter) ConfigParameters() []string { + return []string{EscalateAdapterFailoverParameter} +} + +func (p *EscalateAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *EscalateAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(EscalateAdapterFailoverParameter) + if err != nil { + return nil, err + } + + if err = p.validateArgs(alt); err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *EscalateAdapter) validateArgs(alt map[string]interface{}) error { + domain := p.paramToValue(alt["domain"]) + statusCodes := p.paramToValue(alt["status_codes"]) + if len(domain) == 0 || len(statusCodes) == 0 { + return fmt.Errorf("%q and %q are required parameters", "domain", "status_codes") + } + + value := p.paramToValue(alt["host_header"]) + if len(value) > 0 && (value != "alias" && value != "origin") { + return fmt.Errorf("invalid 'host_header' option: %q", value) + } + + return nil +} + +func convertStatusCodes(in interface{}) []string { + switch val := in.(type) { + case []string: + return val + case []interface{}: + var vals []string + for _, v1 := range in.([]interface{}) { + vS := strconv.Itoa(v1.(int)) + vals = append(vals, vS) + } + return vals + } + return nil +} + +func (p *EscalateAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + + { + value := p.paramToValue(alt["host_header"]) + if len(value) > 0 && value == "origin" { + args = append(args, "--pristine") + } + } + + statusCodes := convertStatusCodes(alt["status_codes"]) + domain := p.paramToValue(alt["domain"]) + + args = append(args, fmt.Sprintf("%s:%s", strings.Join(statusCodes, ","), domain)) + + return args, nil +} + +func (p *EscalateAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case bool: + return strconv.FormatBool(val) + + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := strconv.Itoa(v1.(int)) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} diff --git a/tools/voluspa/adapters/escalate_test.go b/tools/voluspa/adapters/escalate_test.go new file mode 100644 index 00000000000..83de0032c07 --- /dev/null +++ b/tools/voluspa/adapters/escalate_test.go @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestEscalateAdapter(t *testing.T) { + var plugin EscalateAdapter + ro := &voluspa.RemapOptions{} + + subOptions := make(map[interface{}]interface{}) + (*ro)[EscalateAdapterFailoverParameter] = subOptions + + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + _, err := plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["domain"] = "" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["domain"] = "domain.example.com" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["status_codes"] = []string{} + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["status_codes"] = []string{"401", "403"} + out, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected := "401,403:domain.example.com" + if out[0] != expected { + t.Fatalf("Error: got %q; expected %q", out[0], expected) + } + + subOptions["host_header"] = "origin" + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected = "--pristine" + if out[0] != expected { + t.Fatalf("Error: got %q; expected %q", out[0], expected) + } + + expected = "401,403:domain.example.com" + if out[1] != expected { + t.Fatalf("Error: got %q; expected %q", out[1], expected) + } +} diff --git a/tools/voluspa/adapters/gzip.go b/tools/voluspa/adapters/gzip.go new file mode 100644 index 00000000000..ee486c552f0 --- /dev/null +++ b/tools/voluspa/adapters/gzip.go @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + GzipAdapterParameterCompressibleContentType = "static_origin_compressible_content_type" + GzipAdapterParameter = "edge_compression" + GzipAdapterParameterEnabled = "enabled" + GzipAdapterParameterAlgorithms = "algorithms" + GzipAdapterParameterRemoveAcceptEncoding = "remove_accept_encoding" + GzipAdapterParameterFlush = "flush" + GzipAdapterParameterMinimumContentLength = "minimum_content_length" +) + +var ( + defaultCompressibleTypes = []string{ + "*font*", + "*javascript", + "*json", + "*ml;*", + "*mpegURL", + "*mpegurl", + "*otf", + "*ttf", + "*type", + "*xml", + "application/eot", + "application/pkix-crl", + "application/x-httpd-cgi", + "application/x-perl", + "image/vnd.microsoft.icon", + "image/x-icon", + "text/*", + } +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&GzipAdapter{}) +} + +var ( + GzipDefaultAlgorithms = []string{"gzip"} +) + +type GzipAdapter struct { +} + +func (p *GzipAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *GzipAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("gzip") +} + +func (p *GzipAdapter) Name() string { + return "gzip" +} + +func (p *GzipAdapter) SharedLibraryName() string { + return "gzip.so" +} + +func (p *GzipAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *GzipAdapter) ConfigParameters() []string { + return []string{GzipAdapterParameter} +} + +func (p *GzipAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + config, err := env.RemapOptions.ValueByNameAsStringMapInterface(GzipAdapterParameter) + if err != nil { + return nil, err + } + + { + value, hasOption := config[GzipAdapterParameterEnabled] + if hasOption { + bvalue, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("invalid 'enabled' value (%s): %t", value, value) + } + + if !bvalue { + return content, nil + } + } + } + + var algorithms []string + avalue, hasOption := config[GzipAdapterParameterAlgorithms] + if hasOption { + sliceValue, ok := avalue.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid value for %q: (%s) %t", GzipAdapterParameterAlgorithms, avalue, avalue) + } + + if len(sliceValue) > 0 { + for _, a := range sliceValue { + value := a.(string) + switch value { + case "gzip": + fallthrough + case "br": + algorithms = append(algorithms, value) + default: + return nil, fmt.Errorf("invalid value for %q: (%s)", GzipAdapterParameterAlgorithms, value) + } + } + } + } else { + algorithms = GzipDefaultAlgorithms + } + sort.Sort(sort.StringSlice(algorithms)) + + var compressibleTypes []string + params, found := config[GzipAdapterParameterCompressibleContentType] + if found { + typesSlice, ok := params.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid value for %q: (%s) %t", GzipAdapterParameterCompressibleContentType, params, params) + } + + for _, a := range typesSlice { + compressibleTypes = append(compressibleTypes, a.(string)) + } + } else { + compressibleTypes = defaultCompressibleTypes + } + + sort.Sort(sort.StringSlice(compressibleTypes)) + + content.WriteString(`enabled true +cache false +`) + + value, hasOption := config[GzipAdapterParameterRemoveAcceptEncoding] + if hasOption { + on, ok := value.(bool) + if ok && on { + content.WriteString("remove-accept-encoding true\n") + } + } + + value, hasOption = config[GzipAdapterParameterFlush] + if hasOption { + on, ok := value.(bool) + if ok && on { + content.WriteString("flush true\n") + } + } + + value, hasOption = config[GzipAdapterParameterMinimumContentLength] + if hasOption { + minLen, ok := value.(int) + if ok && minLen > 0 { + content.WriteString(fmt.Sprintf("minimum-content-length %d\n", minLen)) + } + } + + content.WriteString(fmt.Sprintf("supported-algorithms %s\n", strings.Join(algorithms, ","))) + + for _, param := range compressibleTypes { + content.WriteString(fmt.Sprintf("compressible-content-type %s\n", param)) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/add_header.go b/tools/voluspa/adapters/headerrewrite/add_header.go new file mode 100644 index 00000000000..ec5fcce7f54 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/add_header.go @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + AddHeaderAdapterParameter = "add_header" + AddHeaderAdapterOriginParameter = "add_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AddHeaderAdapter{}) +} + +type AddHeaderAdapter struct { +} + +func (p *AddHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *AddHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("add_header") +} + +func (p *AddHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *AddHeaderAdapter) ConfigParameters() []string { + return []string{AddHeaderAdapterParameter, AddHeaderAdapterOriginParameter} +} + +func (p *AddHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(AddHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("READ_RESPONSE_HDR_HOOK", fmt.Sprintf("add-header %s", header))) + } + + header, _ = env.RemapOptions.ValueByNameAsString(AddHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("add-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go b/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go new file mode 100644 index 00000000000..9c2daed2288 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseHeaderRewriteAdapter{}) +} + +type BaseHeaderRewriteAdapter struct { +} + +func (*BaseHeaderRewriteAdapter) Weight() int { + return 25 +} + +func (p *BaseHeaderRewriteAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseHeaderRewriteAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseHeaderRewriteAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseHeaderRewriteAdapter) Name() string { + return "hdrs" +} + +func (p *BaseHeaderRewriteAdapter) SharedLibraryName() string { + return "header_rewrite.so" +} + +func (p *BaseHeaderRewriteAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseHeaderRewriteAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return &bytes.Buffer{}, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/common.go b/tools/voluspa/adapters/headerrewrite/common.go new file mode 100644 index 00000000000..dd14765f974 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/common.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("header_rewrite") diff --git a/tools/voluspa/adapters/headerrewrite/content_type_forge.go b/tools/voluspa/adapters/headerrewrite/content_type_forge.go new file mode 100644 index 00000000000..b92b1a94f00 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/content_type_forge.go @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + "sort" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ContentTypeForgeAdapterParameter = "content_type_forge" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ContentTypeForgeAdapter{}) +} + +type ContentTypeForgeAdapter struct { +} + +func (p *ContentTypeForgeAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ContentTypeForgeAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("content_type_forge") +} + +func (p *ContentTypeForgeAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ContentTypeForgeAdapter) Name() string { + return "" +} + +func (p *ContentTypeForgeAdapter) ConfigParameters() []string { + return []string{ContentTypeForgeAdapterParameter} +} + +func (p *ContentTypeForgeAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + contentTypes, err := env.RemapOptions.ValueByNameAsStringMapString(ContentTypeForgeAdapterParameter) + if err != nil { + return nil, util.FormatError(ContentTypeForgeAdapterParameter, err) + } + + var keys []string + for k := range contentTypes { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, regex := range keys { + contentType := contentTypes[regex] + content.WriteString(fmt.Sprintf( + `cond %%{SEND_RESPONSE_HDR_HOOK} [AND] + cond %%{PATH} /%s/ + set-header Content-Type %s`, + regex, contentType, + )) + content.WriteString("\n") + } + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/defaultttl.go b/tools/voluspa/adapters/headerrewrite/defaultttl.go new file mode 100644 index 00000000000..6cadc29d2cd --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/defaultttl.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + DefaultTTLAdapterParameter = "default_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DefaultTTLAdapter{}) +} + +type DefaultTTLAdapter struct { +} + +func (p *DefaultTTLAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DefaultTTLAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("default_ttl") +} + +func (p *DefaultTTLAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DefaultTTLAdapter) Name() string { + return "" +} + +func (p *DefaultTTLAdapter) ConfigParameters() []string { + return []string{DefaultTTLAdapterParameter} +} + +func (p *DefaultTTLAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + cacheControl, err := env.RemapOptions.ValueByNameAsString(DefaultTTLAdapterParameter) + if err != nil { + return nil, util.FormatError(DefaultTTLAdapterParameter, err) + } + + // translate a simple duration to a max-age public directive + if maxAge, err := util.DurationToSeconds(cacheControl); err == nil { + cacheControl = fmt.Sprintf("max-age=%s, public", maxAge) + } + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} [AND] +cond %%{HEADER:Cache-Control} ="" [AND] +%s + set-header Cache-Control %q`, + conditionStatusCodeSuccess, cacheControl, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/defaultttl_test.go b/tools/voluspa/adapters/headerrewrite/defaultttl_test.go new file mode 100644 index 00000000000..305f3d6a833 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/defaultttl_test.go @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestDefaultTTL(t *testing.T) { + + plugin := &DefaultTTLAdapter{} + + if plugin.Type() != "default_ttl" { + t.Errorf("Type() returned unexpected %q", plugin.Type()) + } + + if fmt.Sprintf("%v", plugin.ConfigParameters()) != "[default_ttl]" { + t.Errorf("ConfigParameters() returned unexpected %v", plugin.ConfigParameters()) + } + + for i, tt := range []struct { + param interface{} + expected string + err error + }{ + { + param: nil, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: `), + }, + { + param: true, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: true`), + }, + { + param: 42, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: 42`), + }, + { + param: "5d", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=432000, public"`, + }, + { + param: "no-cache, no-store, must-revalidate, stale-while-revalidate=30", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "no-cache, no-store, must-revalidate, stale-while-revalidate=30"`, + }, + } { + + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + t.Run(fmt.Sprintf("%d-%v", i, tt.param), func(t *testing.T) { + + (*ro)[DefaultTTLAdapterParameter] = tt.param + got, err := plugin.Content(env) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tt.err) { + t.Errorf("Content() error expected `%v` got: %v", tt.err, err) + return + } + expected := strings.TrimPrefix(tt.expected, "\n") + if err == nil && got.String() != expected { + t.Errorf("Content() buffer expected:\n%s\ngot:\n%s", expected, got.String()) + } + }) + } +} diff --git a/tools/voluspa/adapters/headerrewrite/dscp.go b/tools/voluspa/adapters/headerrewrite/dscp.go new file mode 100644 index 00000000000..94618995503 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/dscp.go @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + DSCPAdapterParameterPriority = "priority" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DSCPAdapter{}) +} + +type DSCPAdapter struct { +} + +func (p *DSCPAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DSCPAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("dscp") +} + +func (p *DSCPAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DSCPAdapter) ConfigParameters() []string { + return []string{DSCPAdapterParameterPriority} +} + +func (p *DSCPAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *DSCPAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + value, err := env.RemapOptions.ValueByNameAsString(DSCPAdapterParameterPriority) + if err != nil { + return nil, err + } + + // http://www.tucny.com/Home/dscp-tos + var dscpvalue int + switch value { + case "background": + // Queue 1 - Low-Priority Data - CS1 + dscpvalue = 8 + case "foreground": + // Queue 0 - Best-Effort - AF31, AF32, AF33, AF11, AF12, AF13 + // + // AF11 = High-Throughput Data - iTunes Downloads (App Store, Songs, etc.) + dscpvalue = 10 + case "streamingaudio": + // AF31 = Multimedia Streaming - HTTP Live Streaming - Streaming Audio + dscpvalue = 26 + case "streamingvideo": + // AF32 = Multimedia Streaming - HTTP Live Streaming - Streaming Video + dscpvalue = 28 + default: + return nil, util.FormatError(DSCPAdapterParameterPriority, fmt.Errorf("Unhandled priority %q", value)) + } + + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf("set-conn-dscp %d", dscpvalue))) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/forcettl.go b/tools/voluspa/adapters/headerrewrite/forcettl.go new file mode 100644 index 00000000000..c70b91db394 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/forcettl.go @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ForceTTLAdapterParameter = "force_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ForceTTLAdapter{}) +} + +type ForceTTLAdapter struct { +} + +func (p *ForceTTLAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ForceTTLAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("force_ttl") +} + +func (p *ForceTTLAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ForceTTLAdapter) Name() string { + return "" +} + +func (p *ForceTTLAdapter) ConfigParameters() []string { + return []string{ForceTTLAdapterParameter} +} + +func (p *ForceTTLAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + cacheControl, err := env.RemapOptions.ValueByNameAsString(ForceTTLAdapterParameter) + if err != nil { + return nil, util.FormatError(ForceTTLAdapterParameter, err) + } + + // translate a simple duration to a max-age public directive + if maxAge, err := util.DurationToSeconds(cacheControl); err == nil { + cacheControl = fmt.Sprintf("max-age=%s, public", maxAge) + } + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} [AND] +%s + set-header Cache-Control %q`, + conditionStatusCodeSuccess, cacheControl, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/forcettl_test.go b/tools/voluspa/adapters/headerrewrite/forcettl_test.go new file mode 100644 index 00000000000..3ae8bacdf63 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/forcettl_test.go @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestForceTTL(t *testing.T) { + + plugin := &ForceTTLAdapter{} + + if plugin.Type() != "force_ttl" { + t.Errorf("Type() returned unexpected %q", plugin.Type()) + } + + if fmt.Sprintf("%v", plugin.ConfigParameters()) != "[force_ttl]" { + t.Errorf("ConfigParameters() returned unexpected %v", plugin.ConfigParameters()) + } + + for i, tt := range []struct { + param interface{} + expected string + err error + }{ + { + param: nil, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: `), + }, + { + param: true, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: true`), + }, + { + param: 42, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: 42`), + }, + { + param: "5d", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=432000, public"`, + }, + { + param: "no-cache, no-store, must-revalidate, stale-while-revalidate=30", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "no-cache, no-store, must-revalidate, stale-while-revalidate=30"`, + }, + } { + + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + t.Run(fmt.Sprintf("%d-%v", i, tt.param), func(t *testing.T) { + + (*ro)[ForceTTLAdapterParameter] = tt.param + got, err := plugin.Content(env) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tt.err) { + t.Errorf("Content() error expected `%v` got: %v", tt.err, err) + return + } + expected := strings.TrimPrefix(tt.expected, "\n") + if err == nil && got.String() != expected { + t.Errorf("Content() buffer expected:\n%s\ngot:\n%s", expected, got.String()) + } + }) + } +} diff --git a/tools/voluspa/adapters/headerrewrite/header_rewrite.go b/tools/voluspa/adapters/headerrewrite/header_rewrite.go new file mode 100644 index 00000000000..8e77d83540d --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/header_rewrite.go @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + HeaderRewriteAdapterParameter = "header_rewrite" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformHeaderRewriteAdapter{}) +} + +type FreeformHeaderRewriteAdapter struct { +} + +func (*FreeformHeaderRewriteAdapter) Weight() int { + return 25 +} + +func (p *FreeformHeaderRewriteAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformHeaderRewriteAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_header_rewrite") +} + +func (p *FreeformHeaderRewriteAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformHeaderRewriteAdapter) Name() string { + return "" +} + +func (p *FreeformHeaderRewriteAdapter) ConfigParameters() []string { + return []string{HeaderRewriteAdapterParameter} +} + +func (p *FreeformHeaderRewriteAdapter) SharedLibraryName() string { + return "header_rewrite.so" +} + +func (p *FreeformHeaderRewriteAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + val, err := env.RemapOptions.ValueByNameAsString(HeaderRewriteAdapterParameter) + if err != nil { + return nil, util.FormatError(HeaderRewriteAdapterParameter, err) + } + + // Do nothing with the config, just pass through + return bytes.NewBufferString(fmt.Sprintf("%s\n", val)), nil +} diff --git a/tools/voluspa/adapters/headerrewrite/log_cookie_header.go b/tools/voluspa/adapters/headerrewrite/log_cookie_header.go new file mode 100644 index 00000000000..14043051c91 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/log_cookie_header.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + "sort" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + logCookieDirective = "log_cookie" + logHeaderDirective = "log_header" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&LogAdapter{}) +} + +type LogAdapter struct { +} + +func (p *LogAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *LogAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("log_field") +} + +func (p *LogAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *LogAdapter) ConfigParameters() []string { + return []string{logCookieDirective, logHeaderDirective} +} + +func (p *LogAdapter) logCookieHeader(logFields map[string]string, content *bytes.Buffer, parameterName string, index int) (*bytes.Buffer, int) { + var logFieldNames []string + + for k := range logFields { + logFieldNames = append(logFieldNames, k) + } + sort.Strings(logFieldNames) + + content.WriteString(fmt.Sprintf( + `cond %%{REMAP_PSEUDO_HOOK}`, + )) + content.WriteString("\n") + + for _, logFieldName := range logFieldNames { + logFieldValue := logFields[logFieldName] + if parameterName == "log_cookie" { + content.WriteString(fmt.Sprintf( + ` set-header Bazinga%d %%{COOKIE:%s}`, + index, logFieldValue, + )) + index++ + } else if parameterName == "log_header" { + content.WriteString(fmt.Sprintf( + ` set-header Bazinga%d %%{HEADER:%s}`, + index, logFieldValue, + )) + index++ + } + content.WriteString("\n") + } + return content, index +} + +func (p *LogAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + index := 1 + + // Log Cookies + logFields, _ := env.RemapOptions.ValueByNameAsStringMapString(logCookieDirective) + if len(logFields) > 0 { + content, index = p.logCookieHeader(logFields, content, logCookieDirective, index) + } + + // Log headers + logFields, _ = env.RemapOptions.ValueByNameAsStringMapString(logHeaderDirective) + if len(logFields) > 0 { + content, _ = p.logCookieHeader(logFields, content, logHeaderDirective, index) + } + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/log_type.go b/tools/voluspa/adapters/headerrewrite/log_type.go new file mode 100644 index 00000000000..caa085e64af --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/log_type.go @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + LogTypeAdapterParameter = "log_type" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&LogTypeAdapter{}) +} + +type LogTypeAdapter struct { +} + +func (p *LogTypeAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *LogTypeAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("log_type") +} + +func (p *LogTypeAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *LogTypeAdapter) Name() string { + return "" +} + +func (p *LogTypeAdapter) ConfigParameters() []string { + return []string{LogTypeAdapterParameter} +} + +func (p *LogTypeAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + val, err := env.RemapOptions.ValueByNameAsString(LogTypeAdapterParameter) + if err != nil { + return nil, err + } + + if val == "public" { + return nil, nil + } + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", `set-header @cdnlog "private"`)) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go b/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go new file mode 100644 index 00000000000..9d2a36b48a0 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ProxyCacheControlAdapterParameter = "proxy_cache_control" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ProxyCacheControlAdapter{}) +} + +type ProxyCacheControlAdapter struct { +} + +func (p *ProxyCacheControlAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ProxyCacheControlAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("proxy_cache_control") +} + +func (p *ProxyCacheControlAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ProxyCacheControlAdapter) ConfigParameters() []string { + return []string{ProxyCacheControlAdapterParameter} +} + +func (p *ProxyCacheControlAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, err := env.RemapOptions.ValueByNameAsString(ProxyCacheControlAdapterParameter) + if err != nil { + return nil, util.FormatError(ProxyCacheControlAdapterParameter, err) + } + + content := &bytes.Buffer{} + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} + set-header @Original-Cache-Control %%{HEADER:Cache-Control} + set-header Cache-Control %%{HEADER:%s} + +cond %%{SEND_RESPONSE_HDR_HOOK} + set-header Cache-Control %%{HEADER:@Original-Cache-Control} + rm-header Proxy-Cache-Control`, + header, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/receipt.go b/tools/voluspa/adapters/headerrewrite/receipt.go new file mode 100644 index 00000000000..e077132bff4 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/receipt.go @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ReceiptsAdapterParameter = "receipts" + ReceiptsAdapterParameterEnabled = "enabled" + ReceiptsAdapterParameterName = "name" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ReceiptAdapter{}) +} + +type ReceiptAdapter struct { +} + +func (*ReceiptAdapter) Weight() int { + return 25 +} + +func (p *ReceiptAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ReceiptAdapter) Type() voluspa.AdapterType { + return voluspa.Receipt +} + +func (p *ReceiptAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ReceiptAdapter) Name() string { + return "" +} + +func (p *ReceiptAdapter) ConfigParameters() []string { + return []string{ReceiptsAdapterParameter} +} + +func (p *ReceiptAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + receiptsConfig, err := env.RemapOptions.ValueByNameAsStringMapInterface(ReceiptsAdapterParameter) + if err != nil { + return nil, err + } + + value, hasOption := receiptsConfig[ReceiptsAdapterParameterEnabled] + if hasOption { + bvalue, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("invalid 'enabled' value (%s)", value) + } + + if !bvalue { + return content, nil + } + } + + receiptName := env.Property + value, hasOption = receiptsConfig[ReceiptsAdapterParameterName] + if hasOption { + strvalue, ok := value.(string) + if !ok { + return nil, fmt.Errorf("invalid 'name' value (%s)", value) + } + receiptName = strvalue + } + + receipt := fmt.Sprintf("%s{{hosttype}}", receiptName) + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf(`set-header @ReceiptService "%s"`, receipt))) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/remove_header.go b/tools/voluspa/adapters/headerrewrite/remove_header.go new file mode 100644 index 00000000000..4843137fd16 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/remove_header.go @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + RemoveHeaderAdapterParameter = "remove_header" + RemoveHeaderAdapterOriginParameter = "remove_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RemoveHeaderAdapter{}) +} + +type RemoveHeaderAdapter struct { +} + +func (p *RemoveHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *RemoveHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("remove_header") +} + +func (p *RemoveHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *RemoveHeaderAdapter) ConfigParameters() []string { + return []string{RemoveHeaderAdapterParameter, RemoveHeaderAdapterOriginParameter} +} + +func (p *RemoveHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(RemoveHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf("rm-header %s", header))) + } + header, _ = env.RemapOptions.ValueByNameAsString(RemoveHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("rm-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/rewrite_utils.go b/tools/voluspa/adapters/headerrewrite/rewrite_utils.go new file mode 100644 index 00000000000..6634a62477b --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/rewrite_utils.go @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +const conditionStatusCodeSuccess = `cond %{STATUS} >199 [AND] +cond %{STATUS} <400` diff --git a/tools/voluspa/adapters/headerrewrite/set_header.go b/tools/voluspa/adapters/headerrewrite/set_header.go new file mode 100644 index 00000000000..011036773e0 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/set_header.go @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + SetHeaderAdapterParameter = "set_header" + SetHeaderAdapterOriginParameter = "set_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&SetHeaderAdapter{}) +} + +type SetHeaderAdapter struct { +} + +func (p *SetHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *SetHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("set_header") +} + +func (p *SetHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *SetHeaderAdapter) ConfigParameters() []string { + return []string{SetHeaderAdapterParameter, SetHeaderAdapterOriginParameter} +} + +func (p *SetHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(SetHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("READ_RESPONSE_HDR_HOOK", fmt.Sprintf("set-header %s", header))) + } + + header, _ = env.RemapOptions.ValueByNameAsString(SetHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("set-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/init.go b/tools/voluspa/adapters/init.go new file mode 100644 index 00000000000..c83ad48814d --- /dev/null +++ b/tools/voluspa/adapters/init.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/confremap" // for convenience +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/headerrewrite" // for convenience +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/lua" // for convenience diff --git a/tools/voluspa/adapters/lua/base_lua.go b/tools/voluspa/adapters/lua/base_lua.go new file mode 100644 index 00000000000..19fa496a63f --- /dev/null +++ b/tools/voluspa/adapters/lua/base_lua.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseLuaAdapter{}) +} + +type BaseLuaAdapter struct { +} + +func (*BaseLuaAdapter) Weight() int { + return 99 +} + +func (p *BaseLuaAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseLuaAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseLuaAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseLuaAdapter) Name() string { + return luaScriptName +} + +func (p *BaseLuaAdapter) SharedLibraryName() string { + return "tslua.so" +} + +func (p *BaseLuaAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseLuaAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return &bytes.Buffer{}, nil +} diff --git a/tools/voluspa/adapters/lua/common.go b/tools/voluspa/adapters/lua/common.go new file mode 100644 index 00000000000..470aa3f67e4 --- /dev/null +++ b/tools/voluspa/adapters/lua/common.go @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("lua") +var luaScriptName = "youdidntsetluascriptname.lua" diff --git a/tools/voluspa/adapters/lua/echo_cors.go b/tools/voluspa/adapters/lua/echo_cors.go new file mode 100644 index 00000000000..b5e01e276f7 --- /dev/null +++ b/tools/voluspa/adapters/lua/echo_cors.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + EchoCORSAdapterParameter = "echo_cors" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&EchoCORSAdapter{}) + + // Every standard script in adapters/lua has to set this + luaScriptName = "echo_cors.lua" +} + +type EchoCORSAdapter struct { +} + +func (p *EchoCORSAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *EchoCORSAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("echo_cors") +} + +func (p *EchoCORSAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *EchoCORSAdapter) ConfigParameters() []string { + return []string{EchoCORSAdapterParameter} +} + +func (p *EchoCORSAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + headerName, err := env.RemapOptions.ValueByNameAsString(EchoCORSAdapterParameter) + if err != nil { + return nil, err + } + + content.WriteString(fmt.Sprintf( + ` +function send_response() + ts.client_response.header['Access-Control-Allow-Origin'] = ts.ctx['origin'] + ts.client_response.header['Access-Control-Allow-Authentication'] = 'True' + return 0 +end + +function do_remap() + ts.ctx['origin'] = ts.client_request.header.%s + ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) + return 0 +end + `, headerName)) + + return content, nil +} diff --git a/tools/voluspa/adapters/lua/lua.go b/tools/voluspa/adapters/lua/lua.go new file mode 100644 index 00000000000..1fcf35c0254 --- /dev/null +++ b/tools/voluspa/adapters/lua/lua.go @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + LuaAdapterParameter = "lua" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformLuaAdapter{}) + luaScriptName = "inline.lua" +} + +type FreeformLuaAdapter struct { +} + +func (*FreeformLuaAdapter) Weight() int { + return 25 +} + +func (p *FreeformLuaAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformLuaAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_lua") +} + +func (p *FreeformLuaAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformLuaAdapter) Name() string { + return "" +} + +func (p *FreeformLuaAdapter) ConfigParameters() []string { + return []string{LuaAdapterParameter} +} + +func (p *FreeformLuaAdapter) SharedLibraryName() string { + return "tslua.so" +} + +func (p *FreeformLuaAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + val, err := env.RemapOptions.ValueByNameAsString(LuaAdapterParameter) + if err != nil { + return nil, util.FormatError(LuaAdapterParameter, err) + } + + // Do nothing with the config, just pass through + return bytes.NewBufferString(fmt.Sprintf("%s\n", val)), nil +} diff --git a/tools/voluspa/adapters/null.go b/tools/voluspa/adapters/null.go new file mode 100644 index 00000000000..898de95cf38 --- /dev/null +++ b/tools/voluspa/adapters/null.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&NullAdapter{}) +} + +type NullAdapter struct { +} + +func (p *NullAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *NullAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("null") +} + +func (p *NullAdapter) Name() string { + return "null" +} + +func (p *NullAdapter) ConfigParameters() []string { + return []string{"null"} +} + +func (p *NullAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return nil, nil +} diff --git a/tools/voluspa/adapters/redirect.go b/tools/voluspa/adapters/redirect.go new file mode 100644 index 00000000000..55b0c089e54 --- /dev/null +++ b/tools/voluspa/adapters/redirect.go @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RedirectAdapter{}) +} + +type RedirectAdapter struct { +} + +func (*RedirectAdapter) Weight() int { + return 30 +} + +func (p *RedirectAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *RedirectAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("redirect") +} + +func (p *RedirectAdapter) Name() string { + return "redirect" +} + +func (p *RedirectAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *RedirectAdapter) ConfigParameters() []string { + return []string{"redirect"} +} + +func (p *RedirectAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + urls, err := env.RemapOptions.ValueByNameAsStringMapInterface(p.Name()) + if err != nil { + return nil, err + } + + url, ok := urls["url"].(string) + if !ok { + return nil, fmt.Errorf("Expecting url as string and Redirect url is mandatory") + } + + // default status code to 302 if not provided + httpCode, ok := urls["http_code"].(int) + if !ok { + httpCode = 302 + } + + // configurable source regex pattern defaults to (.*) + src, ok := urls["src"] + if !ok { + src = "(.*)" + } + + content.WriteString(fmt.Sprintf("%s %s @status=%d", src, url, httpCode)) + + return content, nil +} diff --git a/tools/voluspa/adapters/regex_remap.go b/tools/voluspa/adapters/regex_remap.go new file mode 100644 index 00000000000..a4dbbb37310 --- /dev/null +++ b/tools/voluspa/adapters/regex_remap.go @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + RegexRemapAdapterParameter = "regex_remap" + regexRemapSharedLibraryName = "regex_remap.so" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RegexRemapAdapter{}) +} + +type RegexRemapAdapter struct { +} + +func (*RegexRemapAdapter) Weight() int { + return 15 +} + +func (p *RegexRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *RegexRemapAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("regex_remap") +} + +func (p *RegexRemapAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *RegexRemapAdapter) Name() string { + return "regex" +} + +func (p *RegexRemapAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *RegexRemapAdapter) ConfigParameters() []string { + return []string{RegexRemapAdapterParameter} +} + +func (p *RegexRemapAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + regex, err := env.RemapOptions.ValueByNameAsString(RegexRemapAdapterParameter) + if err != nil { + return nil, util.FormatError(RegexRemapAdapterParameter, err) + } + + return bytes.NewBufferString(regex), nil +} diff --git a/tools/voluspa/adapters/s3_auth.go b/tools/voluspa/adapters/s3_auth.go new file mode 100644 index 00000000000..b9776164c0c --- /dev/null +++ b/tools/voluspa/adapters/s3_auth.go @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strconv" + + "fmt" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + S3AuthAdapterParameter = "s3" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&S3AuthAdapter{}) +} + +type S3AuthAdapter struct { +} + +func (*S3AuthAdapter) Weight() int { + return 16 +} + +func (p *S3AuthAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *S3AuthAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("s3_auth") +} + +func (p *S3AuthAdapter) SharedLibraryName() string { + return "s3_auth.so" +} + +func (p *S3AuthAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *S3AuthAdapter) ConfigParameters() []string { + return []string{S3AuthAdapterParameter} +} + +func (p *S3AuthAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + var alt map[string]interface{} + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(S3AuthAdapterParameter) + if err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *S3AuthAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["path"], "config", args) + args = p.maybeAddArgs(alt["auth"], "config", args) + args = p.maybeAddArgs(alt["virtual_host"], "virtual_host", args) + args = p.maybeAddArgs(alt["version"], "version", args) + args = p.maybeAddArgs(alt["v4_include_headers"], "v4-include-headers", args) + args = p.maybeAddArgs(alt["v4_exclude_headers"], "v4-exclude-headers", args) + args = p.maybeAddArgs(alt["region_map"], "v4-region-map", args) + + // if this is v4 and there is no v4_exclude_headers defined, add a @pparam=--v4-exclude-headers=x-forwarded-for,forwarded,via,authorization + _, excludeHeadersDefined := alt["v4_exclude_headers"] + if alt["version"] == 4 && !excludeHeadersDefined { + args = p.maybeAddArgs("x-forwarded-for,forwarded,via,authorization", "v4-exclude-headers", args) + } + + return args, nil +} + +func (p *S3AuthAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := v1.(string) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} + +func (p *S3AuthAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := p.paramToValue(option) + switch param { + case "virtual_host": + if value == "true" { + return append(args, "--virtual_host") + } + case "config": + if len(value) > 0 { + return append(args, []string{"--config", value}...) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/strip_query.go b/tools/voluspa/adapters/strip_query.go new file mode 100644 index 00000000000..57c9a4d41b9 --- /dev/null +++ b/tools/voluspa/adapters/strip_query.go @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&StripQueryAdapter{}) +} + +type StripQueryAdapter struct { +} + +func (p *StripQueryAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *StripQueryAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("strip_query") +} + +func (p *StripQueryAdapter) Name() string { + return "strip_query" +} + +func (p *StripQueryAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *StripQueryAdapter) ConfigParameters() []string { + return []string{"strip_query"} +} + +func (p *StripQueryAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return bytes.NewBufferString(". $s://$t/$P\n"), nil +} diff --git a/tools/voluspa/adapters/util/ttl_utils.go b/tools/voluspa/adapters/util/ttl_utils.go new file mode 100644 index 00000000000..d18f0a83aa8 --- /dev/null +++ b/tools/voluspa/adapters/util/ttl_utils.go @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import ( + "fmt" + "regexp" + "strconv" + "time" +) + +var durationRE = regexp.MustCompile(`^(\d+)([a-z])$`) + +// DurationToSeconds takes a duration string and returns number of seconds +// required to be a single time unit (eg 7d or 24h or 3s) +// complex duration not supported(yet) (eg 7d24h3s) +func DurationToSeconds(value string) (string, error) { + if value == "0" { + return value, nil + } + + if !durationRE.MatchString(value) { + return "", fmt.Errorf("Bad duration \"%s\"", value) + } + + matches := durationRE.FindStringSubmatch(value) + count, err := strconv.ParseUint(matches[1], 10, 64) + if err != nil { + return "", fmt.Errorf("Bad duration \"%s\"", value) + } + + switch matches[2] { + case "d": + return strconv.FormatUint(count*60*60*24, 10), nil + case "w": + return strconv.FormatUint(count*60*60*24*7, 10), nil + + default: + duration, err := time.ParseDuration(value) + if err == nil { + return fmt.Sprintf("%d", int(duration.Seconds())), nil + } + return "", fmt.Errorf("Unhandled spec '%s' for '%s'. err=%s", matches[2], value, err) + } +} diff --git a/tools/voluspa/adapters/util/ttl_utils_test.go b/tools/voluspa/adapters/util/ttl_utils_test.go new file mode 100644 index 00000000000..d8baf8f178e --- /dev/null +++ b/tools/voluspa/adapters/util/ttl_utils_test.go @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import "testing" + +func TestDurationToSeconds(t *testing.T) { + in := "30d" + expected := "2592000" + val, err := DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "0" + expected = "0" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "7w" + expected = "4233600" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "1h" + expected = "3600" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "2h3s" + expected = "7203" + val, err = DurationToSeconds(in) + + if err == nil || val == expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "7r" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } + + in = "5" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } + + in = "7w2d3h" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } +} diff --git a/tools/voluspa/adapters/util/util.go b/tools/voluspa/adapters/util/util.go new file mode 100644 index 00000000000..20a21c81459 --- /dev/null +++ b/tools/voluspa/adapters/util/util.go @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import ( + "fmt" + "strconv" +) + +func FormatSimpleHeaderRewrite(condition, action string) string { + return fmt.Sprintf("cond %%{%s}\n %s", condition, action) +} + +func FormatError(option string, err error) error { + return fmt.Errorf("option %q: %s", option, err.Error()) +} + +func ParamToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + default: + return "" + } +} diff --git a/tools/voluspa/adapters/video_background_fetch.go b/tools/voluspa/adapters/video_background_fetch.go new file mode 100644 index 00000000000..23ccf69a2eb --- /dev/null +++ b/tools/voluspa/adapters/video_background_fetch.go @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&VideoBackgroundFetchAdapter{}) +} + +const ( + VideoBackgroundFetchDirective = "video_background_fetch" + + VideoBackgroundFetchAdapterFrontend = "frontend" + VideoBackgroundFetchAdapterBackend = "backend" + + VideoBackgroundFetchAdapterFetchCount = "fetch_count" + VideoBackgroundFetchAdapterFetchPolicy = "fetch_policy" + VideoBackgroundFetchAdapterFetchMax = "fetch_max" + VideoBackgroundFetchAdapterAPIHeader = "api_header" + VideoBackgroundFetchAdapterFetchPathPattern = "fetch_path_pattern" + VideoBackgroundFetchAdapterReplaceHost = "replace_host" + VideoBackgroundFetchAdapterNameSpace = "name_space" + VideoBackgroundFetchAdapterMetricsPrefix = "metrics_prefix" + VideoBackgroundFetchAdapterExactMatch = "exact_match" + VideoBackgroundFetchAdapterLogName = "log_name" +) + +type VideoBackgroundFetchAdapter struct { +} + +func (p *VideoBackgroundFetchAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *VideoBackgroundFetchAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("video_background_fetch") +} + +func (p *VideoBackgroundFetchAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *VideoBackgroundFetchAdapter) ConfigParameters() []string { + return []string{VideoBackgroundFetchDirective} +} + +func (p *VideoBackgroundFetchAdapter) Name() string { + return "volcano" +} + +func (p *VideoBackgroundFetchAdapter) SharedLibraryName() string { + return "volcano.so" +} + +func (p *VideoBackgroundFetchAdapter) PParams(env *voluspa.ConfigEnvironment) (voluspa.RolePParams, error) { + + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(VideoBackgroundFetchDirective) + if err != nil { + return nil, util.FormatError(VideoBackgroundFetchDirective, err) + } + + // process default args + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + results := voluspa.RolePParams{} + results[voluspa.DefaultRole] = args + + for _, v := range []string{VideoBackgroundFetchAdapterFrontend, VideoBackgroundFetchAdapterBackend} { + sub, found := alt[v] + if !found { + continue + } + + submap, ok := sub.(map[interface{}]interface{}) + if !ok { + continue + } + + convertedMap, err := convertMap(submap) + if err != nil { + return nil, err + } + + args, err := p.processArgs(convertedMap) + if err != nil { + return nil, err + } + + args = append(args, fmt.Sprintf("--front=%t", v == VideoBackgroundFetchAdapterFrontend)) + + // eg roles_video_fetch_backend or roles_video_fetch_frontend + role := fmt.Sprintf("roles_video_fetch_%s", v) + results[role] = args + } + + return results, nil +} + +// convertMap attempts to create a usable submap from the very generic map returned by RemapOptions +func convertMap(in map[interface{}]interface{}) (map[string]interface{}, error) { + results := make(map[string]interface{}) + for k, v := range in { + ck, ok := k.(string) + if !ok { + return nil, fmt.Errorf("key %+q is not string", k) + } + results[ck] = v + } + return results, nil +} + +func (p *VideoBackgroundFetchAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchPolicy], "fetch-policy", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchCount], "fetch-count", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchMax], "fetch-max", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterAPIHeader], "api-header", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchPathPattern], "fetch-path-pattern", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterReplaceHost], "replace-host", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterNameSpace], "name-space", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterMetricsPrefix], "metrics-prefix", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterExactMatch], "exact-match", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterLogName], "log-name", args) + + // from here on out, it's parsing fetch_path_pattern + if _, found := alt[VideoBackgroundFetchAdapterFetchPathPattern]; !found { + return args, nil + } + + var fetchPatterns []string + switch val := alt[VideoBackgroundFetchAdapterFetchPathPattern].(type) { + case []interface{}: + for _, ipattern := range val { + pattern, ok := ipattern.(string) + if !ok { + return nil, util.FormatError(VideoBackgroundFetchAdapterFetchPathPattern, fmt.Errorf("Expected a string")) + } + fetchPatterns = append(fetchPatterns, pattern) + } + case []string: + fetchPatterns = val + default: + return nil, util.FormatError(VideoBackgroundFetchAdapterFetchPathPattern, fmt.Errorf("Expected an array of strings")) + } + + for _, pattern := range fetchPatterns { + args = p.maybeAddArgs(pattern, "fetch-path-pattern", args) + } + + return args, nil +} + +func isValidFetchPolicy(policy string) bool { + return policy == "simple" || strings.Contains(policy, "lru") +} + +func (p *VideoBackgroundFetchAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := util.ParamToValue(option) + switch param { + case "fetch-policy": + if isValidFetchPolicy(value) { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/video_background_fetch_test.go b/tools/voluspa/adapters/video_background_fetch_test.go new file mode 100644 index 00000000000..b9ecb68b04b --- /dev/null +++ b/tools/voluspa/adapters/video_background_fetch_test.go @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "sort" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestVideoBackgroundFetch(t *testing.T) { + var plugin VideoBackgroundFetchAdapter + + alt := make(map[interface{}]interface{}) + ro := voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{ + Property: "testVideoBackgroundFetch", + RemapOptions: ro, + } + + alt[VideoBackgroundFetchAdapterFetchCount] = "100" + alt[VideoBackgroundFetchAdapterFetchMax] = "1000" + alt[VideoBackgroundFetchAdapterFetchPolicy] = "simple" + alt[VideoBackgroundFetchAdapterAPIHeader] = "header1" + alt[VideoBackgroundFetchAdapterFetchPathPattern] = []string{"(.*)"} + alt[VideoBackgroundFetchAdapterReplaceHost] = "host1" + alt[VideoBackgroundFetchAdapterNameSpace] = "ns1" + alt[VideoBackgroundFetchAdapterMetricsPrefix] = "prefix1" + alt[VideoBackgroundFetchAdapterExactMatch] = "true" + alt[VideoBackgroundFetchAdapterLogName] = "log1" + + ro["video_background_fetch"] = alt + + pparams, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Failed to generate output for plugin. error=%v", err) + } + + out := pparams[voluspa.DefaultRole] + sort.Sort(sort.StringSlice(out)) + + if len(out) == 0 { + t.Fatalf("Failed to generate output for plugin. no output.") + } + + if expected := "--api-header=header1"; out[0] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[0], expected) + } + + if expected := "--exact-match=true"; out[1] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[1], expected) + } + + if expected := "--fetch-count=100"; out[2] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[2], expected) + } + + if expected := "--fetch-max=1000"; out[3] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[3], expected) + } + + if expected := "--fetch-path-pattern=(.*)"; out[4] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[4], expected) + } + + if expected := "--fetch-policy=simple"; out[5] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[5], expected) + } + + if expected := "--log-name=log1"; out[6] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[6], expected) + } + + if expected := "--metrics-prefix=prefix1"; out[7] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[7], expected) + } + + if expected := "--name-space=ns1"; out[8] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[8], expected) + } + + if expected := "--replace-host=host1"; out[9] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[9], expected) + } +} diff --git a/tools/voluspa/attributes.go b/tools/voluspa/attributes.go new file mode 100644 index 00000000000..826fec24258 --- /dev/null +++ b/tools/voluspa/attributes.go @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "sort" + +// GetCDNs returns a list of CDNs in current config set +func (v *Voluspa) GetCDNs() []string { + cdns := make(map[string]interface{}) + for _, parsedConfig := range v.parsedConfigs { + for cdn := range parsedConfig.cdn { + cdns[cdn] = nil + } + } + + var all []string + for k := range cdns { + all = append(all, k) + } + sort.Strings(all) + + return all +} + +// GetRoles returns a list of Roles in current config set +func (v *Voluspa) GetRoles() []string { + roles := make(map[string]interface{}) + for _, parsedConfig := range v.parsedConfigs { + roles[parsedConfig.role] = nil + } + + var all []string + for k := range roles { + all = append(all, k) + } + sort.Strings(all) + + return all +} diff --git a/tools/voluspa/cdncheck.go b/tools/voluspa/cdncheck.go new file mode 100644 index 00000000000..2501d03b35f --- /dev/null +++ b/tools/voluspa/cdncheck.go @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" +) + +type CDNTest struct { + Type CheckType `json:"type,omitempty" yaml:"type,omitempty"` + DomainNames []string `json:"domain_names,omitempty" yaml:"domain_names,omitempty"` + URLs []string `json:"urls,omitempty" yaml:"urls,omitempty"` + HostTypes []HostType `json:"host_types,omitempty" yaml:"host_types,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Role string `json:"role,omitempty" yaml:"role,omitempty"` + Range string `json:"range,omitempty" yaml:"range,omitempty"` + Headers map[string]string `json:"headers,omitempty" yaml:"headers,omitempty"` + PurgeBefore bool `json:"purge_before,omitempty" yaml:"purge_before,omitempty"` + Insecure bool `json:"insecure" yaml:"insecure"` + Success *CDNCheckSuccessCriteria `json:"success,omitempty" yaml:"success,omitempty"` + Ciphers []string `json:"ciphers,omitempty" yaml:"ciphers,omitempty"` +} + +type CDNCheckSuccessCriteria struct { + StatusCode int `json:"status_code" yaml:"status_code"` + Headers map[string]string `json:"headers,omitempty" yaml:"headers,omitempty"` + SerialNumbers []string `json:"serial_numbers,omitempty" yaml:"serial_numbers,omitempty"` +} + +type HostType string + +const ( + UnknownHostType HostType = "unknown" + ParentHost HostType = Parent + ChildHost HostType = Child +) + +type CheckType string + +const ( + UnknownCheckType CheckType = "unknown" + HTTPCheckType CheckType = "http" + SSLCheckType CheckType = "ssl" +) + +func (h *HostType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var in string + if err := unmarshal(&in); err != nil { + return err + } + + if len(in) == 0 { + return fmt.Errorf("empty host type") + } + + switch in { + case Parent, Child: + *h = HostType(in) + default: + return fmt.Errorf("unknown host type '%s'", in) + } + + return nil +} diff --git a/tools/voluspa/cmd/voluspa/main.go b/tools/voluspa/cmd/voluspa/main.go new file mode 100644 index 00000000000..ad047e38966 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/main.go @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + _ "github.com/apache/trafficserver/tools/voluspa/adapters" +) + +var ( + configDest = flag.String("dest", "out", "destination directory for written out configs") + installLocation = flag.String("install", voluspa.DefaultTrafficserverConfigurationDir, "trafficserver configuration location") + defaultsLocation = flag.String("defaults", "", "location of default configurations") + schemaLocation = flag.String("schema-location", "", "location of schema (defaults to cwd)") + disableAdapters = flag.String("disable-adapters", "", "a comma separated list of adapters to disable") + + cdnList = flag.String("cdns", "", "a comma separated list of CDNs") + roleList = flag.String("roles", "", "a comma separated list of roles") + showVersion = flag.Bool("version", false, "Show version") + verbose = flag.Bool("verbose", false, "Verbose output") + skipSchemaValidation = flag.Bool("skip-schema-validation", false, "skip schema validation") + treatRolesAsCDN = flag.Bool("promote-roles-to-cdn", false, "promote roles to CDN") + ramDiskRole = flag.String("ramdisk-role", "roles_trafficserver_ramdisk", "The role that indicates this host has a ramdisk volume") + + skipValidation = flag.Bool("skip-validation", false, "skip validation") + validationOnly = flag.Bool("validate-only", false, "stop after validation") + strictMode = flag.Bool("strict", false, "enable strict validation") + dumpSchemaVersion = flag.Int("dump-schema", -1, "dump this version of the schema (0 = highest) to STDOUT and exit 0") +) + +func init() { + log.SetFlags(0) + + flag.Parse() +} + +var exit = func(code int) { + os.Exit(code) +} + +func main() { + if *showVersion { + fmt.Println("voluspa", voluspa.Version) + exit(0) + } + + v, err := voluspa.NewVoluspaWithOptions(&voluspa.Options{ + Verbose: *verbose, + Destination: *configDest, + SkipSchemaValidation: *skipSchemaValidation, + PromoteRolesToCDN: *treatRolesAsCDN, + DefaultsLocation: *defaultsLocation, + SchemaLocation: *schemaLocation, + RamDiskRole: *ramDiskRole, + }, *installLocation) + if err != nil { + log.Printf("%s", err) + exit(1) + } + + if *dumpSchemaVersion >= 0 { + schema, err := v.SchemaDefinition(*dumpSchemaVersion) + if err != nil { + log.Printf("%s", err) + exit(1) + } + fmt.Fprint(os.Stdout, string(schema)) + exit(0) + } + + if len(*disableAdapters) > 0 { + adapters := strings.Split(*disableAdapters, ",") + for _, adapter := range adapters { + if err = voluspa.AdaptersRegistry.RemoveAdapterByType(adapter); err != nil { + log.Fatalf("Could not disable adapter %q: %s", adapter, err) + } + } + } + + if len(flag.Args()) == 0 { + log.Printf("Must specify at least 1 config file") + exit(1) + } + + filenames := flag.Args() + sort.Strings(filenames) + + for _, filename := range filenames { + err = v.AddConfig(filename) + if err != nil { + log.Printf("%s", err) + exit(1) + } + } + + if !*skipValidation { + if err = v.Validate(*strictMode); err == nil { + goto validationDone + } + + fmt.Fprintf(os.Stderr, "voluspa: errors validating configs:\n") + + errs, ok := err.(voluspa.Errors) + if !ok { + fmt.Fprintln(os.Stderr, err) + exit(1) + } + + // sort and unique-ify the set of errors + // like errs.Error() but also uniques and has different format + seen := make(map[string]interface{}) + var errOut []string + for _, e := range errs { + if _, found := seen[e.Error()]; found { + continue + } + errOut = append(errOut, fmt.Sprintf(" - %s", e.Error())) + seen[e.Error()] = nil + } + + sort.Strings(errOut) + + for _, str := range errOut { + fmt.Fprintln(os.Stderr, str) + } + exit(1) + } + +validationDone: + if *validationOnly { + exit(0) + } + + var cdns []string + if len(*cdnList) > 0 { + cdns = strings.Split(*cdnList, ",") + } + + var roles []string + if len(*roleList) > 0 { + roles = strings.Split(*roleList, ",") + } + + if err = v.WriteAllFiles(cdns, roles); err != nil { + log.Printf("%s", err) + exit(1) + } +} diff --git a/tools/voluspa/cmd/voluspa/main_test.go b/tools/voluspa/cmd/voluspa/main_test.go new file mode 100644 index 00000000000..7093562c366 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/main_test.go @@ -0,0 +1,33 @@ +// +build testrunmain + +package main + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "testing" +) + +func TestRunMain(t *testing.T) { + exit = func(code int) { + // ignore exit code + } + main() +} diff --git a/tools/voluspa/cmd/voluspa/no_test.go b/tools/voluspa/cmd/voluspa/no_test.go new file mode 100644 index 00000000000..822b16bcbd4 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/no_test.go @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main diff --git a/tools/voluspa/config_generator.go b/tools/voluspa/config_generator.go new file mode 100644 index 00000000000..18c62c1d6eb --- /dev/null +++ b/tools/voluspa/config_generator.go @@ -0,0 +1,567 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +// NewCustomerConfig creates a new CustomerConfig from a PropertyConfig +func NewCustomerConfig(p *PropertyConfig, configFilename string, options Options) (*CustomerConfig, error) { + config, err := newCustomerConfig(p) + if err != nil { + return nil, err + } + + if options.PromoteRolesToCDN && isPromotableRole(p.Role) { + parts := strings.SplitN(p.Role, "_", 2) + if len(parts) == 2 { + // If the only CDN is DefaultCDN, purge and use the role + if len(p.CDN) == 1 && p.CDN[0] == DefaultCDN { + config.cdn = make(map[string]interface{}) + } + + config.cdn[parts[1]] = nil + config.role = "" + } + } + + config.filename = configFilename + + configLocations := []ConfigLocation{UnknownLocation} + if p.ParentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + var mappings []*Mapping + + // mappings are like explicit, but smarter + // They're parent/child aware, no longer requiring different rulesets for each type. + for _, m := range p.Mappings { + + schemes := p.Schemes + if len(m.Schemes) > 0 { + schemes = m.schemes() + } + + for _, alias := range m.Alias { + for _, scheme := range schemes { + for _, configLocation := range configLocations { + + incoming := formatURL(scheme, alias) + originURL := formatURL(scheme, m.Origin) + + // for parent/child, use the first incoming alias as the origin URL. on the parent side, + // we map it as the inbound + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + originURL = formatURL(scheme, m.Alias[0]) + } + + if configLocation == ParentConfig && m.RemapOptions.isParentChildEnabled() { + if len(m.Alias) > 0 && alias != m.Alias[0] { + continue + } + } + + // clone separate versions of RemapOptions with the appropriate key set + ro := m.RemapOptions.Clone() + + // clear out manually set attributes + delete(ro, Parent) + delete(ro, Child) + + switch configLocation { + case ParentConfig: + ro[Parent] = true + case ChildConfig: + ro[Child] = true + } + + mapping := &Mapping{ + Origin: originURL, + Alias: []string{incoming}, + Group: DefaultGroup, + Schemes: []string{scheme}, + RegexMap: m.RegexMap, + RuleName: m.RuleName, + RemapOptions: ro, + id: m.id, + ParentOverride: m.ParentOverride, + } + + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + // only include the first alias of an alias set. as seen above, we only map + // through the first alias to the parent + if alias == m.Alias[0] { + if len(m.ParentDestDomain) > 0 { + mapping.ParentDestDomain = m.ParentDestDomain + } else { + mapping.ParentDestDomain = stripScheme(incoming) + } + } + } + + mappings = append(mappings, mapping) + } + } + } + } + + for _, m := range p.ExplicitMappings { + m.explicit = true + mappings = append(mappings, m) + } + + for _, m := range mappings { + + schemes := p.Schemes + if len(m.Schemes) > 0 { + schemes = m.schemes() + } + + for _, scheme := range schemes { + for _, incoming := range m.Alias { + + for _, configLocation := range configLocations { + if !forCurrentMatch(configLocation, &m.RemapOptions) { + continue + } + + env := newConfigEnvironment(p, options, m.id, configLocation) + + mrules, err := convertToMappingRules(m.RemapOptions, env, m.explicit) + if err != nil { + return nil, err + } + + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + config.parentDomains = append(config.parentDomains, m.ParentDomain()) + } + + incomingURL := formatURL(scheme, incoming) + originURL := formatURL(scheme, m.Origin) + + remap := Remap{ + IncomingURL: incomingURL, + OriginURL: originURL, + Group: m.Group, + Role: m.Role, + RegexMap: m.RegexMap, + mappingRules: mrules, + ConfigLocation: env.ConfigLocation, + sourceConfig: env, + scheme: scheme, + parentOverrides: m.ParentOverride, + parentDomain: m.ParentDomain(), + } + + config.Remaps = append(config.Remaps, remap) + } + } + } + } + + // For now, sort if the config has mappings. Later, introduce the change for explicit and others + if len(p.Mappings) > 0 { + sort.Stable(sort.Reverse(byIncomingURL(config.Remaps))) + } + + return config, nil +} + +func stripScheme(in string) string { + if !strings.Contains(in, "/") { + return in + } + parts := strings.Split(in, "/") + return parts[2] +} + +func forCurrentMatch(configLocation ConfigLocation, ro *RemapOptions) bool { + if configLocation == ChildConfig && ro.isParentConfig() { + return false + } + if configLocation == ParentConfig && ro.isChildConfig() { + return false + } + return true +} + +func (m *Mapping) schemes() []string { + if len(m.Schemes) > 0 { + return m.Schemes + } + + return []string{""} +} + +// formatURL will create URL from scheme and incoming, unless incoming already has scheme applied to it +func formatURL(scheme, incoming string) string { + if strings.HasPrefix(incoming, "/") { + return incoming + } + + if len(scheme) == 0 { + return incoming + } + + u, err := url.Parse(incoming) + if err == nil && strings.HasPrefix(u.Scheme, "http") { + return incoming + } + + // in the error case, it could be a regex URL that url.Parse barfs on. try a brute force method + if strings.HasPrefix(incoming, "http:") || strings.HasPrefix(incoming, "https:") { + return incoming + } + + return fmt.Sprintf("%s://%s", scheme, incoming) +} + +// isParentChildEnabled returns the value specified by "parent_child". defaults to true +func (options *RemapOptions) isParentChildEnabled() bool { + value, err := options.ValueByNameAsBool("parent_child") + if err == nil { + return value + } + return true +} + +// isParentConfig returns the value specified by "parent". defaults to false +func (options *RemapOptions) isParentConfig() bool { + value, err := options.ValueByNameAsBool(Parent) + if err == nil { + return value + } + return false +} + +// isChildConfig returns the value specified by "child". defaults to false +func (options *RemapOptions) isChildConfig() bool { + value, err := options.ValueByNameAsBool(Child) + if err == nil { + return value + } + return false +} + +// Iterate over all "rules". adding to array of *ATSPLugin +// If a general adapter, use general logic +// If an action, use action logic +// otherwise, assume a compound adapter +// - group by type +// Iterate over groups of compounds (calculated above) +// Append to adapter array +// Sort array by weights and return + +func convertToMappingRules(ro RemapOptions, env *ConfigEnvironment, explicit bool) ([]mappingRule, error) { + var adapters []mappingRule + + compounds := make(map[AdapterType][]AdapterType) + + // stable ordering + var options []string + for option := range ro { + options = append(options, option) + } + sort.Strings(options) + + var errs Errors + + seen := make(map[Adapter]interface{}) + for _, option := range options { + if option == "storage_volume" { // prevent warning on rule property that isn't associated with an adapter + continue + } + adapterType := AdaptersRegistry.adapterTypeByConfigName(option) + if adapterType == UnknownAdapter { + // Warn when there's garbage in + if !IsValidConfigurationOption(option) { + errs = append(errs, fmt.Errorf("unknown configuration option %q", option)) + } + continue + } + + vp := AdaptersRegistry.adapterForType(adapterType) + if vp == nil { + errs = append(errs, fmt.Errorf("unknown adapter %q", adapterType)) + continue + } + + // NOTE: even though an adapter can have multiple options associated with it, + // it's the first invocation that only matters. Once inside PParams, the adapter + // is responsible for determining any other usage of itself + if _, ok := seen[vp]; ok { + continue + } + seen[vp] = nil + + if env.ConfigLocation != UnknownLocation { + vpcp, ok := vp.(ParentChildAdapter) + matches := false + if ok { + for _, loc := range vpcp.ConfigLocations() { + if loc == env.ConfigLocation { + matches = true + } + } + + if !matches && explicit { + // ignore and let through + } else if !matches { + continue + } + } + } + + env.RemapOptions = ro + + switch vp.PluginType() { + case GeneralAdapter: + adapter, err := createAdapter(vp, env) + if err != nil { + errs = append(errs, err) + continue + } + + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + case ActionAdapter: + adapter, err := createAction(vp, env) + if err != nil { + errs = append(errs, err) + continue + } + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + case CompoundAdapter: + cta, ok := vp.(CompoundTypeAdapter) + if !ok { + errs = append(errs, fmt.Errorf("compound adapter does not implement CompoundTypeAdapter")) + continue + } + compounds[cta.CompoundType()] = append(compounds[cta.CompoundType()], adapterType) + } + } + + for pt, sp := range compounds { + + sort.Sort(receiptsFirst(sp)) + + adapter, err := createCompoundMappingRule(pt, ro, sp, env) + if err != nil { + errs = append(errs, err) + continue + } + + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + + } + + sort.Stable(byRuleWeight(adapters)) + + if len(errs) > 0 { + return nil, errs + } + + return adapters, nil +} + +func createCompoundMappingRule(compoundType AdapterType, ro RemapOptions, adapters []AdapterType, env *ConfigEnvironment) (*mappingRule, error) { + vp := AdaptersRegistry.adapterForType(compoundType) + if vp == nil { + return nil, fmt.Errorf("unknown AdapterType %q", compoundType) + } + + var errs Errors + var content bytes.Buffer + + adapter := baseCreateAdapter(vp, env) + + ccontent, err := ConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + adapter.ConfigContent = ccontent + + ccontent, err = SubConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + + if ccontent != nil { + content.Write(ccontent.Bytes()) + } + + var buffer bytes.Buffer + for _, adapterType := range adapters { + vps := AdaptersRegistry.adaptersForType(adapterType) + for _, vp = range vps { + if vp == nil { + return nil, fmt.Errorf("unknown AdapterType %s", adapterType) + } + + // filter out unrelated types + cta, ok := vp.(CompoundTypeAdapter) + if ok && cta.CompoundType() != compoundType { + continue + } + + vpp, ok := vp.(ParameterAdapter) + if ok { + pparams, err := vpp.PParams(env) + if err != nil { + errs = append(errs, err) + } + + for _, val := range pparams { + fmt.Fprintf(&buffer, " @pparam=%s", val) + } + } + + env.RemapOptions = ro + val, err := SubConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + + if val != nil && val.Len() > 0 { + content.Write(val.Bytes()) + content.WriteString("\n") + } + } + } + + if len(errs) > 0 { + return nil, errs + } + + if buffer.Len() > 0 { + adapter.ConfigContent.WriteString(buffer.String()) + } else if content.Len() == 0 { + // if no subconfig content, then there's nothing to be output (at least at this time) + adapter.ConfigContent = &bytes.Buffer{} + } + + adapter.SubConfigContent = &content + + return adapter, nil +} + +func baseCreateAdapter(vp Adapter, env *ConfigEnvironment) *mappingRule { + adapter := newMappingRule(vp) + + scp, ok := vp.(SubConfigAdapter) + if ok { + adapter.SubConfigFileName = SubConfigFilename(scp, env) + } + + return adapter +} + +func createAdapter(vp Adapter, env *ConfigEnvironment) (*mappingRule, error) { + adapter := baseCreateAdapter(vp, env) + + content, err := ConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.ConfigContent = content + + content, err = SubConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.SubConfigContent = content + + return adapter, nil +} + +func createAction(vp Adapter, env *ConfigEnvironment) (*mappingRule, error) { + adapter := baseCreateAdapter(vp, env) + content, err := SubConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.SubConfigContent = content + adapter.ConfigContent = content + + return adapter, nil +} + +// receiptsFirst implements sort.Interface, sorting by AdapterType, bubbling Receipt to the top. +type receiptsFirst []AdapterType + +func (s receiptsFirst) Len() int { return len(s) } +func (s receiptsFirst) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s receiptsFirst) Less(i, j int) bool { return s[i] == Receipt } + +// byRuleWeight implements sort.Interface, sorting by a Rule's Weight. +type byRuleWeight []mappingRule + +func (s byRuleWeight) Len() int { return len(s) } +func (s byRuleWeight) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byRuleWeight) Less(i, j int) bool { return s[i].Weight < s[j].Weight } + +// byIncomingURL will sort by incoming URL domain +// for the same scheme and inbound host: +// http will be put before https +// empty paths come last +// sorting alphabetically +// ignoring host-less and regex remaps +type byIncomingURL []Remap + +func (s byIncomingURL) Len() int { return len(s) } +func (s byIncomingURL) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byIncomingURL) Less(i, j int) bool { + if s[i].RegexMap || s[j].RegexMap { + return false + } + + u1, _ := url.Parse(s[i].IncomingURL) + u2, _ := url.Parse(s[j].IncomingURL) + if u1 == nil || u2 == nil { + return len(s[i].IncomingURL) < len(s[j].IncomingURL) + } + + if len(u1.Host) == 0 || len(u2.Host) == 0 { + return false + } + + if u1.Host == u2.Host { + if u1.Scheme == u2.Scheme { + if len(u1.Path) == 0 || len(u2.Path) == 0 { + return u1.Path < u2.Path + } + return u1.Path > u2.Path + } + return !(u1.Scheme == "http" || u2.Scheme == "http") + } + + return u1.Path < u2.Path +} diff --git a/tools/voluspa/config_writer.go b/tools/voluspa/config_writer.go new file mode 100644 index 00000000000..73abed44011 --- /dev/null +++ b/tools/voluspa/config_writer.go @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" +) + +type FilesystemWriter struct { + options *Options +} + +func NewFilesystemWriter(options *Options) (*FilesystemWriter, error) { + fw := &FilesystemWriter{options: options} + err := os.MkdirAll(fw.options.Destination, 0755) + if err != nil { + return nil, err + } + return fw, nil +} + +func (f *FilesystemWriter) WriteFiles(files []ManagedFile) error { + groups := make(map[string]interface{}) + for _, mf := range files { + groups[mf.Role] = nil + } + + for _, managedFile := range files { + if _, exists := groups[managedFile.Role]; managedFile.Role != "" && !exists { + continue + } + + filename := fmt.Sprintf("%s/%s", f.options.Destination, managedFile.Filename) + if f.options.Verbose { + log.Printf("Writing %s", filename) + } + + err := os.MkdirAll(filepath.Dir(filename), 0755) + if err != nil { + return err + } + + err = ioutil.WriteFile(filename, managedFile.Contents.Bytes(), 0644) + if err != nil { + return err + } + } + return nil +} + +type ConfigExpander struct { + options *Options +} + +func NewConfigExpander(options *Options) *ConfigExpander { + return &ConfigExpander{options: options} +} + +func (c *ConfigExpander) ExpandRemapConf(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + groups := make(map[string]bool) + for _, config := range parsedConfig.Remaps { + groups[config.Group] = true + } + + configLocations := []ConfigLocation{UnknownLocation} + if parsedConfig.parentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + for _, configLocation := range configLocations { + for group := range groups { + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + var hasContent bool + for _, config := range parsedConfig.Remaps { + if group != DefaultGroup && config.Group != group { + continue + } + + if configLocation != UnknownLocation && config.ConfigLocation != configLocation { + continue + } + + var roleEnabled bool + if len(config.Role) > 0 { + roleEnabled = true + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n\n", config.Role)) + } + + buf.WriteString(config.asRemapConf()) + buf.WriteString("\n") + + if roleEnabled { + buf.WriteString("{% endif %}\n\n") + } + + hasContent = true + } + + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), parsedConfig.remapConfigFilename(group, configLocation)) + + if !hasContent { + return nil, fmt.Errorf("Configuration for %s empty. Not writing %s", parsedConfig.property, filename) + } + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, configLocation)) + } + } + } + + return managedFiles, nil +} + +func (c *ConfigExpander) ExpandSubConfigs(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, remap := range parsedConfig.Remaps { + managedSubFiles, err := c.expandSubConfig(parsedConfig, remap) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, managedSubFiles...) + } + return managedFiles, nil +} + +func (c *ConfigExpander) expandSubConfig(parsedConfig *CustomerConfig, remap Remap) ([]ManagedFile, error) { + var managedFiles []ManagedFile + for i := range remap.mappingRules { + adapter := remap.mappingRules[i] + value := adapter.SubConfigContent + + if len(adapter.SubConfigFileName) == 0 { + continue + } + if value == nil || value.Len() == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.Write(value.Bytes()) + + // append newline if config does not end with one + if value.Bytes()[len(value.Bytes())-1] != '\n' { + buf.WriteByte('\n') + } + + // TODO new method off of PCC? + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), adapter.SubConfigFileName) + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, remap.ConfigLocation)) + } + } + + return managedFiles, nil +} diff --git a/tools/voluspa/consts.go b/tools/voluspa/consts.go new file mode 100644 index 00000000000..d51a0f98c71 --- /dev/null +++ b/tools/voluspa/consts.go @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +const ( + // DefaultTrafficserverConfigurationDir is the default install dir for ATS + DefaultTrafficserverConfigurationDir = "/opt/ats/etc/trafficserver" + + // Child is the label for child hosts, configurations or attributes + Child = "child" + // Parent is the label for parent hosts, configurations, or attributes + Parent = "parent" +) diff --git a/tools/voluspa/customer_config.go b/tools/voluspa/customer_config.go new file mode 100644 index 00000000000..59960d10ae0 --- /dev/null +++ b/tools/voluspa/customer_config.go @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +const ( + // DefaultGroup is the default group name + DefaultGroup = "all" + + DefaultRole = "default" + + remapPrefixBuffer = " " + remapMaxLineLength = 100 + + // generatedFileBanner is the text prepended to all config files + generatedFileBanner = "# Code generated by Voluspa. DO NOT EDIT.\n\n" +) + +// NOTE CustomerConfig and PropertyConfig are closely related. +// Every time I think I want to embed one in the other (or share a common struct), +// I'm reminded that the YAML parser does not handle embedded structs. + +type CustomerConfig struct { + owner string + reference string + lifecycle LifecycleState + property string + role string + filename string + sslCertNames []string + qps int + parentChild bool + parentDomains []string + schemes []string + cdn map[string]interface{} + Remaps []Remap + HAProxyVIPs []HAProxyVIP +} + +type Remap struct { + OriginURL string + IncomingURL string + Group string + Role string + scheme string + RegexMap bool + ConfigLocation ConfigLocation + mappingRules []mappingRule + sourceConfig *ConfigEnvironment + parentOverrides []ParentOverride + parentDomain string +} + +// newCustomerConfig creates a new CustomerConfig from a PropertyConfig +func newCustomerConfig(p *PropertyConfig) (*CustomerConfig, error) { + cc := &CustomerConfig{ + property: p.Property, + sslCertNames: p.SSLCertNames, + parentChild: p.ParentChild, + lifecycle: p.Lifecycle, + qps: p.QPS, + owner: p.Owner, + reference: p.Reference, + role: p.Role, + cdn: make(map[string]interface{}), + schemes: p.Schemes, + HAProxyVIPs: p.HAProxyVIPs, + } + for _, cdn := range p.CDN { + cc.cdn[cdn] = nil + } + return cc, nil +} + +func (c Remap) hasContent() bool { + hasContent := false + for i := range c.mappingRules { + adapter := c.mappingRules[i] + + if adapter.hasContent() { + hasContent = true + } + } + return hasContent +} + +func splitLine(value, prefix string) string { + if len(value) < remapMaxLineLength { + return value + } + + if strings.Contains(value, "{%") { + return value + } + + words := strings.Split(value, " ") + if len(words) < 2 { + return value + } + + out := bytes.Buffer{} + out.WriteString(fmt.Sprintf("%s %s", words[0], words[1])) + + for i := 2; i < len(words); i++ { + if words[i-1] == "@pparam=--config" { + out.WriteString(fmt.Sprintf(" %s", words[i])) + continue + } + out.WriteString(fmt.Sprintf(" \\\n%s %s", prefix, words[i])) + } + return out.String() + +} + +func (c Remap) asRemapConf() string { + var buf bytes.Buffer + if c.RegexMap { + buf.WriteString(fmt.Sprintf("regex_map")) + } else { + buf.WriteString(fmt.Sprintf("map")) + } + buf.WriteString(fmt.Sprintf(" %s \\\n %s", c.IncomingURL, c.OriginURL)) + + if !c.hasContent() { + buf.WriteString("\n") + return buf.String() + } + + buf.WriteString(" \\\n") + + for i := range c.mappingRules { + adapter := c.mappingRules[i] + + if !adapter.hasContent() { + continue + } + + prefixBuffer := remapPrefixBuffer + content := adapter.ConfigContent.String() + if adapter.isCommandLineTemplated() && strings.HasPrefix(content, "{%") { + prefixBuffer = "" + } + + buf.WriteString(fmt.Sprintf("%s%s", prefixBuffer, splitLine(content, prefixBuffer))) + if adapter.isCommandLineTemplated() { + continue + } + + if i < len(c.mappingRules)-1 && c.mappingRules[i+1].hasContent() { + buf.WriteString(" \\") + } + buf.WriteString("\n") + } + + return buf.String() +} + +func (c *CustomerConfig) remapConfigFilename(group string, configLocation ConfigLocation) string { + return configFilename(strings.ToLower(c.property), group, configLocation) +} + +func configFilename(baseFilename, group string, configLocation ConfigLocation) string { + if group == DefaultGroup || group == "" { + if configLocation == ParentConfig { + return fmt.Sprintf("%s_parent.config", baseFilename) + } + return fmt.Sprintf("%s.config", baseFilename) + } + + if configLocation == ParentConfig { + return fmt.Sprintf("%s_%s_parent.config", baseFilename, group) + } + + return fmt.Sprintf("%s_%s.config", baseFilename, group) +} diff --git a/tools/voluspa/doc.go b/tools/voluspa/doc.go new file mode 100644 index 00000000000..59005d41277 --- /dev/null +++ b/tools/voluspa/doc.go @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* +Package voluspa generates ATS configuration files for a CDN from a set of +simple YAML files that express the desired behaviour of each of the hosted properties. + +For example, given this property specification: + + schema_version: 1 + owner: Web Group + rcpt: httpbin + + mappings: + - alias: + - httpbin.example.com + origin: https://httpbin.origin.example.com + schemes: [http, https] + + rules: + default: + priority: background + origin_host_header: origin + deny_methods: [ CONNECT ] + +Voluspa will generate the corresponding ATS configuration files in the +specified output directory. A `httpbin/httpbin.config` file: + + # Code generated by Voluspa. DO NOT EDIT. + map http://httpbin.example.com \ + https://https://httpbin.origin.example.com \ + @plugin=conf_remap.so @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @plugin=header_rewrite.so @pparam=httpbin/hdrs.config \ + @plugin=propstats.so @pparam=--prefix=proxy.domain.httpbin \ + @action=deny @method=CONNECT + +and a `httpbin/hdrs.config` file: + + # Code generated by Voluspa. DO NOT EDIT. + cond %{REMAP_PSEUDO_HOOK} + set-header @ReceiptService "httpbin{{hosttype}}" + cond %{REMAP_PSEUDO_HOOK} + set-conn-dscp 8 + +The voluspa package enables fine control over the generation of configurations +and enables functionality to be extended via "adapters". + +The distribution includes a `voluspa` command line utility that loads a set of +YAML files and generates the ATS configuration files using code like this: + + import ( + "flag" + "fmt" + "log" + + "github.com/apache/trafficserver/tools/voluspa" + _ "github.com/apache/trafficserver/tools/voluspa/adapters" + ) + + v, err := voluspa.NewVoluspa() + if err != nil { + log.Fatalf("voluspa: %s", err) + } + + // Load the YAML config file for each property in the CDN + for _, filename := range flag.Args() { + err = v.AddConfig(filename) + if err != nil { + log.Fatalf("%s", err) // err includes the filename + } + } + + // Validate the set of configurations, individually and as a whole + if err = v.Validate(true); err != nil { + log.Fatalf("%s", err) + } + + // Generate the resulting ATS config files + if err = v.WriteAllATSFiles(); err != nil { + log.Fatalf("%s", err) + } + +*/ +package voluspa diff --git a/tools/voluspa/haproxy.go b/tools/voluspa/haproxy.go new file mode 100644 index 00000000000..798633e5e8f --- /dev/null +++ b/tools/voluspa/haproxy.go @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +// HAProxyVIP describes a haproxy VIP +type HAProxyVIP struct { + Port int + RIPs []RIP +} + +// RIP is the real IP and Port for a HAProxyVIP +type RIP struct { + IP string + Port int +} + +type HAProxyConfigGenerator struct { +} + +func (h *HAProxyConfigGenerator) Do(parsedConfigs []*CustomerConfig, merged bool) ([]ManagedFile, error) { + var results []ManagedFile + for _, parsedConfig := range parsedConfigs { + if len(parsedConfig.HAProxyVIPs) == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + for _, hapConfig := range parsedConfig.HAProxyVIPs { + + buf.WriteString(fmt.Sprintf("frontend %s 127.0.0.1:%d\n", parsedConfig.property, hapConfig.Port)) + buf.WriteString(" default_backend app\n") + buf.WriteString(" mode http\n") + buf.WriteString("\n") + buf.WriteString(fmt.Sprintf("backend %s\n", parsedConfig.property)) + buf.WriteString(" mode http\n") + buf.WriteString(" balance roundrobin\n") + buf.WriteString(" option httpchk GET / HTTP/1.1\\r\\nHost:localhost\n") + + for i, rip := range hapConfig.RIPs { + buf.WriteString(fmt.Sprintf(" server %s%03d %s:%d check\n", parsedConfig.property, i+1, rip.IP, rip.Port)) + } + + filename := fmt.Sprintf("%s/%s.hap.config", strings.ToLower(parsedConfig.property), strings.ToLower(parsedConfig.property)) + + results = append(results, ManagedFile{ + Filename: filename, + Role: "", + Property: parsedConfig.property, + Contents: &buf, + ConfigType: UnknownLocation, + }) + } + } + + return results, nil +} diff --git a/tools/voluspa/hosting_config.go b/tools/voluspa/hosting_config.go new file mode 100644 index 00000000000..315dfe00c1e --- /dev/null +++ b/tools/voluspa/hosting_config.go @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +type HostingConfigurator struct { + options *Options +} + +const ( + ramVolString string = "{{ ramdisk_volume }}" + defaultVolString string = "{{ default_volumes }}" + + HostingConfigDefaultFilename = "hosting.config_default" +) + +type StorageType int + +const ( + ramVolume StorageType = iota + diskVolume +) + +// hostingHostnameMap maps hostnames to StorageType +type hostingHostnameMap map[string]StorageType + +type hostingHostname struct { +} + +func newHostingHostname(hn string) *hostingHostname { + return &hostingHostname{} +} + +func newHostingConfigurator(options *Options) *HostingConfigurator { + return &HostingConfigurator{ + options: options, + } +} + +func (h *HostingConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return h.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate hosting.config for each CDN separately + grouped := groupCustomerConfigsByCDN(parsedConfigs) + + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := h.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil + +} + +func (h *HostingConfigurator) get(parsedConfigs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + + hostingHostnames := make(map[string]hostingHostnameMap) + for _, parsedConfig := range parsedConfigs { + role := parsedConfig.role + if hostingHostnames[role] == nil { + hostingHostnames[role] = make(hostingHostnameMap) + } + for _, remap := range parsedConfig.Remaps { + destURL := remap.OriginURL + if remap.sourceConfig.RemapOptions.HasOptionSet("origin_host_header") { + value, err := remap.sourceConfig.RemapOptions.ValueByNameAsString("origin_host_header") + if err != nil { + return nil, err + } + if value == "alias" { + destURL = remap.IncomingURL + } + } + u, err := url.Parse(destURL) + if err != nil { + return nil, err + } + storageConfig := "disk_volume" // default + if remap.sourceConfig.RemapOptions.HasOptionSet("storage_volume") { + value, err := remap.sourceConfig.RemapOptions.ValueByNameAsString("storage_volume") + if err != nil { + return nil, err + } + storageConfig = value + } + if storageConfig == "ramdisk_volume" { + hostingHostnames[role][u.Host] = ramVolume + } else { + hostingHostnames[role][u.Host] = diskVolume + } + } + } + + return h.ExpandConfigTemplate(hostingHostnames, cdn) +} + +func (h *HostingConfigurator) ExpandConfigTemplate(hostingHostnamesByRole map[string]hostingHostnameMap, cdn string) ([]ManagedFile, error) { + seen := make(map[string]bool) + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n", h.options.RamDiskRole)) + + var roles []string + for k := range hostingHostnamesByRole { + roles = append(roles, k) + } + sort.Strings(roles) + + for _, role := range roles { + hostingHostnames := hostingHostnamesByRole[role] + + var hostnames []string + for hostname := range hostingHostnames { + hostnames = append(hostnames, hostname) + } + + if len(hostnames) == 0 { + continue + } + + sort.Strings(hostnames) + + hasRamVolumeHosts := false + for _, hostname := range hostnames { + storage := hostingHostnames[hostname] + if storage == ramVolume { + hasRamVolumeHosts = true + break + } + } + + if !hasRamVolumeHosts { + continue + } + + buf.WriteString(h.startRoleGuard(role)) + + for _, hostname := range hostnames { + if _, exists := seen[hostname]; exists { + continue + } + + storage := hostingHostnames[hostname] + if storage == ramVolume { + seen[hostname] = true + buf.WriteString(fmt.Sprintf("hostname=%s %s\n", hostname, ramVolString)) + } + } + buf.WriteString(h.endRoleGuard(role)) + } + + buf.WriteString(fmt.Sprintf("hostname=* %s\n", defaultVolString)) + + buf.WriteString("{% else %}\n# hosting.config disabled on this host\n{% endif %}\n") + + filename := HostingConfigDefaultFilename + if len(cdn) > 0 && (cdn != DefaultCDN && cdn != NoCDN) { + filename = fmt.Sprintf("hosting.config_%s", cdn) + } + + return []ManagedFile{NewManagedFile(filename, "hosting.config", cdn, "", "", &buf, UnknownLocation)}, nil + +} + +func (h *HostingConfigurator) startRoleGuard(role string) string { + if len(role) == 0 { + return "" + } + + roleName := role + if !strings.HasPrefix(role, "roles_") { + roleName = fmt.Sprintf("roles_%s", role) + } + return fmt.Sprintf("{%% if salt.pillar.get('%s') %%}\n", roleName) +} + +func (h *HostingConfigurator) endRoleGuard(role string) string { + if len(role) == 0 { + return "" + } + return "{% endif %}\n\n" +} diff --git a/tools/voluspa/internal/util/regex/regex.go b/tools/voluspa/internal/util/regex/regex.go new file mode 100644 index 00000000000..f5717b40687 --- /dev/null +++ b/tools/voluspa/internal/util/regex/regex.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +type Validator interface { + IsValid(in string) (bool, error) +} diff --git a/tools/voluspa/internal/util/regex/regex_test.go b/tools/voluspa/internal/util/regex/regex_test.go new file mode 100644 index 00000000000..ad438a75b92 --- /dev/null +++ b/tools/voluspa/internal/util/regex/regex_test.go @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +import ( + "testing" +) + +func TestGoLangRegex(t *testing.T) { + testCases := []struct { + u string + e bool + }{ + {u: "asdf", e: true}, + {u: `^[-_A-Za-z0-9\.]+$`, e: true}, + {u: "", e: true}, + {u: " ^/([^/]*-eu-dub-[^/]*)/(.*) https://s3-eu-west-1.amazonaws.com$0", e: true}, + {u: "[", e: false}, + } + + glr := &GoLangRegex{} + for _, tc := range testCases { + e, err := glr.IsValid(tc.u) + if e != tc.e { + t.Errorf("%q: expected %t got %t: %s", tc.u, tc.e, e, err) + } + } +} diff --git a/tools/voluspa/internal/util/regex/vregex.go b/tools/voluspa/internal/util/regex/vregex.go new file mode 100644 index 00000000000..cde553967f2 --- /dev/null +++ b/tools/voluspa/internal/util/regex/vregex.go @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +import ( + "regexp" +) + +// GoLangRegex implements regex.Validator interface using go's builtin regex methods +type GoLangRegex struct { +} + +func (e *GoLangRegex) IsValid(in string) (bool, error) { + _, err := regexp.Compile(in) + return err == nil, err +} diff --git a/tools/voluspa/lifecycle.go b/tools/voluspa/lifecycle.go new file mode 100644 index 00000000000..c59186feaa3 --- /dev/null +++ b/tools/voluspa/lifecycle.go @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +type LifecycleState string + +const ( + UnknownLifecycleState LifecycleState = "" + Onboarding LifecycleState = "onboarding" + Live LifecycleState = "live" + Retired LifecycleState = "retired" +) diff --git a/tools/voluspa/managed_file.go b/tools/voluspa/managed_file.go new file mode 100644 index 00000000000..48971622998 --- /dev/null +++ b/tools/voluspa/managed_file.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" +) + +type ManagedFile struct { + Filename string + RemoteFilename string + Role string + CDN string + Property string + Contents *bytes.Buffer + ConfigType ConfigLocation +} + +func NewManagedFile(filename, remoteFilename, cdn, role, property string, contents *bytes.Buffer, configType ConfigLocation) ManagedFile { + return ManagedFile{ + Filename: filename, + RemoteFilename: remoteFilename, + Role: role, + CDN: cdn, + Property: property, + Contents: contents, + ConfigType: configType, + } +} + +func (mf ManagedFile) String() string { + return fmt.Sprintf("Filename=%s RFN=%s Role=%s CDN=%s Property=%s Size=%d Type=%d", mf.Filename, mf.RemoteFilename, mf.Role, mf.CDN, mf.Property, mf.Contents.Len(), mf.ConfigType) +} diff --git a/tools/voluspa/options.go b/tools/voluspa/options.go new file mode 100644 index 00000000000..6888ede3903 --- /dev/null +++ b/tools/voluspa/options.go @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +type Options struct { + Verbose bool + SkipSchemaValidation bool + PromoteRolesToCDN bool + Destination string + DefaultsLocation string + SchemaLocation string + RamDiskRole string +} diff --git a/tools/voluspa/parent_config.go b/tools/voluspa/parent_config.go new file mode 100644 index 00000000000..5b825d12080 --- /dev/null +++ b/tools/voluspa/parent_config.go @@ -0,0 +1,393 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +// parentDomainMap maps domain names to parentDomain struct +type parentDomainMap map[string]*parentDomain + +type ParentConfigurator struct { + options *Options +} + +type Override struct { + schemes map[string]bool + primaryList []string + secondaryList []string + HTTPPort int + HTTPSPort int + strategy string + goDirect bool + ignoreQueryString bool + overrideDomain string +} + +type parentDomain struct { + originalDomain string + schemes map[string]bool + ignoreQueryString bool + OverridesByRole map[string]Override +} + +func newParentDomain(domain string) *parentDomain { + return &parentDomain{originalDomain: domain, schemes: make(map[string]bool), OverridesByRole: make(map[string]Override)} +} + +func newParentConfigurator(options *Options) *ParentConfigurator { + return &ParentConfigurator{ + options: options, + } +} + +func (p *ParentConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return p.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate parent.config for each CDN separately + grouped := groupCustomerConfigsByCDN(parsedConfigs) + + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := p.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil + +} + +func (p *ParentConfigurator) get(parsedConfigs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + // a map of roles to parentDomainMaps + parentDomains := make(map[string]parentDomainMap) + for _, parsedConfig := range parsedConfigs { + + role := parsedConfig.role + if _, exists := parentDomains[role]; !exists { + parentDomains[role] = make(parentDomainMap) + } + + for _, domain := range parsedConfig.parentDomains { + if parsedConfig.lifecycle == Retired { + continue + } + + for _, remap := range parsedConfig.Remaps { + if remap.ConfigLocation == ParentConfig { + continue + } + + pd, ok := parentDomains[role][domain] + if !ok { + pd = newParentDomain(domain) + parentDomains[role][domain] = pd + } + + // extract schemes from toplevel or from remap rules + if len(remap.scheme) > 0 { + pd.schemes[remap.scheme] = true + } else { + for _, scheme := range parsedConfig.schemes { + pd.schemes[scheme] = true + } + } + + scheme, err := extractScheme(remap.IncomingURL) + if err != nil { + return nil, fmt.Errorf("could not extract scheme from %q: %s", remap.IncomingURL, err) + } + + if len(scheme) > 0 { + pd.schemes[scheme] = true + } + + if len(remap.parentOverrides) > 0 && remap.parentDomain == pd.originalDomain { + for _, o := range remap.parentOverrides { + ignore := o.IgnoreQuerystring + if hasRemoveAllParamsOption(remap.sourceConfig.RemapOptions) { + ignore = true + } + pd.OverridesByRole[o.Role] = Override{ + primaryList: o.PrimaryList, + secondaryList: o.SecondaryList, + HTTPPort: o.HTTPPort, + HTTPSPort: o.HTTPSPort, + ignoreQueryString: ignore, + strategy: o.Strategy, + goDirect: o.GoDirect, + overrideDomain: o.DestDomain, + } + } + } + + if remap.sourceConfig.RemapOptions == nil { + continue + } + + // We obey if it was set for any remap + if hasRemoveAllParamsOption(remap.sourceConfig.RemapOptions) || hasIgnoreQuery(remap.sourceConfig.RemapOptions) { + pd.ignoreQueryString = true + } + + } + } + } + + return p.ExpandConfigTemplate(parentDomains, cdn) +} + +func extractScheme(in string) (string, error) { + parsedURL, err := url.Parse(in) + if err == nil { + return parsedURL.Scheme, nil + } + + if strings.HasPrefix(in, "https:") { + return "https", nil + } + + if strings.HasPrefix(in, "http:") { + return "http", nil + } + + return "", nil +} + +func hasRemoveAllParamsOption(remapOptions RemapOptions) bool { + vals, err := remapOptions.ValueByNameAsStringMapInterface("cachekey") + if err != nil { + return false + } + + val, ok := vals["remove_all_params"] + if !ok { + return false + } + + optionValue, ok := val.(bool) + if !ok { + return false + } + + return optionValue +} + +func hasIgnoreQuery(remapOptions RemapOptions) bool { + vals, err := remapOptions.ValueByNameAsStringMapInterface("cachekey") + if err != nil { + return false + } + + val, ok := vals["parent_selection"] + if !ok { + return false + } + + optionValue, ok := val.(string) + if !ok { + return false + } + return optionValue == "ignore_query" +} + +func (p *ParentConfigurator) ExpandConfigTemplate(parentDomains map[string]parentDomainMap, cdn string) ([]ManagedFile, error) { + var files []ManagedFile + for role := range parentDomains { + parentDomains, ok := parentDomains[role] + if !ok { + return nil, fmt.Errorf("role %q not found", role) + } + + mf, err := p.expandConfigTemplate(parentDomains, cdn, role) + if err != nil { + return nil, err + } + + if mf == nil { + continue + } + + files = append(files, *mf) + + // ATS wants a file on parent hosts. match name that's used in salt + filename := "parent_empty.config" + if len(role) > 0 { + filename = fmt.Sprintf("parent_empty.config_%s", role) + } + + emptyParentConfig := NewManagedFile(filename, "parent.config", "", role, "", intentionallyEmptyMessage, ParentConfig) + files = append(files, emptyParentConfig) + } + + return files, nil +} + +// DomainNames allows for implementing the sort interface; regular string sort is not good enough +// we want the longest match to win in the case of bar.com and foo.bar.com. +type DomainNames []string + +// var _ sort.Interface = DomainNames{} + +func (d DomainNames) Len() int { + return len(d) +} +func (d DomainNames) Swap(i, j int) { + d[i], d[j] = d[j], d[i] +} + +func (d DomainNames) Less(i, j int) bool { + + if len(strings.Split(d[i], ".")) != len(strings.Split(d[j], ".")) { + return len(strings.Split(d[i], ".")) > len(strings.Split(d[j], ".")) + } + if strings.Compare(d[i], d[j]) < 0 { + return true + } + return false +} + +var intentionallyEmptyMessage = bytes.NewBufferString("# This file is intentionally left blank\n") + +func (p *ParentConfigurator) expandConfigTemplate(parentDomains parentDomainMap, cdn, role string) (*ManagedFile, error) { + seen := make(map[string]bool) + + var buf bytes.Buffer + buf.WriteString("\n") + + buf.WriteString(generatedFileBanner) + var domains DomainNames + for _, pd := range parentDomains { + domains = append(domains, pd.originalDomain) + } + sort.Sort(domains) + + for _, domain := range domains { + + if _, exists := seen[domain]; exists { + continue + } + + pd := parentDomains[domain] + + var schemes []string + for scheme := range pd.schemes { + schemes = append(schemes, scheme) + } + sort.Strings(schemes) + + for _, scheme := range schemes { + if len(pd.OverridesByRole) > 0 { + + // sort roles to make diffs more predictable + roles := make([]string, 0) + for role := range pd.OverridesByRole { + roles = append(roles, role) + } + sort.Strings(roles) + + for _, role := range roles { + override := pd.OverridesByRole[role] + port := override.HTTPPort + if scheme == "https" { + port = override.HTTPSPort + } + pstr := fmt.Sprintf(`parent="{{ %s_parents }}" {{ secondary_%s_parents }}`, scheme, scheme) // the default + if len(override.primaryList) > 0 { + primaryString := "parent=\"" + secondaryString := "secondary_parent=\"" + for _, h := range override.primaryList { + primaryString += fmt.Sprintf("%s:%d,", h, port) + } + primaryString = strings.TrimSuffix(primaryString, ",") + primaryString += "\"" + if len(override.secondaryList) > 0 { + for _, h := range override.secondaryList { + secondaryString += fmt.Sprintf("%s:%d,", h, port) + } + secondaryString = strings.TrimSuffix(secondaryString, ",") + secondaryString += "\"" + } else { + secondaryString = "" + } + pstr = fmt.Sprintf("%s %s", primaryString, secondaryString) + } + if role != "" { + buf.WriteString(fmt.Sprintf("\n{%% if salt.pillar.get(\"%s\") %%}\n", role)) + } + strategy := override.strategy + if strategy == "" { + strategy = "consistent_hash" + } + direct := override.goDirect // false is default? + finalDomain := domain + if override.overrideDomain != "" { + finalDomain = override.overrideDomain + } + buf.WriteString(fmt.Sprintf(`dest_domain=%s scheme=%s %s round_robin=%s go_direct=%t`, finalDomain, scheme, pstr, strategy, direct)) + if override.ignoreQueryString { + buf.WriteString(" qstring=ignore") + } + buf.WriteString("\n") + if role != "" { + buf.WriteString("{% endif %}\n\n") + } + } + } else { + // This is the "legacy way" with parent_dest_domwain at the mapping level NOTE DEPRECATED + buf.WriteString(fmt.Sprintf( + `dest_domain=%s scheme=%s parent="{{ %s_parents }}" {{ secondary_%s_parents }} round_robin=consistent_hash go_direct=false`, domain, scheme, scheme, scheme)) + if pd.ignoreQueryString { + buf.WriteString(" qstring=ignore") + } + buf.WriteString("\n") + } + } + seen[domain] = true + } + + buf.WriteString("\n") + + if len(seen) == 0 { + return nil, nil + } + + filename := "parent.config_default" + if len(role) > 0 { + filename = fmt.Sprintf("parent.config_%s", role) + } + + if cdn != DefaultCDN && cdn != NoCDN { + filename = fmt.Sprintf("parent.config_%s", cdn) + } + + mf := NewManagedFile(filename, "parent.config", cdn, role, "", &buf, ChildConfig) + + return &mf, nil +} diff --git a/tools/voluspa/parent_config_test.go b/tools/voluspa/parent_config_test.go new file mode 100644 index 00000000000..b4558690cfd --- /dev/null +++ b/tools/voluspa/parent_config_test.go @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "testing" +) + +func TestExtractScheme(t *testing.T) { + testCases := []struct { + in string + out string + }{ + {"https://store-(fred|123|test).domain.com/", "https"}, + {"http://store-(bob|test).domain.com/", "http"}, + {"http://www.domain.com/", "http"}, + {"www.domain.com/", ""}, + } + for i, tc := range testCases { + scheme, err := extractScheme(tc.in) + if err != nil { + t.Errorf("test #%d: unexpected error: %s", i, err) + continue + } + + if scheme != tc.out { + t.Errorf("test #%d: Sorted order expected %q got %q", i, tc.out, scheme) + } + } +} diff --git a/tools/voluspa/property_config.go b/tools/voluspa/property_config.go new file mode 100644 index 00000000000..fb327b6382f --- /dev/null +++ b/tools/voluspa/property_config.go @@ -0,0 +1,326 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "regexp" + "sort" + "unicode" + + yaml "gopkg.in/yaml.v2" +) + +// PropertyConfig represents the raw parsed structure from YAML +type PropertyConfig struct { + Owner string `json:"owner,omitempty" yaml:",omitempty"` + Reference string `json:"reference,omitempty" yaml:",omitempty"` + Lifecycle LifecycleState `json:"lifecycle,omitempty" yaml:",omitempty"` + Role string `json:"role,omitempty" yaml:",omitempty"` + Property string `json:"rcpt,omitempty" yaml:"rcpt,omitempty"` + SchemaVersion string `json:"schema_version,omitempty" yaml:"schema_version,omitempty"` + SSLCertNames []string `json:"ssl_cert_names,omitempty" yaml:"ssl_cert_names,omitempty"` + Schemes []string `json:"schemes,omitempty" yaml:",omitempty"` + CDN []string `json:"cdn,omitempty" yaml:",omitempty"` + Org string `json:"org,omitempty" yaml:"org,omitempty"` + Rules map[string]RemapOptions + ExplicitMappings []*Mapping `json:"explicit,omitempty" yaml:"explicit,omitempty"` + Mappings []*Mapping `json:"mappings,omitempty" yaml:"mappings,omitempty"` + ParentChild bool `json:"parent_child,omitempty" yaml:"parent_child,omitempty"` + QPS int `json:"qps" yaml:"qps"` + HAProxyVIPs []HAProxyVIP `json:"ha_proxy,omitempty" yaml:"ha_proxy,omitempty"` + Tests map[string]CDNTest `json:"tests,omitempty" yaml:"tests,omitempty"` + Receiptsd *ReceiptsdConfig `json:"receiptsd,omitempty" yaml:"receiptsd,omitempty"` +} + +type Mapping struct { + Origin string `json:"origin,omitempty" yaml:"origin,omitempty"` + Group string `json:"group,omitempty" yaml:"group,omitempty"` + Role string `json:"role,omitempty" yaml:"role,omitempty"` + Alias []string `json:"alias,omitempty" yaml:"alias,omitempty"` + Schemes []string `json:"schemes,omitempty" yaml:"schemes,omitempty"` + + RuleName string `json:"rule" yaml:"rule"` + ParentDestDomain string `json:"parent_dest_domain,omitempty" yaml:"parent_dest_domain,omitempty"` + ParentOverride []ParentOverride `json:"parent_override,omitempty" yaml:"parent_override,omitempty"` + RegexMap bool `json:"regex_map" yaml:"regex_map"` + RemapOptions RemapOptions `json:",omitempty" yaml:",omitempty"` + explicit bool + id int +} + +type ParentOverride struct { + Role string `json:"role,omitempty" yaml:"role,omitempty"` + DestDomain string `json:"dest_domain,omitempty" yaml:"dest_domain"` + PrimaryList []string `json:"primary_list,omitempty" yaml:"primary_list,omitempty"` + SecondaryList []string `json:"secondary_list,omitempty" yaml:"secondary_list,omitempty"` + Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` + IgnoreQuerystring bool `json:"ignore_querystring,omitempty" yaml:"ignore_querystring,omitempty"` + HTTPPort int `json:"http_port,omitempty" yaml:"http_port,omitempty"` + HTTPSPort int `json:"https_port,omitempty" yaml:"https_port,omitempty"` + GoDirect bool `json:"go_direct,omitempty" yaml:"go_direct,omitempty"` +} + +func (m *Mapping) ParentDomain() string { + + // a po without a role but with a domain + for _, po := range m.ParentOverride { + if po.Role == "" && len(po.DestDomain) > 0 { + return po.DestDomain + } + } + + // Deprecated + if len(m.ParentDestDomain) > 0 { + return m.ParentDestDomain + } + // the domain is not overriden, old style or new style, return the origin domain + parentURL, err := url.Parse(m.Origin) + if err == nil && len(parentURL.Scheme) > 0 { + return parentURL.Host + } + + return m.Origin +} + +const ( + NoCDN = "" + NoRole = "" + DefaultCDN = "edge" + DefaultScheme = "http" + DefaultRulesetName = "default" +) + +var propertyNameRE = regexp.MustCompile(`^[-_A-Za-z0-9\.]+$`) + +func isValidPropertyName(in string) bool { + return propertyNameRE.MatchString(in) +} + +// Setup validates and sets up a PropertyConfig +func (p *PropertyConfig) Setup() error { + + if len(p.Schemes) > 0 { + for _, scheme := range p.Schemes { + if !isValidScheme(scheme) { + return fmt.Errorf("invalid/unsupported scheme %q", scheme) + } + } + } + + // rcpt is the public-facing name of the field, Property is the internal name + if len(p.Property) == 0 { + return fmt.Errorf("rcpt is required") + } + + if !isValidPropertyName(p.Property) { + return fmt.Errorf("property name can only contain alphanumeric characters") + } + + // assign an id for each rule name + // sort keys/path for stable ids + matchIds := make(map[string]int) + { + var keys []string + for k := range p.Rules { + keys = append(keys, k) + } + sort.Strings(keys) + + i := 1 + for _, k := range keys { + matchIds[k] = i + i++ + } + } + + for _, m := range p.Mappings { + if m.RuleName == "" { + if _, ok := p.Rules["default"]; ok { + m.RuleName = "default" + } + } + + if m.Group == "" { + m.Group = DefaultGroup + } + + m.id = matchIds[m.RuleName] + + if ruleset, ok := p.Rules[m.RuleName]; ok { + m.RemapOptions = ruleset + } else { + return fmt.Errorf("invalid ruleset name: %q for origin %q", m.RuleName, m.Origin) + } + + if len(m.Alias) == 0 { + return fmt.Errorf("no alias specified for origin %q", m.Origin) + } + if hasLoop(m) { + return fmt.Errorf("alias and origin %q will loop", m.Origin) + } + } + + for _, m := range p.ExplicitMappings { + if m.RuleName == "" { + if _, ok := p.Rules[DefaultRulesetName]; ok { + m.RuleName = DefaultRulesetName + } + } + + if m.Group == "" { + m.Group = DefaultGroup + } + + m.id = matchIds[m.RuleName] + + if ruleset, ok := p.Rules[m.RuleName]; ok { + m.RemapOptions = ruleset + } else { + return fmt.Errorf("invalid ruleset name: %q for origin %q", m.RuleName, m.Origin) + } + + if len(m.Alias) == 0 { + return fmt.Errorf("no alias specified for origin %q", m.Origin) + } + } + + return nil +} + +func hasLoop(m *Mapping) bool { + for _, scheme := range m.schemes() { + for _, incoming := range m.Alias { + incomingURL := formatURL(scheme, incoming) + originURL := formatURL(scheme, m.Origin) + if incomingURL == originURL { + return true + } + } + } + return false +} + +func (p *PropertyConfig) loadDefaultConfig(defaultsLocation, filename string) error { + var fullPath string + if len(defaultsLocation) > 0 { + fullPath = fmt.Sprintf("%s/%s", defaultsLocation, filename) + } else { + fullPath = filename + } + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + return nil + } + + cfgContents, err := ioutil.ReadFile(fullPath) + if err != nil { + return err + } + + return p.parseConfig(cfgContents) +} + +func isASCII(in string) (bool, rune) { + for _, c := range in { + if c > unicode.MaxASCII { + return false, c + } + } + return true, 0 +} + +func (p *PropertyConfig) parseConfig(cfgContents []byte) error { + valid, char := isASCII(string(cfgContents)) + if !valid { + return fmt.Errorf("non-ASCII characters not supported: invalid rune %q", string(char)) + } + + return yaml.Unmarshal(cfgContents, &p) +} + +func (p *PropertyConfig) mergeConfig(c *PropertyConfig) { + overrides, ok := c.Rules[DefaultRulesetName] + if !ok { + return + } + + for _, ruleset := range p.Rules { + if ruleset == nil { + continue + } + for k, v := range overrides { + if _, exists := ruleset[k]; !exists { + ruleset[k] = v + } + } + } +} + +func newPropertyConfig() *PropertyConfig { + return &PropertyConfig{ + Rules: make(map[string]RemapOptions), + Schemes: []string{DefaultScheme}, + CDN: []string{DefaultCDN}, + Lifecycle: Live, + } +} + +// NewPropertyConfig creates a new PropertyConfig from a buffer containing a YAML file +func NewPropertyConfig(buffer []byte) (*PropertyConfig, error) { + return NewPropertyConfigWithDefaults(buffer, "") +} + +// NewPropertyConfigWithDefaults creates a new PropertyConfig from a buffer containing a YAML file +func NewPropertyConfigWithDefaults(buffer []byte, defaultsLocation string) (*PropertyConfig, error) { + cfg := newPropertyConfig() + + if err := cfg.parseConfig(buffer); err != nil { + return nil, fmt.Errorf("Could not load YAML file. err=%s", err) + } + + return inflatePropertyConfig(cfg, defaultsLocation) +} + +func inflatePropertyConfig(cfg *PropertyConfig, defaultsLocation string) (*PropertyConfig, error) { + dcfg := newPropertyConfig() + + if err := dcfg.loadDefaultConfig(defaultsLocation, "default.conf"); err != nil { + return nil, fmt.Errorf("Could not load %q: %s", "defaults/default.conf", err) + } + + cfg.mergeConfig(dcfg) + + if cfg.ParentChild { + dpCfg := newPropertyConfig() + + if err := dpCfg.loadDefaultConfig(defaultsLocation, "default_parent.conf"); err != nil { + return nil, fmt.Errorf("Could not load %q: %s", "defaults/default_parent.conf", err) + } + cfg.mergeConfig(dpCfg) + } + + if err := cfg.Setup(); err != nil { + return nil, err + } + return cfg, nil +} diff --git a/tools/voluspa/property_remaps.go b/tools/voluspa/property_remaps.go new file mode 100644 index 00000000000..19685ab70a7 --- /dev/null +++ b/tools/voluspa/property_remaps.go @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +type PropertyRemapGenerator struct { + options *Options +} + +func newPropertyRemapGenerator(options *Options) *PropertyRemapGenerator { + return &PropertyRemapGenerator{options: options} +} + +func (p *PropertyRemapGenerator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if len(parsedConfigs) == 0 { + return nil, ErrMinimumConfigsNotMet + } + + var managedFiles []ManagedFile + for _, parsedConfig := range parsedConfigs { + if parsedConfig.lifecycle == Retired { + continue + } + expanded, err := p.expandRemapConf(parsedConfig) + if err != nil { + return nil, fmt.Errorf("Could not write remap config for property %q: %s", parsedConfig.property, err) + } + + managedFiles = append(managedFiles, expanded...) + + expanded, err = p.expandSubConfigs(parsedConfig) + if err != nil { + return nil, fmt.Errorf("Could not write subconfigs for property %q: %s", parsedConfig.property, err) + } + + managedFiles = append(managedFiles, expanded...) + } + + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandRemapConf(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + groups := make(map[string]bool) + for _, config := range parsedConfig.Remaps { + groups[config.Group] = true + } + + configLocations := []ConfigLocation{UnknownLocation} + if parsedConfig.parentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + for _, configLocation := range configLocations { + for group := range groups { + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + var hasContent bool + for _, config := range parsedConfig.Remaps { + if group != DefaultGroup && config.Group != group { + continue + } + + if configLocation != UnknownLocation && config.ConfigLocation != configLocation { + continue + } + + var roleEnabled bool + if len(config.Role) > 0 { + roleEnabled = true + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n\n", config.Role)) + } + + buf.WriteString(config.asRemapConf()) + buf.WriteString("\n") + + if roleEnabled { + buf.WriteString("{% endif %}\n\n") + } + + hasContent = true + } + + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), parsedConfig.remapConfigFilename(group, configLocation)) + + if !hasContent { + return nil, fmt.Errorf("Configuration for %s empty. Not writing %s", parsedConfig.property, filename) + } + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, configLocation)) + } + } + } + + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandSubConfigs(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, remap := range parsedConfig.Remaps { + managedSubFiles, err := p.expandSubConfig(parsedConfig, remap) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, managedSubFiles...) + } + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandSubConfig(parsedConfig *CustomerConfig, remap Remap) ([]ManagedFile, error) { + var managedFiles []ManagedFile + for i := range remap.mappingRules { + adapter := remap.mappingRules[i] + value := adapter.SubConfigContent + + if len(adapter.SubConfigFileName) == 0 { + continue + } + if value == nil || value.Len() == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.Write(value.Bytes()) + + // append newline if config does not end with one + if value.Bytes()[len(value.Bytes())-1] != '\n' { + buf.WriteByte('\n') + } + + // TODO new method off of PCC? + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), adapter.SubConfigFileName) + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, remap.ConfigLocation)) + } + } + + return managedFiles, nil +} diff --git a/tools/voluspa/receiptsd.go b/tools/voluspa/receiptsd.go new file mode 100644 index 00000000000..6ee8434bae8 --- /dev/null +++ b/tools/voluspa/receiptsd.go @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +// ReceiptsdConfig is the config for receiptsd managed by voluspa +type ReceiptsdConfig struct { + IgnoredExtensions []string `json:"ignored_extensions,omitempty" yaml:"ignored_extensions"` + + Sampling *struct { + Posters Posters `json:"posters,omitempty" yaml:"posters"` + } `json:"sampling,omitempty"` +} + +// Posters is a key/value map of poster name to % to post +type Posters map[string]string diff --git a/tools/voluspa/registry.go b/tools/voluspa/registry.go new file mode 100644 index 00000000000..9bbcbf5a54e --- /dev/null +++ b/tools/voluspa/registry.go @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" + "log" +) + +var AdaptersRegistry *AdapterRegistry + +func init() { + AdaptersRegistry = NewAdapterRegistry() +} + +// AdapterRegistry is the central registry for all plugin config plugins +type AdapterRegistry struct { + adapters map[AdapterType]Adapter + compoundAdapters map[AdapterType][]Adapter + configNameAdapterMap map[string]AdapterType + fileNameMap map[string]AdapterType +} + +func NewAdapterRegistry() *AdapterRegistry { + return &AdapterRegistry{ + adapters: make(map[AdapterType]Adapter), + compoundAdapters: make(map[AdapterType][]Adapter), + configNameAdapterMap: make(map[string]AdapterType), + fileNameMap: make(map[string]AdapterType), + } +} + +func (r *AdapterRegistry) AddAdapter(p Adapter) { + if p.PluginType() == CompoundAdapter { + r.compoundAdapters[p.Type()] = append(r.compoundAdapters[p.Type()], p) + } else { + r.adapters[p.Type()] = p + } + + for _, v := range p.ConfigParameters() { + if v == "" { + continue + } + if previous, ok := r.configNameAdapterMap[v]; !ok { + r.configNameAdapterMap[v] = p.Type() + } else { + log.Printf("Duplicate config parameter: %s for %s. Previous AdapterType=%s", v, p.Type(), previous) + } + } + + scp, ok := p.(SubConfigAdapter) + if !ok { + return + } + if scp.Name() == "" { + return + } + if previous, ok := r.fileNameMap[scp.Name()]; !ok { + r.fileNameMap[scp.Name()] = p.Type() + } else { + log.Printf("Duplicate subconfig filename: %s for %s. Previous AdapterType=%s", p.Type(), scp.Name(), previous) + } +} + +func (r *AdapterRegistry) adapterForType(t AdapterType) Adapter { + vp := r.adapters[t] + if vp != nil { + return vp + } + + plugins := r.compoundAdapters[t] + if len(plugins) > 0 { + return plugins[0] + } + return nil +} + +func (r *AdapterRegistry) RemoveAdapterByType(adapterType string) error { + at := AdapterType(adapterType) + + var adapter Adapter + + nullAdapter, found := r.adapters[AdapterType("null")] + if !found { + return fmt.Errorf("%q adapter not found", "null") + } + + adapter, found = r.adapters[at] + if found { + r.adapters[at] = nullAdapter + } + + if !found { + return fmt.Errorf("unknown adapter type %q", adapterType) + } + + // override keywords + for _, v := range adapter.ConfigParameters() { + r.configNameAdapterMap[v] = nullAdapter.Type() + } + + return nil +} + +func (r *AdapterRegistry) adaptersForType(t AdapterType) []Adapter { + return r.compoundAdapters[t] +} + +func (r *AdapterRegistry) adapterTypeByConfigName(key string) AdapterType { + v, ok := r.configNameAdapterMap[key] + if !ok { + return UnknownAdapter + } + return v +} diff --git a/tools/voluspa/remap_options.go b/tools/voluspa/remap_options.go new file mode 100644 index 00000000000..e61d0c269f7 --- /dev/null +++ b/tools/voluspa/remap_options.go @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "encoding/json" + "fmt" + "strconv" +) + +// RemapOptions key/value pairs from yaml configuration +type RemapOptions map[string]interface{} + +func (r RemapOptions) HasOptionSet(name string) bool { + var isSet bool + switch v := r[name].(type) { + case string: + isSet = len(v) > 0 + case bool: + isSet = v + default: + return false + } + return isSet +} + +func (r RemapOptions) Clone() RemapOptions { + ro := RemapOptions{} + for k, v := range r { + ro[k] = v + } + return ro +} + +func (r RemapOptions) HasOption(name string) bool { + _, ok := r[name] + return ok +} + +func (r RemapOptions) AddOption(name string, val interface{}) bool { + r[name] = val + return true +} + +func (r RemapOptions) RemoveOption(name string) bool { + delete(r, name) + return true +} + +func (r RemapOptions) ValueByNameAsBool(name string) (bool, error) { + v, ok := r[name].(bool) + if !ok { + return false, fmt.Errorf("value for %q is not a boolean: %v", name, r[name]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsString(name string) (string, error) { + v, ok := r[name].(string) + if !ok { + return "", fmt.Errorf("value for %q is not a string: %v", name, r[name]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsStringMapString(name string) (map[string]string, error) { + v, ok := r[name].(map[interface{}]interface{}) + if !ok { + return nil, fmt.Errorf("value for %q is of unexpected type, \"%T\". expected a map", name, r[name]) + } + + newv := make(map[string]string) + for k, v := range v { + newv[k.(string)] = v.(string) + } + return newv, nil +} + +func (r RemapOptions) ValueByNameAsInt(key string) (int, error) { + v, ok := r[key].(int) + if !ok { + return 0, fmt.Errorf("value for %q is not a integer: %v", key, r[key]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsSlice(key string) ([]string, error) { + switch t := r[key].(type) { + case []interface{}: + vs := make([]string, len(t)) + for i, d := range t { + var val string + switch d := d.(type) { + case string: + val = d + case int: + val = strconv.Itoa(d) + } + + vs[i] = val + } + return vs, nil + case []string: + vs := make([]string, len(t)) + copy(vs, t) + + return vs, nil + } + return []string{}, fmt.Errorf("field %s is unhandled type: %T", key, r[key]) +} + +type remapOptionsJSON map[string]interface{} + +func (r RemapOptions) makeRemapOptionsJSONStruct() (remapOptionsJSON, error) { + rmo := remapOptionsJSON{} + for k, v := range r { + im, ok := r[k].(map[interface{}]interface{}) + if ok { + m := make(map[string]interface{}) + for ik, iv := range im { + sik, ok := ik.(string) + if !ok { + return nil, fmt.Errorf("non-string for key in map %+v", im) + } + m[sik] = iv + } + rmo[k] = m + continue + } + rmo[k] = v + } + return rmo, nil +} + +func (r RemapOptions) MarshalJSON() ([]byte, error) { + roj, err := r.makeRemapOptionsJSONStruct() + if err != nil { + return nil, err + } + return json.Marshal(roj) +} + +func (r RemapOptions) ValueByNameAsStringMapInterface(name string) (map[string]interface{}, error) { + v, ok := r[name].(map[interface{}]interface{}) + if !ok { + return nil, fmt.Errorf("value is of unexpected type. name=%s val=%v\n r=%+v\nt=%T", name, r[name], r, v) + } + + newv := make(map[string]interface{}) + for k, v := range v { + if v != nil { + newv[k.(string)] = v.(interface{}) + } + } + return newv, nil +} diff --git a/tools/voluspa/remap_options_test.go b/tools/voluspa/remap_options_test.go new file mode 100644 index 00000000000..faccbaf0d13 --- /dev/null +++ b/tools/voluspa/remap_options_test.go @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "testing" + +func TestRemapOptions_ValueByNameAsBool(t *testing.T) { + ro := &RemapOptions{} + for _, v := range []bool{true, false} { + (*ro)["key"] = v + + out, err := ro.ValueByNameAsBool("key") + if err != nil || out != v { + t.Fatalf("Expected value '%t', got '%t'", v, out) + } + } + + _, err := ro.ValueByNameAsBool("non-existent-key") + if err == nil { + t.Fatalf("Expected err getting value by non-existent key") + } + + (*ro)["key2"] = "stringval" + _, err = ro.ValueByNameAsBool("key2") + if err == nil { + t.Fatalf("Expected err getting boolean value for key") + } +} + +func TestRemapOptions_ValueByNameAsString(t *testing.T) { + ro := &RemapOptions{} + (*ro)["key"] = "value" + + out, err := ro.ValueByNameAsString("key") + if err != nil || out != "value" { + t.Fatalf("Expected value '%s', got '%s'", (*ro)["key"], out) + } + + _, err = ro.ValueByNameAsString("non-existent-key") + if err == nil { + t.Fatalf("Expected err getting value by non-existent key") + } + + (*ro)["key2"] = false + _, err = ro.ValueByNameAsString("key2") + if err == nil { + t.Fatalf("Expected err getting string value for key") + } +} diff --git a/tools/voluspa/sample.conf-template b/tools/voluspa/sample.conf-template new file mode 100644 index 00000000000..7685dd4bfba --- /dev/null +++ b/tools/voluspa/sample.conf-template @@ -0,0 +1,178 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +schema_version: '1.0' +owner: Owner Name +reference: reference number here +rcpt: MyReceipt +parent_child: true # use parent/child hierarchy +qps: 2 # Approximate queries per second to order rules +role: roles_edge_child # If this should be applied to specific salt role. +lifecycle: onboarding|live|retired #live: in production; onboarding: various things may not yet be working; retired: No longer active (no need for new ssl certs, etc); +cdn: [edge, internal] +ssl_cert_names: # a list of SSL cert filenames. used to populate ssl_multicert.config + - /secret/mysecret1.domain.com + - /secret/mysecret2.domain.com + +mappings: + - alias: + - mydomain1.com # can have multiple. If no scheme, uses from schemes: + origin: http://myorigin.domain.com + schemes: [http, https] # applied to alias and origin (if not specified) + rule: default #not needed if "default" + +rules: + default: # named set of rules to apply to this target/origin + parent_child: false # child remap goes direct to origin. (Overrides metadata's parent_child for this ruleset) + strip_query: true # strips the query string completely -- the origin won't see it. + force_ttl: 7d # Always set Cache-Control: max-age: $time; overwrites if origin had set. + default_ttl: 7d # If the Origin doesn't set a Cache-Control, set it to Cache-Control: max-age: $time + negative_ttl: 10s # amount of time to cache http errors (404, 502, 504, etc) + transaction_timeout: 10m # Sets transaction timeouts (CDN default is 30s) + content_type_forge: # Change the content type based on a series of regexes (Need this line too) + # 'regex': content-type. urls matching regex will have have the Content-type header set to content-type + '.*\.json$': application/json + '.*\.css$': text/css + edge_compression: # if the origin does not surpport compression, CDN should do the compression instead + enabled: true + algorithms: [gzip] # only gzip for now + static_origin_compressible_content_type: + - application/json #compress only these mime types -- not the defaults + header_rewrite: |- # raw values to put into header_rewrite configration + cond %{SEND_REQUEST_HDR_HOOK} + set-header Authorization "Basic 123412341234" + deny_methods: [ CONNECT, POST, PUT, DELETE ] #Reject these HTTP verbs + priority: background | foreground | streamingaudio | streamingvideo # QOS priority (DSCP bits) + set_header: 'X-Client-Protocol "%"'# Set this header; overwrites if origin had set. + add_header: 'X-Client-Protocol "%"' # add this header; + remove_header: "X-Client-Protocol" # remove this header; + add_header_origin: 'X-Client-Protocol "%"' # Send this header to the origin + set_header_origin: 'X-Client-Protocol "%"' # Send this header to the origin; overwrites if client had set. + proxy_cache_control: "MyProxyHeader" # Use this header's value for Cache-Control. + disable_cache: true # turn off the cache for this rule + regex_remap: "^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2" # use regex to match the inbound path (only) and substition for origin. Can add optional http status code (like @status=301). Can also use "|-" format + conf_remap: |- # raw values to override specific ATS settings. + CONFIG proxy.config.http.connect_attempts_timeout INT 600 + CONFIG proxy.config.http.negative_caching_enabled FLOAT .050 + allow_ip: [10.0.0.0-10.0.255.255, 10.13.0.0-10.13.255.255] # Only accept requests from these IP ranges + log_cookie: # log specific cookies for this property. Logger-cookie-Name1 will become the key name in logger + Logger-cookie-Name1: NameOfCookie1 + Logger-cookie-Name2: NameOfCookie2 + log_header: # log specific header for this property. Logger-header-Name1 will become the key name in logger + Logger-header-Name1: NameOfHeader1 + Logger-header-Name2: NameOfHeader2 + log_type: private|public # Private logs no personal info (no client IP, url, referalurl ...); Public is default + cachekey: #modify the cache key + include_params: ["param1", "param2"] # only use these query parameters in the cachekey + exclude_params: ["param1", "param2"] # ignore these query parameters in the cachekey + sort_query: true # sort the query parameters for the cachekey + remove_all_params: true # Don't use the query string as part of the cachekey. + include_headers: ["Origin", "Content-Type"] # include specific headers in the cachekey + include_cookies: ["Cookie1", "Cookie2"] # include specific cookies in the cachekey + static_prefix: SomeString # specified value will be added to the cachekey + regex_replace_path: '/.*/filename.html/' # replace cachekey URI path with regex capture + cache_promote: # cache promotion plugin options -- only cache the popular objects + policy: lru | chance + sample: 25 + lru_hits: 10 + lru_buckets: 10000 + failover: # escalate plugin options -- when object is unavailable, try another origin + domain: domain.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: alias|origin # What hostname should we use when contacting the failover? (alias or origin) +# Flags to disable options automatically enabled: + receipts: + enabled: false # don't set a rcpt field in logs + name: override_name # override receipt name on ruleset + propstats: false # don't send stats to epic for this remap rule + origin_host_header: alias| origin # origin=Send the origin's Host: header to the origin. alias=send the inbound Host: header to origin (default)" +# Property Specific options: + s3: #Use Amazon S3 authentication + path: "path/to/config" # Origin is s3; eg certs/s3.config + version: 2|4 # which version of Amazon AWS signature algorithm to use; 2 or 4 with default of 2 + virtual_host: true # populates Host: header with S3 virtual host + authproxy: range | head # Send Range/Head request to secondary origin to authenticate request. Will need 2nd remap rule to catch these requests. +# Video background fetch plugin options + video_background_fetch: + fetch_count: 10 + fetch_max: 1000 + api_header: header1 + replace_host: host1 + name_space: ns1 + metrics_prefix: prefix1 + exact_match: true + log_name: log1 + frontend: + fetch_path_pattern: '/(.*-)(\d+)(.*)/$1{$2+2}$3/' + fetch_policy: lru:10000000 + backend: + fetch_policy: simple + +tests: +# Status code checks + health: # arbitrary name for test + urls: # urls to test + - http://mydomain1.com/show/health + - https://mydomain1.com/show/health + success: # rules defining a successful test + status_code: 200 + +# Header checks + parent_cache_key_and_etag: + description: run only on hosts in trafficserver_parent salt role + role: roles_uat # specify role of host to run test against + insecure: true # Default False. Ignore ssl errors thus allowing insecure connection + range: 0-499 # specifies the first 500 bytes + purge_before: true # Default False. Purge url before running a test + headers: # request headers + X-Debug: X-Cache-Key + host_types: # valid values: parent|child + - parent + urls: + - http://mydomain1.com/test.txt + success: + status_code: 200 + headers: + X-Cache-Key: /mydomain1.com:80/test.txt + ETag: \"d36f8f9425c4a8000ad9c4a97185aca5\" +# domain names for SSL tests + ssl_tests: + domain_names: + - working.domain.com + + cert_serial_number_check: + description: devimages has a pinned cert + type: ssl + domain_names: + - devimages-cdn.domain.com + success: + serial_numbers: + - ABC123 + +# log settings +receiptsd: # configure receipts (logs) sent + sampling: + posters: + global: 10% # only send 10% of logs to main instance + adcdownload: 100% # also send 100% of logs to the download instance + ignored_extensions: # don't send any logs for files with these extensions + - .dist + - .dist.gz + - .pkm + +# vim: set et sw=2 ts=2 ft=yaml : diff --git a/tools/voluspa/schema_v1.json b/tools/voluspa/schema_v1.json new file mode 100644 index 00000000000..2abbcc0235d --- /dev/null +++ b/tools/voluspa/schema_v1.json @@ -0,0 +1,1126 @@ +{ + "$schema": "http://json-schema.org/schema#", + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "rcpt", + "schema_version", + "owner" + ], + "definitions": { + "schemes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "http", + "https" + ], + "minItems": 1 + }, + "description": "http|https applied to alias and origin (if not specified there)" + }, + "role": { + "type": "string", + "enum": [ + "roles_edge_child" + ], + "description": "If this should be applied to specific set of machines." + }, + "parent_child": { + "type": "boolean", + "description": "Use parent/child hierarchy" + }, + "alias": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Inbound URI. Can have multiple. If there is no scheme, uses from schemes:. Any extra path and query parameters are passed along at end of origin." + }, + "origin": { + "type": "string", + "description": "Outbound URL. If there is no scheme, uses it from schemes:. Extra path and query parameters after aliases are passed along at the end." + }, + "propstats": { + "type": "boolean", + "description": "Keep metrics on this property or not." + }, + "fetch_policy": { + "type": "string", + "description": "simple or lru (with lru size). eg lru:10000000" + }, + "mapping": { + "type": "object", + "additionalProperties": false, + "required": [ + "alias", + "origin" + ], + "items": { + "type": "object" + }, + "properties": { + "comment": { + "type": "string" + }, + "alias": { + "$ref": "#/definitions/alias" + }, + "origin":{ + "$ref": "#/definitions/origin" + }, + "group": { + "type": "string", + "enum": [ + "group1", + "group2", + "group3", + "group4" + ], + "description": "Used to override resolver." + }, + "rule": { + "type": "string", + "description": "Which ruleset to apply to this mapping set" + }, + "parent_dest_domain": { + "type": "string", + "description": "May need to tell Voluspa what domain to use when finding the parent. Note: this option is being deprecated! Use parent_override instead!" + }, + "regex_map": { + "type": "boolean", + "description": "if there is a regular expression in the hostname of this uri." + }, + "role": { + "$ref": "#/definitions/role" + }, + "schemes": { + "$ref": "#/definitions/schemes" + }, + "parent_override": { + "description": "List of overrides for parent.config.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "role": { + "$ref": "#/definitions/role" + }, + "dest_domain": { + "type": "string", + "description": "What destination domain when finding the parent." + }, + "strategy": { + "type": "string", + "enum": [ + "true_round_robin", + "strict_round_robin", + "no_round_robin", + "consistent_hash", + "latched" + ], + "default": "consistent_hash", + "description": "Routing strategy." + }, + "ignore_querystring": { + "type": "boolean", + "default": false, + "description": "When set to true the query string will not be included in the hash to select the parent." + }, + "http_port": { + "type": "integer", + "description": "Port to use for http scheme." + }, + "https_port": { + "type": "integer", + "description": "Port to use for https scheme." + }, + "go_direct": { + "type":"boolean", + "default": false, + "description": "When set to true, requests bypass parent hierarchies and go directly to the origin server." + }, + "primary_list": { + "type":"array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "string", + "format": "hostname" + }, + { + "type": "string", + "format":"ipv4" + }, + { + "type": "string", + "format":"ipv6" + } + ] + }, + "description": "Array of hostnames (or IP adderesses) of the primary parents." + }, + "secondary_list": { + "type":"array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "string", + "format": "hostname" + }, + { + "type": "string", + "format":"ipv4" + }, + { + "type": "string", + "format":"ipv6" + } + ] + }, + "description": "Array of hostnames (or IP adderesses) of the secondary parents." + } + } + } + } + } + } + }, + "properties": { + "comment": { + "type": "string" + }, + "schema_version": { + "oneOf": [ + { "type": "string", "pattern": "^1(\\.0)?$" }, + { "type": "integer", "minimum": 1, "maximum": 1 } + ], + "description": "Declares the version of the Voluspa schema this configuration is defined in terms of." + }, + "alias": { + "$ref": "#/definitions/alias" + }, + "cdn": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "edge", + "internal" + ] + }, + "description": "Which machines to target." + }, + "lifecycle": { + "type": "string", + "enum": [ + "onboarding", + "live", + "retired" + ], + "description": "live: in production; onboarding: various things may not yet be working; retired: No longer active (no need for new ssl certs, etc);" + }, + "org": { + "type": "string", + "enum": [ + "IST", + "InternetServices", + "SoftwareEngineering" + ], + "description": "Used for billing the correct org. Metadata only, not used for cache configuration." + }, + "owner": { + "type": "string", + "description": "What group should we email/page if there are problems?" + }, + "propstats": { + "$ref": "#/definitions/propstats" + }, + "rcpt": { + "type": "string", + "description": "A unique property name." + }, + "reference": { + "type": "string", + "description": "Where to find more information about this property, such as a URL." + }, + "role": { + "$ref": "#/definitions/role" + }, + "rules": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "set_header": { + "type": "string", + "description": "Set this header; overwrites if origin had set." + }, + "cachekey": { + "type": "object", + "additionalProperties": false, + "items": { + "type": "string" + }, + "description": "Defines how the cachekey is composed. Changing any of these parameters is likely to effectively clear the cache.", + "properties": { + "comment": { + "type": "string" + }, + "remove_all_params": { + "type": "boolean", + "description": "Don't use the query string as part of the cachekey or parent_selection." + }, + "include_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Include specific headers from the request in the cachekey." + }, + "regex_replace_path": { + "type": "string", + "examples": [ + "/.*/probe-info-ok.html/" + ], + "description": "Replace cachekey URI path with regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + }, + "regex_replace_path_uri": { + "type": "string", + "examples": [ + ["/http[s]?:\\/\\/[^\\/]*(.*\\.html|.*).*/$1/"] + ], + "description": "Replace cachekey full URI with regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + }, + "include_params": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + ["queryparam1", "queryparam2"] + ], + "description": "Only use these query parameters in the cachekey." + }, + "static_prefix": { + "type": "string", + "description": "Specified value will start the cachekey." + }, + "include_cookies": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Include these specific cookies in the cachekey." + }, + "exclude_params": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "examples": [ + ["queryparam1", "queryparam2"] + ], + "description": "Ignore these query parameters in the cachekey." + }, + "sort_query": { + "type": "boolean", + "description": "Sort the query parameters for the cachekey." + }, + "parent_selection": { + "type": "string", + "enum": [ + "ignore_query" + ], + "description": "ignore_query: Ignore the query string when choosing the parent. Note: this option is being deprecated! Use parent_override in stead!" + }, + "capture_header": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "examples": [ + [ "Authorization:/AWS\\s([^:]+).*/clientID:$1/", "Authorization:/BYT\\s([^,]+).*/clientID:$1/" ] + ], + "description": "Add specific parts of headers to the cachekey via a regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + } + } + }, + "allow_ip": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Only accept requests from these IPs and IP ranges.", + "examples": [ + [ "127.0.0.1", "::1" ], + [ "10.0.0.0-10.255.255.255", "127.0.0.1", "::1" ] + ] + }, + "allow_ip_parent": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Only accept requests from these IPs and IP ranges (applies to parent hosts).", + "examples": [ + [ "10.0.0.0-10.255.255.255", "10.127.0.0-10.127.255.255" ] + ] + + }, + "header_rewrite": { + "type": "string", + "description": "Raw text to put into ATS header_rewrite configuration." + }, + "add_header": { + "type": "string", + "description": "Add this header to the response.", + "examples": [ "Strict-Transport-Security \"max-age=31536000; includeSubDomains\"" ] + }, + "add_header_origin": { + "type": "string", + "description": "Add this header to the origin request.", + "examples": [ "X-CLIENT-IP \"%\"" ] + }, + "origin_host_header": { + "type": "string", + "enum": [ + "origin", + "alias" + ], + "description": "What Host: to send to the origin. origin=Send the origin's Host; alias=send the inbound Host: (default)" + }, + "content_type_forge": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string", + "description": "'URL regex': content-type. URLs matching regex will have have the response Content-Type set to content-type" + } + }, + "description": "Change the Content-Type based on a series of URL path regexes", + "examples": [ "\".*\\.json$\": \"application/json\"" ] + }, + "conf_remap": { + "type": "string", + "description": "Raw text to put into ATS conf_remap configuration." + }, + "cache_promote": { + "type": "object", + "additionalProperties": false, + "description": "Configuration for the cache_promote plugin. Selectively cache objects. For really large working sets.", + "properties": { + "comment": { + "type": "string" + }, + "policy": { + "type": "string", + "enum": [ + "lru", + "chance" + ], + "description": "`chance`: randomly choose to cache the item (based on sample number); `lru`: Track most recently used URLs, and cache those with more than lru_hits." + }, + "sample": { + "type": "number", + "minimum": 1, + "maximum": 100, + "description": "The percentage sampling rate for the request to be considered, for either policy." + }, + "lru_hits": { + "type": "integer", + "description": "In `lru` policy, how often the object is fetched before promotion." + }, + "lru_buckets": { + "type": "integer", + "description": "In `lru` policy, how many objects are tracked." + } + } + }, + "propstats": { + "$ref": "#/definitions/propstats" + }, + "regex_remap": { + "type": "string", + "description": "Use regex to match the inbound path (only) and substition for origin. Can add optional http status code (like @status=301). Can also use '|-' format", + "examples": [ "^/([^/]+)/([^\\?]*)\\??.* https://$1.s3.amazonaws.com/$2" ] + }, + "negative_ttl": { + "type": "string", + "description": "Amount of time to cache http errors (404, 502, 504, etc). Default 10s.", + "examples": [ "10s" ] + }, + "priority": { + "type": "string", + "enum": [ + "background", + "foreground", + "streamingaudio", + "streamingvideo" + ], + "description": "QOS priority (DSCP bits)" + }, + "default_ttl": { + "type": "string", + "description": "If the Origin doesn't set a Cache-Control, set it to this value. A number followed by a letter (like 17s, 5m, 3h, 2d, 1w) is a shortcut for 'max-age=N, public'", + "examples": [ "10s", "60m", "12h", "no-cache, no-store, must-revalidate, stale-while-revalidate=30" ], + "pattern": "^([0-9]+[smhdw])$|^((no-cache|no-store|no-transform|must-revalidate|public|private|proxy-revalidate|immutable|(max-age=|s-maxage=|stale-while-revalidate=|stale-if-error=)[0-9]+) *(, *|$))+$" + }, + "set_header_origin": { + "type": "string", + "description": "Send this header to the origin; overwrites if client had set." + }, + "authproxy": { + "type": "string", + "emum": [ + "range", + "head" + ], + "description": "Send Range/Head request to secondary origin to authenticate request. Will need 2nd remap rule to catch these requests." + }, + "parent_child": { + "$ref": "#/definitions/parent_child" + }, + "parent": { + "type": "boolean" + }, + "child": { + "type": "boolean" + }, + "redirect": { + "type": "object", + "additionalProperties": false, + "description": "Configure the Regex Remap Plugin. See https://docs.trafficserver.apache.org/en/7.1.x/admin-guide/plugins/regex_remap.en.html", + "properties": { + "comment": { + "type": "string" + }, + "src": { + "type": "string", + "description": "regex src pattern, defaults to `(.*)`" + }, + "url": { + "type": "string", + "description": "regex URL to redirect to, may include substitution variables" + }, + "http_code": { + "type": "integer", + "minimum": 100, + "maximum": 999, + "description": "HTTP status code to return, defaults to 302" + } + }, + "examples": [ + { "url": "https://example.com$0", "http_code": 410 }, + { "src": "http://old.(.*).z.com", "url": "http://new.$1.z.com" } + ] + }, + "video_background_fetch": { + "type": "object", + "description": "Prefetch sequential URLs for improved caching. Usually for HLS video.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "log_name": { + "type": "string", + "description": "Rcpt to log these requests." + }, + "fetch_count": { + "type": "integer", + "description": "How many URLs ahead to fetch." + }, + "replace_host": { + "type": "string", + "description": "What Host: header to use in pre-fetch." + }, + "exact_match": { + "type": "boolean" + }, + "api_header": { + "type": "string", + "description": "What header to use to signal this is a prefetch request." + }, + "name_space": { + "type": "string", + "description": "Name Space to segregate LRU count." + }, + "metrics_prefix": { + "type": "string", + "description": "Prefix for prefetch metrics." + }, + "fetch_policy": { + "$ref": "#/definitions/fetch_policy" + }, + "frontend": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "fetch_policy": { + "$ref": "#/definitions/fetch_policy" + }, + "fetch_path_pattern": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Regex to find the portion of the url that can be incremented." + } + } + }, + "fetch_max": { + "type": "integer" + }, + "backend": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "fetch_policy": { + "type": "string", + "description": "simple or lru (with lru size). eg lru:10000000", + "pattern": "^(simple|lru:[0-9]+)$" + } + } + } + } + }, + "log_header": { + "type": "object", + "properties": {}, + "description": "Log specific request headers with the specified keyword names.", + "examples": [ { "name-in-log": "name-of-header", "cacheControlValue": "Cache-Control", "playbackSessionID": "X-PlayBack-Session-Id" } ] + }, + "proxy_cache_control": { + "type": "string", + "description": "Use this header's value for Cache-Control.", + "examples": [ "Proxy-Cache-Control" ] + }, + "failover": { + "type": "object", + "description": "When object is unavailable, try a different origin.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "domain": { + "type": "string", + "description": "When object is unavailable, try what origin?" + }, + "status_codes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 100, + "maximum": 999 + }, + "description": "What HTTP status codes indicate we should try another origin?" + }, + "host_header": { + "type": "string", + "enum": [ + "origin", + "alias" + ], + "description": "What hostname should we use when contacting the failover? (alias or origin)" + } + } + }, + "remove_header": { + "type": "string", + "description": "Remove this header." + }, + "remove_header_origin": { + "type": "string", + "description": "Remove this header in origin request." + }, + "log_cookie": { + "type": "object", + "properties": {}, + "description": "Log specific request cookies with the specified keyword names.", + "examples": [ { "name-in-log": "name-of-cookie", "session": "s", "auth": "ADCDownloadAuth" } ] + }, + "force_ttl": { + "type": "string", + "description": "Always set Cache-Control to this value; overwrites if origin had set. A number followed by a letter (like 17s, 5m, 3h, 2d, 1w) is a shortcut for 'max-age=N, public'", + "pattern": "^([0-9]+[smhdw])$|^((no-cache|no-store|no-transform|must-revalidate|public|private|proxy-revalidate|immutable|(max-age=|s-maxage=|stale-while-revalidate=|stale-if-error=)[0-9]+) *(, *|$))+$" + }, + "receipts": { + "type": "object", + "description": "Control receipt (eg log) production.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "enabled": { + "type": "boolean", + "description": "Should we send receipts (eg logs) for this property?" + }, + "name": { + "type": "string", + "description": "Should we use a different rcpt name for this ruleset?" + } + } + }, + "allow_methods": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "GET", + "HEAD", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PURGE" + ] + }, + "description": "Accept only these HTTP verbs" + }, + "deny_methods": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "GET", + "HEAD", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH" + ] + }, + "description": "Reject these HTTP verbs" + }, + "strip_query": { + "type": "boolean", + "description": "Strips the query string completely -- the origin won't see it." + }, + "transaction_timeout": { + "type": "string", + "description": "Sets transaction timeouts (CDN default is 60s)" + }, + "s3": { + "type": "object", + "additionalProperties": false, + "description": "Use Amazon S3 authentication when contacting origin.", + "required": [ + "path" + ], + "properties": { + "comment": { + "type": "string" + }, + "path": { + "type": "string", + "description": "Path to the s3 configuration containing the shared secret." + }, + "virtual_host": { + "type": "boolean", + "description": "Populate Host: header with S3 virtual host." + }, + "version": { + "type": "integer", + "enum": [ + 2, + 4 + ], + "description": "Which version of Amazon AWS signature algorithm to use." + }, + "v4_include_headers": { + "type": "array", + "items": { + "type": "string" + } + }, + "v4_exclude_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Don't use these headers in the authorization signature.", + "examples": [ { "v4_exclude_headers": ["x-forwarded-for","forwarded","via","authorization"] } ] + }, + "region_map": { + "type": "string", + "description": "Path to a region map config file." + } + } + }, + "edge_compression": { + "type": "object", + "additionalProperties": false, + "description": "", + "properties": { + "comment": { + "type": "string" + }, + "enabled": { + "type": "boolean", + "description": "Should we enable compression?" + }, + "algorithms": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "gzip", + "br" + ], + "description": "Specify compression algorithm(s) to use." + } + }, + "static_origin_compressible_content_type": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Compression for specified mime-types.", + "examples": [ [ "application/x-mpegURL" ] ] + }, + "flush": { + "type": "boolean" + }, + "remove_accept_encoding": { + "type": "boolean", + "description": "Removes Accept-Encoding header in origin requests." + }, + "minimum_content_length": { + "type": "integer", + "description": "Minimum content length for compression to be enabled (in bytes)" + } + } + }, + "alt_svc": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "authority": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "ma": { + "type": "integer" + }, + "health": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "disable_cache": { + "type": "boolean", + "description": "Turn off the cache for this rule." + }, + "log_type": { + "type": "string", + "enum": [ + "private", + "public" + ], + "description": "Private if we should not log privacy sensitive information, like the URL, referral URL, client IP, etc. Default is public." + }, + "access_approval": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "raw": { + "type": "string" + } + } + }, + "storage_volume": { + "type": "string", + "description": "Selects which volume will be used for caching (disk_volume or ramdisk_volume); when this is omitted the default used is disk_volume.", + "enum": [ + "disk_volume", + "ramdisk_volume" + ] + }, + "lua": { + "type": "string", + "description": "Raw text to run as Lua remap script." + }, + "echo_cors": { + "type": "string", + "description": "Header to echo back as Access-Control-Allow-Origin in response." + }, + "cache_version": { + "type": "integer", + "description": "To clear the cache for this rule, increment the integer (or add this rule with value 1)." + } + } + } + } + }, + "schemes": { + "$ref": "#/definitions/schemes" + }, + "mappings": { + "type": "array", + "items": { + "$ref": "#/definitions/mapping" + } + }, + "explicit": { + "type": "array", + "items": { + "$ref": "#/definitions/mapping" + }, + "uniqueItems": true + }, + "ha_proxy": { + "type": "array", + "additionalProperties": false, + "items": { + "type": "object", + "required": [ + "port", + "rips" + ] + }, + "properties": { + "comment": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "rips": { + "type": "array", + "items": { + "type": "object", + "required": [ + "ip", + "port" + ] + }, + "properties": { + "ip": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + } + } + }, + "parent_child": { + "$ref": "#/definitions/parent_child" + }, + "qps": { + "type": "integer", + "minimum": 1, + "description": "Approximate queries per second, used to order rules for efficiency." + }, + "receiptsd": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "ignored_extensions": { + "type": "array", + "items": { + "type": "string" + } + }, + "sampling": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "posters": { + "type": "object", + "description": "Name of the program posting the logs. Current values are Global (for iTunes splunk), adcdownload, and adcdownload-uat (for posting to WWDR's prod/uat)", + "examples": [ "global: 1%" ], + "patternProperties": { + ".*": { + "type": "string", + "pattern": "^\\d+%$" + } + } + } + }, + "additionalProperties": false + } + } + }, + "ssl_cert_names": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^([\\w\\-\\.,:/]+)$" + }, + "description": "A list of SSL cert filenames. used to populate ssl_multicert.config." + }, + "tests": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "description": { + "type": [ + "integer", + "string" + ] + }, + "domain_names": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Domain names for SSL tests" + }, + "headers": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "description": "Headers to go into the request." + }, + "host_types": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "parent", + "child" + ] + }, + "description": "Which kind of hosts should the check run on?" + }, + "insecure": { + "type": "boolean", + "description": "If curl should ignore certificate warnings." + }, + "purge_before": { + "type": "boolean", + "description": "If we should run a PURGE command before the test." + }, + "range": { + "type": "string", + "description": "Use a range request -- eg, 0-499 will ask for the 1st 500 bytes." + }, + "role": { + "type": "string", + "description": "Specify role of host to run test against." + }, + "success": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "headers": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "description": "Headers and their values that should be returned" + }, + "serial_numbers": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string" + }, + "description": "Serial Number of cert or Intermediate." + }, + "status_code": { + "type": "integer", + "minimum": 100, + "maximum": 999, + "description": "HTTP status code expected in the response." + } + } + }, + "urls": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "URLs to test" + } + } + } + } + } + } +} diff --git a/tools/voluspa/specs/voluspa.spec b/tools/voluspa/specs/voluspa.spec new file mode 100644 index 00000000000..75c85f6a24a --- /dev/null +++ b/tools/voluspa/specs/voluspa.spec @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +%{!?_release: %define _release 0} +%{!?_version: %define _version UNKNOWN} +%{!?commit0: %define commit0 UNKNOWN} + +Name: voluspa +Version: %{_version} +Release: 1 +Summary: Voluspa +License: Apache Software License 2.0 (AL2) +Group: Applications/System +URL: https://github.com/apache/trafficserver/tools/voluspa + +%description +CDN -> ATS Configuration Tool + +Version: %{commit0} + +%prep + +%build + +%check + +%install +rm -rf $RPM_BUILD_ROOT +mkdir -p $RPM_BUILD_ROOT/usr/bin +cp -p %{_topdir}/dist/linux/voluspa $RPM_BUILD_ROOT/usr/bin + +%files +%defattr(-, root, root, -) +/usr/bin/voluspa + +%clean + +%changelog diff --git a/tools/voluspa/ssl_multicert.go b/tools/voluspa/ssl_multicert.go new file mode 100644 index 00000000000..cb4e491244d --- /dev/null +++ b/tools/voluspa/ssl_multicert.go @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +var ( + // defaultCertNames is a set of cert names to be specified as the default + defaultCertNames = map[string]interface{}{ + "/secret/edgecdn:images.domain.com": nil, + "/secret/images.domain.com.rsa": nil, + "/secret/images.domain.com.rsa,/secret/images.domain.com.ecdsa": nil, + } + + SSLMulticertDefaultFilename = "ssl_multicert.config_default" + SSLMulticertFilename = "ssl_multicert.config" +) + +type sslCert struct { + filename string + comment string +} + +type SSLMulticertConfigurator struct { + options *Options +} + +func newSSLMulticertConfigurator(options *Options) *SSLMulticertConfigurator { + return &SSLMulticertConfigurator{options: options} +} + +func initCertsMap(parsedConfigs []*CustomerConfig) map[string]map[string]sslCert { + // a map of roles to an array of sslCerts + certs := make(map[string]map[string]sslCert) + + sort.Stable(sort.Reverse(byQPSAndName(parsedConfigs))) + + for _, parsedConfig := range parsedConfigs { + if len(parsedConfig.sslCertNames) == 0 { + continue + } + if parsedConfig.lifecycle == Retired { + continue + } + + role := parsedConfig.role + + if _, exists := certs[role]; !exists { + certs[role] = make(map[string]sslCert) + } + + sort.Strings(parsedConfig.sslCertNames) + + for _, sslCertName := range parsedConfig.sslCertNames { + certs[role][sslCertName] = sslCert{ + filename: sslCertName, + comment: parsedConfig.reference, + } + } + } + + return certs +} + +func (s *SSLMulticertConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return s.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate ssl_multicert.config for each CDN + grouped := groupCustomerConfigsByCDN(parsedConfigs) + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := s.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil +} + +func (s *SSLMulticertConfigurator) get(configs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + certs := initCertsMap(configs) + if len(certs) == 0 { + return nil, nil + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + var roles []string + for role := range certs { + roles = append(roles, role) + } + sort.Strings(roles) + + for _, role := range roles { + err := s.expandConfigTemplate(certs, role, &buf) + if err != nil { + return nil, err + } + } + + fileName := SSLMulticertDefaultFilename + if len(cdn) > 0 && (cdn != DefaultCDN && cdn != NoCDN) { + fileName = fmt.Sprintf("ssl_multicert.config_%s", cdn) + } + + return []ManagedFile{ + NewManagedFile(fileName, SSLMulticertFilename, cdn, "", "", &buf, UnknownLocation), + }, nil +} + +func (s *SSLMulticertConfigurator) startRoleGuard(role string) string { + if len(role) == 0 { + if s.options.PromoteRolesToCDN { + return "" + } + + return "{% if not salt.pillar.get('roles_uat') %}\n\n" + } + + roleName := role + if !strings.HasPrefix(role, "roles_") { + roleName = fmt.Sprintf("roles_%s", role) + } + return fmt.Sprintf("{%% if salt.pillar.get('%s') %%}\n\n", roleName) +} + +func (s *SSLMulticertConfigurator) endRoleGuard(role string) string { + if len(role) == 0 && s.options.PromoteRolesToCDN { + return "" + } + return "{% endif %}\n\n" +} + +func (s *SSLMulticertConfigurator) expandConfigTemplate(certsMap map[string]map[string]sslCert, role string, buf *bytes.Buffer) error { + certs, ok := certsMap[role] + if !ok { + return fmt.Errorf("role %q not found", role) + } + + if len(certs) == 0 { + return nil + } + + var certFilenames []string + for certFilename := range certs { + certFilenames = append(certFilenames, certFilename) + } + sort.Strings(certFilenames) + + buf.WriteString(s.startRoleGuard(role)) + + var sawDefaultCert bool + for _, certFilename := range certFilenames { + cert := certs[certFilename] + if len(cert.comment) > 0 { + buf.WriteString(fmt.Sprintf("# %s\n", cert.comment)) + } + if _, found := defaultCertNames[certFilename]; found { + if sawDefaultCert { + return fmt.Errorf("more than one default cert found for role") + } + + buf.WriteString("dest_ip=* ") + sawDefaultCert = true + } + + buf.WriteString(fmt.Sprintf("ssl_cert_name=%s\n\n", certFilename)) + } + + buf.WriteString(s.endRoleGuard(role)) + + return nil +} diff --git a/tools/voluspa/tests/.gitignore b/tools/voluspa/tests/.gitignore new file mode 100644 index 00000000000..b98b0e32b34 --- /dev/null +++ b/tools/voluspa/tests/.gitignore @@ -0,0 +1,5 @@ +*/test +*/*/test +/out +*/out +*.out diff --git a/tools/voluspa/tests/Makefile b/tools/voluspa/tests/Makefile new file mode 100644 index 00000000000..dbe8b164d72 --- /dev/null +++ b/tools/voluspa/tests/Makefile @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test: regress other custom + +clean: + @rm -f */system.json + @rm -f */profile_*.out + +COVERAGE_OPTIONS=-test.run ^TestRunMain$$ -test.coverprofile=system.out + +coverage: + cd regress && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh + cd other && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh + if [ -x custom ]; then \ + cd custom && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh; \ + fi; + +regress: + cd regress && ./test.sh + +other: + cd other && ./test.sh + +custom: + if [ -x custom ]; then \ + cd custom && ./test.sh; \ + fi; + +regen_baseline: + cd regress && ./regen_baseline.sh + cd other && ./regen_baseline.sh + if [ -x custom ]; then \ + cd custom && ./regen_baseline.sh; \ + fi; + +.PHONY: regress other custom +.NOTPARALLEL: diff --git a/tools/voluspa/tests/README.md b/tools/voluspa/tests/README.md new file mode 100644 index 00000000000..98c5420ac46 --- /dev/null +++ b/tools/voluspa/tests/README.md @@ -0,0 +1,16 @@ +# Tests + +This directory contains a series of sanity tests. These are simple tests comparing the output of a known good build/configuration to the latest locally built version of **voluspa**. + +The script "test.sh" will use the current build of the tool to regenerate the remap configs for a given yaml file; and do a diff between base/ and the new output. Generally, there should be no changes between runs. If there are expected changes, the baselines would need to be regenerated. The configurations tested range from very simple to something exercising all current configuration options. + +## Current Tests + +* simple/ + * Basic functionality +* all/ + * Every current configuration option exercising combinations of config types (normal and explicit) +* order/ + * Covers both ordering of properties by QPS and grouping via "role" +* parentchild/ + * A baseline of current parent/child config generation diff --git a/tools/voluspa/tests/other/all_bad/.gitignore b/tools/voluspa/tests/other/all_bad/.gitignore new file mode 100644 index 00000000000..9daeafb9864 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/.gitignore @@ -0,0 +1 @@ +test diff --git a/tools/voluspa/tests/other/all_bad/base/stderr.txt b/tools/voluspa/tests/other/all_bad/base/stderr.txt new file mode 100644 index 00000000000..dd3c698ba51 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/base/stderr.txt @@ -0,0 +1,14 @@ +voluspa: errors validating configs: + - alias http://(.*.myalias.com not unique (all_bad/simple2.conf) + - alias http://inbound.testing.example.com not unique (all_bad/simple2.conf) + - alias https://inbound.testing.example.com not unique (all_bad/simple2.conf) + - invalid regex URL: error parsing regexp: missing closing ): `http://(.*.myalias.com` (all_bad/simple2.conf) + - invalid regex: error parsing regexp: unexpected ): `# If Cookie : ADCDownloadAuth is missing then +# 302 to developer.example.com/unauthorized/ +cond %{COOKIE:Tasty} ="" [AND] +cond %{CLIENT-IP} ^127\.0\.0\.1|10\..*)/ [NOT] + set-redirect 302 https://inbound.example.com/unauthorized/ [L] +# If Origin serves a 000, 500 501 502 503 then +# 302 to inboud.example.com/unauthorized/` (all_bad/simple2.conf) + - property All not unique (all_bad/simple.conf) + - property All not unique (all_bad/simple2.conf) diff --git a/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf b/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf new file mode 100644 index 00000000000..3536d7c9fb9 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf @@ -0,0 +1,13 @@ +schema_version: 1 +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - dupe.testing.example.com + origin: https://dupe.example.com + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] diff --git a/tools/voluspa/tests/other/all_bad/simple.conf b/tools/voluspa/tests/other/all_bad/simple.conf new file mode 100644 index 00000000000..9da5d2e60af --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/simple.conf @@ -0,0 +1,13 @@ +schema_version: '1.0' +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - inbound.testing.example.com + origin: https://origin.example.com + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] diff --git a/tools/voluspa/tests/other/all_bad/simple2.conf b/tools/voluspa/tests/other/all_bad/simple2.conf new file mode 100644 index 00000000000..1fe7f8150b1 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/simple2.conf @@ -0,0 +1,25 @@ +schema_version: 1 +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - inbound.testing.example.com + origin: https://origin.example.com + - alias: + - http://(.*.myalias.com + origin: http://origin3.example.com + regex_map: true + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] + header_rewrite: |- + # If Cookie : ADCDownloadAuth is missing then + # 302 to developer.example.com/unauthorized/ + cond %{COOKIE:Tasty} ="" [AND] + cond %{CLIENT-IP} ^127\.0\.0\.1|10\..*)/ [NOT] + set-redirect 302 https://inbound.example.com/unauthorized/ [L] + # If Origin serves a 000, 500 501 502 503 then + # 302 to inboud.example.com/unauthorized/ diff --git a/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt b/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt new file mode 100644 index 00000000000..9a5cc8b30cc --- /dev/null +++ b/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt @@ -0,0 +1,3 @@ +voluspa: errors validating configs: + - invalid regex: error parsing regexp: missing closing ): `BYT\s([^,]+.*` (cachekey_bad/test.conf) + - invalid regex: unexpected / (cachekey_bad/test.conf) diff --git a/tools/voluspa/tests/other/cachekey_bad/test.conf b/tools/voluspa/tests/other/cachekey_bad/test.conf new file mode 100644 index 00000000000..3f9e41961c2 --- /dev/null +++ b/tools/voluspa/tests/other/cachekey_bad/test.conf @@ -0,0 +1,28 @@ +schema_version: 1 +owner: "" +reference: your reference notes here +rcpt: All +ssl_cert_names: + - /secrets/mysecret.txt + +mappings: + - origin: origin7.example.com + alias: + - myalias7.example.com + - myalias7.example.com/ran + - myalias7.example.com/bob + - /path/test.txt + +rules: + default: + cachekey: + include_params: ["param1", "param2"] + exclude_params: ["param1", "param2"] + include_headers: ["header1", "header2"] + include_cookies: ["cookie1", "cookie2"] + sort_query: true + remove_all_params: true + static_prefix: "SomeString" + regex_replace_path: '/.*/probe-info-ok.html/' + regex_replace_path_uri: '/http[s]?:\//[^\/]*.*\.html|.*).*/$1/' + capture_header: [ '/Authorization:AWS\s([^:]+).*/clientID:$1//', '/Authorization:BYT\s([^,]+.*/clientID:$1/' ] diff --git a/tools/voluspa/tests/other/duped/base/stderr.txt b/tools/voluspa/tests/other/duped/base/stderr.txt new file mode 100644 index 00000000000..fa0270f952c --- /dev/null +++ b/tools/voluspa/tests/other/duped/base/stderr.txt @@ -0,0 +1,2 @@ +voluspa: errors validating configs: + - alias http://inbound.example.com not unique for rule role "roles_edge_child" (duped/test.conf) diff --git a/tools/voluspa/tests/other/duped/test.conf b/tools/voluspa/tests/other/duped/test.conf new file mode 100644 index 00000000000..b06e3c1418d --- /dev/null +++ b/tools/voluspa/tests/other/duped/test.conf @@ -0,0 +1,27 @@ +schema_version: 1 +owner: owner +rcpt: streamer +parent_child: true + +rules: + default_child: + priority: streamingvideo + + default_child2: + priority: background + +explicit: + - alias: + - http://inbound.example.com + origin: http://origin.example.com + rule: default_child + role: roles_edge_child + + # NOTE: Duping above on purpose to trigger duplicate alias + - alias: + - http://inbound.example.com + origin: http://origin2.example.com + rule: default_child2 + role: roles_edge_child + +# vim: set et sw=2 ts=2 ft=yaml : diff --git a/tools/voluspa/tests/other/properties.sh b/tools/voluspa/tests/other/properties.sh new file mode 100644 index 00000000000..d903ee41f53 --- /dev/null +++ b/tools/voluspa/tests/other/properties.sh @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CUST=(all_bad cachekey_bad duped unicode scheme) diff --git a/tools/voluspa/tests/other/regen_baseline.sh b/tools/voluspa/tests/other/regen_baseline.sh new file mode 100755 index 00000000000..96880d7fe4e --- /dev/null +++ b/tools/voluspa/tests/other/regen_baseline.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +CMD=../../voluspa +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/base + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf 2> $DEST/stderr.txt +done + +# Ignore failures from voluspa +exit 0 diff --git a/tools/voluspa/tests/other/scheme/base/stderr.txt b/tools/voluspa/tests/other/scheme/base/stderr.txt new file mode 100644 index 00000000000..7195ca43c90 --- /dev/null +++ b/tools/voluspa/tests/other/scheme/base/stderr.txt @@ -0,0 +1,2 @@ +problem loading "scheme/test.conf": + - mappings.0.schemes.0: mappings.0.schemes.0 must be one of the following: "http", "https" diff --git a/tools/voluspa/tests/other/scheme/test.conf b/tools/voluspa/tests/other/scheme/test.conf new file mode 100644 index 00000000000..779a72a7174 --- /dev/null +++ b/tools/voluspa/tests/other/scheme/test.conf @@ -0,0 +1,9 @@ +schema_version: 1 +rcpt: seed +owner: "nobody" + +mappings: + - alias: + - inbound.example.com + origin: https://origin.example.com + schemes: [httpZs, http] diff --git a/tools/voluspa/tests/other/test.sh b/tools/voluspa/tests/other/test.sh new file mode 100755 index 00000000000..eca4975f2cf --- /dev/null +++ b/tools/voluspa/tests/other/test.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +# this expects 'make voluspa' to be run (make sanity_tests or make test does this) +if [ -z "$VOLUSPA" ]; then + VOLUSPA="voluspa" +fi + +CMD=../../$DEST/$VOLUSPA +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/test + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf 2> $DEST/stderr.txt + if [ "$VOLUSPA" == "voluspa" ]; then + diff -r ${i}/base $DEST + if [ $? != 0 ]; then + exit 1 + fi + else + mv system.out profile_${i}.out + fi +done diff --git a/tools/voluspa/tests/other/unicode/base/stderr.txt b/tools/voluspa/tests/other/unicode/base/stderr.txt new file mode 100644 index 00000000000..0327c36679d --- /dev/null +++ b/tools/voluspa/tests/other/unicode/base/stderr.txt @@ -0,0 +1 @@ +problem loading "unicode/httpbin.conf": Could not load YAML file. err=non-ASCII characters not supported: invalid rune "🍎" diff --git a/tools/voluspa/tests/other/unicode/httpbin.conf b/tools/voluspa/tests/other/unicode/httpbin.conf new file mode 100644 index 00000000000..c630cc59e0e --- /dev/null +++ b/tools/voluspa/tests/other/unicode/httpbin.conf @@ -0,0 +1,16 @@ +schema_version: 1 +owner: Property Owner +reference: For testing ATS behavior 🍎 +rcpt: httpbin +schemes: [http, https] + +mappings: + - alias: + - httpbin.example.com + origin: https://httpbin.edge.example.com + +rules: + default: + priority: background + origin_host_header: alias + deny_methods: [ CONNECT ] diff --git a/tools/voluspa/tests/regress/all/base/all/all.config b/tools/voluspa/tests/regress/all/base/all/all.config new file mode 100644 index 00000000000..fea885654ab --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/all.config @@ -0,0 +1,196 @@ +# Code generated by Voluspa. DO NOT EDIT. + +map http://myalias7.example.com/bob \ + http://origin7.example.com + +map http://myalias7.example.com/ran \ + http://origin7.example.com + +map http://myalias7.example.com \ + http://origin7.example.com + +map /path/test.txt \ + http://origin7.example.com + +map http://redirect.example.com \ + http://origin7.example.com \ + @plugin=regex_remap.so @pparam=all/redirect9.config + +map https://myalias1.example.com \ + https://origin2.example.com + +map https://myalias2.example.com \ + https://origin2.example.com + +regex_map http://(.*).example.com \ + http://origin3.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map http://myalias4.example.com \ + http://origin4.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map https://myalias4.example.com \ + https://origin4.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map http://myalias5.example.com \ + http://origin4.example.com \ + @plugin=header_rewrite.so @pparam=all/hdrs6.config + +{% if salt.pillar.get("roles_edge_child") %} + +map http://myalias6.example.com \ + http://origin6.example.com \ + @plugin=volcano.so {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--front=true \{% else %} \{% endif %} + +{% endif %} + +map http://myalias8.example.com \ + http://origin8.example.com \ + @plugin=header_rewrite.so @pparam=all/hdrs10.config + +map http://myalias9.example.com \ + http://origin9.example.com \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-region-map=region_map_test.config \ + @pparam=--v4-exclude-headers=x-forwarded-for,forwarded,via,authorization + +map http://myaliasa.example.com \ + http://origina.example.com \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=2 \ + @pparam=--v4-region-map=region_map_test.config + diff --git a/tools/voluspa/tests/regress/all/base/all/gzip2.config b/tools/voluspa/tests/regress/all/base/all/gzip2.config new file mode 100644 index 00000000000..6dc3e065479 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/gzip2.config @@ -0,0 +1,10 @@ +# Code generated by Voluspa. DO NOT EDIT. + +enabled true +cache false +remove-accept-encoding true +flush true +minimum-content-length 532 +supported-algorithms br,gzip +compressible-content-type *javascript* +compressible-content-type text/* diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs10.config b/tools/voluspa/tests/regress/all/base/all/hdrs10.config new file mode 100644 index 00000000000..7ae70a9452b --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs10.config @@ -0,0 +1,4 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{REMAP_PSEUDO_HOOK} + set-header @cdnlog "private" diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs2.config b/tools/voluspa/tests/regress/all/base/all/hdrs2.config new file mode 100644 index 00000000000..1f5c961bd93 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs2.config @@ -0,0 +1,52 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{READ_RESPONSE_HDR_HOOK} + add-header X-Client-Protocol % +cond %{SEND_REQUEST_HDR_HOOK} + add-header X-TestOriginHeader "1" +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.crl$/ + set-header Content-Type application/pkix-crl +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.css$/ + set-header Content-Type text/css +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.json$/ + set-header Content-Type application/json + +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=3600, public" +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=0, public" +cond %{SEND_REQUEST_HDR_HOOK} + add-header Authorization "Basic 123412341234" [L] + +cond %{REMAP_PSEUDO_HOOK} + set-header Bazinga1 %{COOKIE:NameOfCookie1} + set-header Bazinga2 %{COOKIE:NameOfCookie2} +cond %{REMAP_PSEUDO_HOOK} + set-header Bazinga3 %{HEADER:NameOfHeader1} + set-header Bazinga4 %{HEADER:NameOfHeader2} + +cond %{REMAP_PSEUDO_HOOK} + set-conn-dscp 8 +cond %{READ_RESPONSE_HDR_HOOK} + set-header @Original-Cache-Control %{HEADER:Cache-Control} + set-header Cache-Control %{HEADER:MyProxyHeader} + +cond %{SEND_RESPONSE_HDR_HOOK} + set-header Cache-Control %{HEADER:@Original-Cache-Control} + rm-header Proxy-Cache-Control +cond %{REMAP_PSEUDO_HOOK} + rm-header Proxy +cond %{SEND_REQUEST_HDR_HOOK} + rm-header Proxy +cond %{READ_RESPONSE_HDR_HOOK} + set-header X-Client-Protocol % +cond %{SEND_REQUEST_HDR_HOOK} + set-header X-TestOriginHeader "1" diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs6.config b/tools/voluspa/tests/regress/all/base/all/hdrs6.config new file mode 100644 index 00000000000..d1606fafb7e --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs6.config @@ -0,0 +1,4 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{REMAP_PSEUDO_HOOK} + set-header @ReceiptService "override{{hosttype}}" diff --git a/tools/voluspa/tests/regress/all/base/all/redirect9.config b/tools/voluspa/tests/regress/all/base/all/redirect9.config new file mode 100644 index 00000000000..866f0304a09 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/redirect9.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +(.*) https://example.com/filenotfound @status=302 diff --git a/tools/voluspa/tests/regress/all/base/all/regex2.config b/tools/voluspa/tests/regress/all/base/all/regex2.config new file mode 100644 index 00000000000..9889c40e6ce --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/regex2.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2 diff --git a/tools/voluspa/tests/regress/all/base/all/strip_query2.config b/tools/voluspa/tests/regress/all/base/all/strip_query2.config new file mode 100644 index 00000000000..7b0312493d4 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/strip_query2.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +. $s://$t/$P diff --git a/tools/voluspa/tests/regress/all/base/hosting.config_default b/tools/voluspa/tests/regress/all/base/hosting.config_default new file mode 100644 index 00000000000..4dd9b032a1c --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/hosting.config_default @@ -0,0 +1,7 @@ +# Code generated by Voluspa. DO NOT EDIT. + +{% if salt.pillar.get("roles_trafficserver_ramdisk") %} +hostname=* {{ default_volumes }} +{% else %} +# hosting.config disabled on this host +{% endif %} diff --git a/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default b/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default new file mode 100644 index 00000000000..4fad5702530 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default @@ -0,0 +1,9 @@ +# Code generated by Voluspa. DO NOT EDIT. + +{% if not salt.pillar.get('roles_uat') %} + +# your reference notes here +ssl_cert_name=/secrets/mysecret.txt + +{% endif %} + diff --git a/tools/voluspa/tests/regress/all/test.conf b/tools/voluspa/tests/regress/all/test.conf new file mode 100644 index 00000000000..71461394018 --- /dev/null +++ b/tools/voluspa/tests/regress/all/test.conf @@ -0,0 +1,239 @@ +schema_version: 1 +owner: "" +reference: your reference notes here +rcpt: All +ssl_cert_names: + - /secrets/mysecret.txt + +rules: + default: + add_header: "X-Client-Protocol %" + add_header_origin: X-TestOriginHeader "1" + set_header: "X-Client-Protocol %" + set_header_origin: X-TestOriginHeader "1" + remove_header: "Proxy" + remove_header_origin: "Proxy" + allow_ip: [10.153.0.0-10.153.255.255, 10.179.0.0-10.179.255.255] + allow_ip_parent: [10.253.0.0-10.253.255.255, 10.254.0.0-10.254.255.255] + strip_query: true + force_ttl: 0s + proxy_cache_control: "MyProxyHeader" + origin_host_header: origin + disable_cache: true + deny_methods: [ POST, PUT, DELETE ] + allow_methods: [ PURGE ] + regex_remap: "^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2" + s3: + path: certs/s3.config + virtual_host: true + version: 4 + v4_include_headers: ["header1", "header2"] + v4_exclude_headers: ["header3", "header4"] + region_map: region_map_test.config + video_background_fetch: + fetch_count: 10 + fetch_max: 1000 + api_header: header1 + replace_host: host1 + name_space: ns1 + metrics_prefix: prefix1 + exact_match: true + log_name: log1 + frontend: + fetch_path_pattern: + - '/(.*-)(\d+)(.*)/$1{$2+2}$3/' + - /(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ + fetch_policy: lru:123 + backend: + fetch_policy: simple + + cache_promote: + policy: lru + lru_hits: 5 + lru_buckets: 100000 + + cachekey: + include_params: ["param1", "param2"] + exclude_params: ["param1", "param2"] + include_headers: ["header1", "header2"] + include_cookies: ["cookie1", "cookie2"] + sort_query: true + remove_all_params: true + static_prefix: "SomeString" + regex_replace_path: '/.*/probe-info-ok.html/' + regex_replace_path_uri: '/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/' + capture_header: [ 'Authorization:/AWS\s([^:]+).*/clientID:$1/', 'Authorization:/BYT\s([^,]+).*/clientID:$1/' ] + failover: + domain: assets-origin.subdomain.example.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: origin + priority: background + default_ttl: 60m + content_type_forge: + '.*\.json$': application/json + '.*\.css$': text/css + '.*\.crl$': application/pkix-crl + conf_remap: "CONFIG proxy.config.http.connect_attempts_timeout INT 600\n + CONFIG proxy.config.http.transaction_no_activity_timeout_in INT 600\n + CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 600" + header_rewrite: "cond %{SEND_REQUEST_HDR_HOOK}\n + add-header Authorization \"Basic 123412341234\" [L]" + log_cookie: + Logger-cookie-Name1: NameOfCookie1 + Logger-cookie-Name2: NameOfCookie2 + log_header: + Logger-header-Name1: NameOfHeader1 + Logger-header-Name2: NameOfHeader2 + negative_ttl: 60m + transaction_timeout: 10m + edge_compression: + enabled: true + flush: true + remove_accept_encoding: true + minimum_content_length: 532 + algorithms: [gzip, br] + static_origin_compressible_content_type: + - "text/*" + - "*javascript*" + access_approval: + raw: |- + @pparam=--symmetric-keys-map=hmac_keys.txt + @pparam=--check-cookie=TokenCookie + @pparam=--extract-subject-to-header=@TokenSubject + @pparam=--extract-tokenid-to-header=@TokenId + @pparam=--extract-status-to-header=@TokenStatus + @pparam=--token-response-header=TokenRespHdr + cache_version: 3 + + no_receipts: + receipts: + enabled: false + + override_receipt: + receipts: + name: override + + # Redirect takes one redirect url and status code. + # If status code is empty, it takes default value of 302. + redirect_this: + redirect: + url: https://example.com/filenotfound + + preserve_host_header: + origin_host_header: alias + receipts: + enabled: false + priority: foreground + s3: + path: /opt/ats/etc/trafficserver/certs/s3.config + virtual_host: false + + escalate_pristine: + failover: + domain: assets-origin.subdomain.example.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: origin + + streamingaudio: + priority: streamingaudio + + streamingvideo: + priority: streamingvideo + + roles_within_roles: + video_background_fetch: + frontend: + fetch_policy: lru:123 + backend: + fetch_policy: simple + + cache_promote: + cache_promote: + policy: lru + lru_hits: 5 + lru_buckets: 100000 + + rlog_type: + log_type: private + + storage_volume_ram: + storage_volume: ramdisk_volume + + storage_volume_disk: + storage_volume: disk_volume + + raw_lua: + lua: |- + function do_remap() + local to_host = ts.remap.get_to_url_host() + ts.debug(to_host) + return 0 + end + + echo_cors_test: + echo_cors: "Origin" + + s3_v4: + s3: + path: certs/s3.config + virtual_host: true + version: 4 + region_map: region_map_test.config + + + s3_v2: + s3: + path: certs/s3.config + virtual_host: true + version: 2 + region_map: region_map_test.config + + +explicit: + - origin: https://origin2.example.com + alias: + - https://myalias1.example.com + - https://myalias2.example.com + rule: no_receipts + - origin: http://origin3.example.com + alias: + - http://(.*).example.com + regex_map: true + - origin: origin4.example.com + alias: + - myalias4.example.com + schemes: [http, https] + - origin: origin4.example.com + alias: + - myalias5.example.com + rule: override_receipt + - origin: origin6.example.com + alias: + - myalias6.example.com + role: roles_edge_child + rule: roles_within_roles + - origin: origin8.example.com + alias: + - myalias8.example.com + rule: rlog_type + - origin: origin9.example.com + alias: + - myalias9.example.com + rule: s3_v4 + - origin: origina.example.com + alias: + - myaliasa.example.com + rule: s3_v2 + +mappings: + - origin: origin7.example.com + alias: + - myalias7.example.com + - myalias7.example.com/ran + - myalias7.example.com/bob + - /path/test.txt + rule: no_receipts + - origin: origin7.example.com + alias: + - redirect.example.com + rule: redirect_this diff --git a/tools/voluspa/tests/regress/properties.sh b/tools/voluspa/tests/regress/properties.sh new file mode 100644 index 00000000000..f5c3cc28307 --- /dev/null +++ b/tools/voluspa/tests/regress/properties.sh @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +CUST=(all) diff --git a/tools/voluspa/tests/regress/regen_baseline.sh b/tools/voluspa/tests/regress/regen_baseline.sh new file mode 100755 index 00000000000..3733ceb352f --- /dev/null +++ b/tools/voluspa/tests/regress/regen_baseline.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set +e + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +CMD=../../voluspa +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/base + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf +done diff --git a/tools/voluspa/tests/regress/test.sh b/tools/voluspa/tests/regress/test.sh new file mode 100755 index 00000000000..5e6b66890ea --- /dev/null +++ b/tools/voluspa/tests/regress/test.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +# this expects 'make voluspa' to be run (make sanity_tests or make test does this) +if [ -z "$VOLUSPA" ]; then + VOLUSPA="voluspa" +fi + +CMD=../../$DEST/$VOLUSPA +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/test + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf > /dev/null + if [ "$VOLUSPA" == "voluspa" ]; then + if [ $? != 0 ]; then + exit 1 + fi + + diff -x .gitignore -x \*.swp -r ${i}/base $DEST + if [ $? != 0 ]; then + exit 1 + fi + else + mv system.out profile_${i}.out + fi +done + diff --git a/tools/voluspa/toplevel_remaps.go b/tools/voluspa/toplevel_remaps.go new file mode 100644 index 00000000000..102d5621662 --- /dev/null +++ b/tools/voluspa/toplevel_remaps.go @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +type TopLevelRemapGenerator struct { + options *Options +} + +func newTopLevelRemapGenerator(options *Options) *TopLevelRemapGenerator { + return &TopLevelRemapGenerator{options: options} +} + +func (tlrg *TopLevelRemapGenerator) formatDescriptionComment(config *CustomerConfig) string { + var domain string + if strings.HasPrefix(config.Remaps[0].IncomingURL, "http") { + parts := strings.Split(config.Remaps[0].IncomingURL, "/") + if len(parts) >= 2 { + domain = parts[2] + } + } + + var buf bytes.Buffer + buf.WriteString("############################################################################\n") + buf.WriteString(fmt.Sprintf("# Host: %s\n", domain)) + buf.WriteString(fmt.Sprintf("# Owner: %s\n", config.owner)) + if len(config.reference) > 0 { + buf.WriteString(fmt.Sprintf("# Reference: %s\n", config.reference)) + } + buf.WriteString("############################################################################\n") + return buf.String() +} + +func (tlrg *TopLevelRemapGenerator) filenameForRole(role, cdn, parentOrChild string) string { + var buf bytes.Buffer + if cdn == DefaultCDN || cdn == NoCDN { + buf.WriteString("remap_") + } else { + buf.WriteString(fmt.Sprintf("%s_remap_", cdn)) + } + + if len(role) > 0 { + buf.WriteString(fmt.Sprintf("%s_", role)) + } + + buf.WriteString(fmt.Sprintf("%s_include.config", parentOrChild)) + + return buf.String() +} + +// groupedRemaps returns true if a Group has been explicitly set +func groupedRemaps(remaps []Remap) bool { + for _, remap := range remaps { + if remap.Group != DefaultGroup { + return true + } + } + return false +} + +func (tlrg *TopLevelRemapGenerator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return tlrg.get(parsedConfigs, NoRole, NoCDN) + } + + if len(parsedConfigs) == 0 { + return nil, ErrMinimumConfigsNotMet + } + + // Generating the include file is next to useless for 1 file + if len(parsedConfigs) == 1 { + return nil, nil + } + + // group properties by CDN and role and generate includeable files for each role/parent/child combo + grouped := make(map[string]map[string][]*CustomerConfig) + + for _, config := range parsedConfigs { + if config.lifecycle == Retired { + continue + } + + for cdn := range config.cdn { + + if _, exists := grouped[cdn]; !exists { + grouped[cdn] = make(map[string][]*CustomerConfig) + } + + role := config.role + grouped[cdn][role] = append(grouped[cdn][role], config) + } + } + var managedFiles []ManagedFile + for cdn, cdnGrouped := range grouped { + for role, configs := range cdnGrouped { + sort.Stable(sort.Reverse(byQPSAndName(configs))) + + remaps, err := tlrg.get(configs, role, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, remaps...) + } + } + + return managedFiles, nil + +} + +func (tlrg *TopLevelRemapGenerator) get(configs []*CustomerConfig, role, cdn string) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, parentOrChild := range []string{Child, Parent} { + + var buf bytes.Buffer + buf.WriteString("# START voluspa-generated config\n") + + if len(role) == 0 { + buf.WriteString("\n") + } else { + buf.WriteString(fmt.Sprintf("# ROLE: %s\n\n", role)) + } + + for _, parsedConfig := range configs { + if _, exists := parsedConfig.cdn[cdn]; !exists && cdn != NoCDN { + continue + } + + // skip grouped remap sets as the output cannot be used + // without knowledge beyond the scope of voluspa + // (feature used by cdn-icloud-content-uat) + if groupedRemaps(parsedConfig.Remaps) { + continue + } + + property := strings.ToLower(parsedConfig.property) + buf.WriteString(tlrg.formatDescriptionComment(parsedConfig)) + + filename := property + + if parentOrChild == Child || !parsedConfig.parentChild { + buf.WriteString(fmt.Sprintf(".include %s/%s.config\n", property, filename)) + } else { + buf.WriteString(fmt.Sprintf(".include %s/%s_parent.config\n", property, filename)) + } + buf.WriteString("\n") + } + buf.WriteString("# END voluspa-generated config\n\n") + + fileName := tlrg.filenameForRole(role, cdn, parentOrChild) + + var remoteFilename string + if len(role) > 0 { + remoteFilename = fmt.Sprintf("remap_voluspa_%s.config", role) + } else { + remoteFilename = "remap_voluspa.config" + } + + managedFiles = append(managedFiles, NewManagedFile(fileName, remoteFilename, cdn, role, "", &buf, tlrg.configType(parentOrChild))) + } + + return managedFiles, nil +} + +func (tlrg *TopLevelRemapGenerator) configType(val string) ConfigLocation { + switch val { + case Child: + return ChildConfig + case Parent: + return ParentConfig + default: + panic(fmt.Errorf("Unhandled configType '%s'", val)) + } +} diff --git a/tools/voluspa/util.go b/tools/voluspa/util.go new file mode 100644 index 00000000000..4d5e984c04f --- /dev/null +++ b/tools/voluspa/util.go @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +// groupCustomerConfigsByCDN groups CustomerConfigs by CDN +func groupCustomerConfigsByCDN(parsedConfigs []*CustomerConfig) map[string][]*CustomerConfig { + // group properties by CDN and generate parent.config for each CDN + grouped := make(map[string][]*CustomerConfig) + for _, config := range parsedConfigs { + for cdn := range config.cdn { + grouped[cdn] = append(grouped[cdn], config) + } + } + return grouped +} diff --git a/tools/voluspa/validate.go b/tools/voluspa/validate.go new file mode 100644 index 00000000000..eebb1e189b8 --- /dev/null +++ b/tools/voluspa/validate.go @@ -0,0 +1,312 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "errors" + "fmt" + "net" + "net/url" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa/internal/util/regex" +) + +type Errors []error + +func (err Errors) Error() string { + if len(err) == 0 { + return "Unknown" + } + + var errOut []string + for _, e := range err { + errOut = append(errOut, e.Error()) + } + sort.Strings(errOut) + + return strings.Join(errOut, "\n") +} + +func (err *Errors) Add(newErr error) { + *err = append(*err, newErr) +} + +func (err *Errors) HasErrors() bool { + return len(*err) > 0 +} + +func isValidScheme(scheme string) bool { + return scheme == "http" || scheme == "https" +} + +func (v *Voluspa) ensureValidSchemes() Errors { + var errs Errors + for _, config := range v.parsedConfigs { + for _, scheme := range config.schemes { + if !isValidScheme(scheme) { + errs.Add(fmt.Errorf("property %q contains invalid scheme: %s (%s)", config.property, scheme, config.filename)) + } + } + + for _, remap := range config.Remaps { + if len(remap.scheme) > 0 && !isValidScheme(remap.scheme) { + errs.Add(fmt.Errorf("property %q contains invalid scheme: %s (%s)", config.property, remap.scheme, config.filename)) + } + + } + } + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensurePropertyNameUniqueness() Errors { + var errs Errors + + properties := make(map[string]interface{}) + + for _, config := range v.parsedConfigs { + if _, exists := properties[config.property]; exists { + errs.Add(fmt.Errorf("property %s not unique (%s)", config.property, config.filename)) + } else { + properties[config.property] = true + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensureAliasUniqueness() Errors { + var errs Errors + // map of cdn to role to configlocation to alias + aliases := make(map[string]map[string]map[ConfigLocation]map[string]interface{}) + + // Initialize map/struct + for _, config := range v.parsedConfigs { + for cdn := range config.cdn { + if _, exists := aliases[cdn]; !exists { + aliases[cdn] = make(map[string]map[ConfigLocation]map[string]interface{}) + } + if _, exists := aliases[cdn][config.role]; !exists { + aliases[cdn][config.role] = make(map[ConfigLocation]map[string]interface{}) + for _, location := range []ConfigLocation{UnknownLocation, ChildConfig, ParentConfig} { + aliases[cdn][config.role][location] = make(map[string]interface{}) + } + } + } + } + + for _, config := range v.parsedConfigs { + for cdn := range config.cdn { + for _, remap := range config.Remaps { + alias := remap.IncomingURL + + // aliases are unique based on the aliases itself + the group it's part of and the role + aliasKey := alias + remap.Group + remap.Role + + if _, exists := aliases[cdn][config.role][remap.ConfigLocation][aliasKey]; exists { + var err error + switch { + case len(config.role) > 0: + err = fmt.Errorf("alias %s not unique for role %q (%s)", alias, config.role, config.filename) + case len(remap.Role) > 0: + err = fmt.Errorf("alias %s not unique for rule role %q (%s)", alias, remap.Role, config.filename) + default: + err = fmt.Errorf("alias %s not unique (%s)", alias, config.filename) + } + errs.Add(err) + } else { + aliases[cdn][config.role][remap.ConfigLocation][aliasKey] = nil + } + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensureHostsResolve() Errors { + var errs Errors + + for _, config := range v.parsedConfigs { + for _, remap := range config.Remaps { + if remap.RegexMap { + continue + } + + url, err := url.Parse(remap.OriginURL) + if err != nil { + errs.Add(fmt.Errorf("%s invalid url; %s (%s)", remap.OriginURL, err, config.filename)) + } + if err = isResolvable(url.Host); err != nil { + errs = append(errs, fmt.Errorf("%s unresolvable host (%s)", url.Host, config.filename)) + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func isResolvable(host string) error { + if strings.Contains(host, ":") { + host = strings.Split(host, ":")[0] + } + + _, err := net.LookupHost(host) + return err +} + +// See https://docs.trafficserver.apache.org/en/latest/admin-guide/plugins/cachekey.en.html#capture-definition +// This is either a regex or a "/" + regex_with_capture + "/" + regex_replace + "/" +func validateCacheKeyCapture(input string) error { + + validator := ®ex.GoLangRegex{} + + regexes := make([]string, 0) + currentRe := "" + prev := ' ' + + // loop over all the characters + for _, c := range input { + if c == '/' && prev != '\\' { // find an unescaped / - it is the boundary + if currentRe != "" { + regexes = append(regexes, currentRe) + } + currentRe = "" + prev = c + continue + } + currentRe += string(c) + prev = c + } + + // if nothing was pushed, there was no / at all, so push the whole string and a dummy + if currentRe != "" { + regexes = append(regexes, currentRe) + regexes = append(regexes, "dummy") + } + + // too many unescaped /'s + if len(regexes) != 2 { + return errors.New("unexpected /") + } + + // check the regexes + for _, re := range regexes { + _, err := validator.IsValid(re) + if err != nil { + return err + } + } + return nil +} + +func (v *Voluspa) validateRegexOptions() Errors { + validator := ®ex.GoLangRegex{} + + var errs Errors + for _, config := range v.parsedConfigs { + for _, remap := range config.Remaps { + if remap.RegexMap { + _, err := validator.IsValid(remap.IncomingURL) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex URL: %s (%s)", err, config.filename)) + } + } + + for k := range remap.sourceConfig.RemapOptions { + switch k { + case "header_rewrite", // HANDLE header_rewrite separately (split and look for Cond) + "regex_remap", + "asset_token_include", + "asset_token_exclude": + + v, err := remap.sourceConfig.RemapOptions.ValueByNameAsString(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + _, err = validator.IsValid(v) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + + case "content_type_forge": + forgeMap, err := remap.sourceConfig.RemapOptions.ValueByNameAsStringMapString(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + for mapKey := range forgeMap { + _, err := validator.IsValid(mapKey) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + } + case "cachekey": + cachekeyMap, err := remap.sourceConfig.RemapOptions.ValueByNameAsStringMapInterface(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + for mapKey, mapValue := range cachekeyMap { + if mapKey == "regex_replace_path" || mapKey == "regex_replace_path_uri" || mapKey == "capture_path" { + err := validateCacheKeyCapture(mapValue.(string)) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + } + if mapKey == "capture_header" { + for _, re := range mapValue.([]interface{}) { + regexParts := strings.Split(re.(string), ":")[1:] + regexString := strings.Join(regexParts, ":") + err := validateCacheKeyCapture(regexString) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + + } + } + } + } + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} diff --git a/tools/voluspa/vendor/.gitignore b/tools/voluspa/vendor/.gitignore new file mode 100644 index 00000000000..7fa54591b3a --- /dev/null +++ b/tools/voluspa/vendor/.gitignore @@ -0,0 +1,9 @@ +github.com/hpcloud/tail/vendor +.travis* +README* +CONTRIB* +*sh +*pl +*conf +Gopkg.* +*yaml diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go new file mode 100644 index 00000000000..7faf5d7f943 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go @@ -0,0 +1,211 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonpointer +// repository-desc An implementation of JSON Pointer - Go language +// +// description Main and unique file. +// +// created 25-02-2013 + +package gojsonpointer + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" +) + +const ( + const_empty_pointer = `` + const_pointer_separator = `/` + + const_invalid_start = `JSON pointer must be empty or start with a "` + const_pointer_separator + `"` +) + +type implStruct struct { + mode string // "SET" or "GET" + + inDocument interface{} + + setInValue interface{} + + getOutNode interface{} + getOutKind reflect.Kind + outError error +} + +type JsonPointer struct { + referenceTokens []string +} + +// NewJsonPointer parses the given string JSON pointer and returns an object +func NewJsonPointer(jsonPointerString string) (p JsonPointer, err error) { + + // Pointer to the root of the document + if len(jsonPointerString) == 0 { + // Keep referenceTokens nil + return + } + if jsonPointerString[0] != '/' { + return p, errors.New(const_invalid_start) + } + + p.referenceTokens = strings.Split(jsonPointerString[1:], const_pointer_separator) + return +} + +// Uses the pointer to retrieve a value from a JSON document +func (p *JsonPointer) Get(document interface{}) (interface{}, reflect.Kind, error) { + + is := &implStruct{mode: "GET", inDocument: document} + p.implementation(is) + return is.getOutNode, is.getOutKind, is.outError + +} + +// Uses the pointer to update a value from a JSON document +func (p *JsonPointer) Set(document interface{}, value interface{}) (interface{}, error) { + + is := &implStruct{mode: "SET", inDocument: document, setInValue: value} + p.implementation(is) + return document, is.outError + +} + +// Uses the pointer to delete a value from a JSON document +func (p *JsonPointer) Delete(document interface{}) (interface{}, error) { + is := &implStruct{mode: "DEL", inDocument: document} + p.implementation(is) + return document, is.outError +} + +// Both Get and Set functions use the same implementation to avoid code duplication +func (p *JsonPointer) implementation(i *implStruct) { + + kind := reflect.Invalid + + // Full document when empty + if len(p.referenceTokens) == 0 { + i.getOutNode = i.inDocument + i.outError = nil + i.getOutKind = kind + i.outError = nil + return + } + + node := i.inDocument + + previousNodes := make([]interface{}, len(p.referenceTokens)) + previousTokens := make([]string, len(p.referenceTokens)) + + for ti, token := range p.referenceTokens { + + isLastToken := ti == len(p.referenceTokens)-1 + previousNodes[ti] = node + previousTokens[ti] = token + + switch v := node.(type) { + + case map[string]interface{}: + decodedToken := decodeReferenceToken(token) + if _, ok := v[decodedToken]; ok { + node = v[decodedToken] + if isLastToken && i.mode == "SET" { + v[decodedToken] = i.setInValue + } else if isLastToken && i.mode =="DEL" { + delete(v,decodedToken) + } + } else if (isLastToken && i.mode == "SET") { + v[decodedToken] = i.setInValue + } else { + i.outError = fmt.Errorf("Object has no key '%s'", decodedToken) + i.getOutKind = reflect.Map + i.getOutNode = nil + return + } + + case []interface{}: + tokenIndex, err := strconv.Atoi(token) + if err != nil { + i.outError = fmt.Errorf("Invalid array index '%s'", token) + i.getOutKind = reflect.Slice + i.getOutNode = nil + return + } + if tokenIndex < 0 || tokenIndex >= len(v) { + i.outError = fmt.Errorf("Out of bound array[0,%d] index '%d'", len(v), tokenIndex) + i.getOutKind = reflect.Slice + i.getOutNode = nil + return + } + + node = v[tokenIndex] + if isLastToken && i.mode == "SET" { + v[tokenIndex] = i.setInValue + } else if isLastToken && i.mode =="DEL" { + v[tokenIndex] = v[len(v)-1] + v[len(v)-1] = nil + v = v[:len(v)-1] + previousNodes[ti-1].(map[string]interface{})[previousTokens[ti-1]] = v + } + + default: + i.outError = fmt.Errorf("Invalid token reference '%s'", token) + i.getOutKind = reflect.ValueOf(node).Kind() + i.getOutNode = nil + return + } + + } + + i.getOutNode = node + i.getOutKind = reflect.ValueOf(node).Kind() + i.outError = nil +} + +// Pointer to string representation function +func (p *JsonPointer) String() string { + + if len(p.referenceTokens) == 0 { + return const_empty_pointer + } + + pointerString := const_pointer_separator + strings.Join(p.referenceTokens, const_pointer_separator) + + return pointerString +} + +// Specific JSON pointer encoding here +// ~0 => ~ +// ~1 => / +// ... and vice versa + +func decodeReferenceToken(token string) string { + step1 := strings.Replace(token, `~1`, `/`, -1) + step2 := strings.Replace(step1, `~0`, `~`, -1) + return step2 +} + +func encodeReferenceToken(token string) string { + step1 := strings.Replace(token, `~`, `~0`, -1) + step2 := strings.Replace(step1, `/`, `~1`, -1) + return step2 +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go new file mode 100644 index 00000000000..6457291301d --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go @@ -0,0 +1,147 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonreference +// repository-desc An implementation of JSON Reference - Go language +// +// description Main and unique file. +// +// created 26-02-2013 + +package gojsonreference + +import ( + "errors" + "net/url" + "path/filepath" + "runtime" + "strings" + + "github.com/xeipuuv/gojsonpointer" +) + +const ( + const_fragment_char = `#` +) + +func NewJsonReference(jsonReferenceString string) (JsonReference, error) { + + var r JsonReference + err := r.parse(jsonReferenceString) + return r, err + +} + +type JsonReference struct { + referenceUrl *url.URL + referencePointer gojsonpointer.JsonPointer + + HasFullUrl bool + HasUrlPathOnly bool + HasFragmentOnly bool + HasFileScheme bool + HasFullFilePath bool +} + +func (r *JsonReference) GetUrl() *url.URL { + return r.referenceUrl +} + +func (r *JsonReference) GetPointer() *gojsonpointer.JsonPointer { + return &r.referencePointer +} + +func (r *JsonReference) String() string { + + if r.referenceUrl != nil { + return r.referenceUrl.String() + } + + if r.HasFragmentOnly { + return const_fragment_char + r.referencePointer.String() + } + + return r.referencePointer.String() +} + +func (r *JsonReference) IsCanonical() bool { + return (r.HasFileScheme && r.HasFullFilePath) || (!r.HasFileScheme && r.HasFullUrl) +} + +// "Constructor", parses the given string JSON reference +func (r *JsonReference) parse(jsonReferenceString string) (err error) { + + r.referenceUrl, err = url.Parse(jsonReferenceString) + if err != nil { + return + } + refUrl := r.referenceUrl + + if refUrl.Scheme != "" && refUrl.Host != "" { + r.HasFullUrl = true + } else { + if refUrl.Path != "" { + r.HasUrlPathOnly = true + } else if refUrl.RawQuery == "" && refUrl.Fragment != "" { + r.HasFragmentOnly = true + } + } + + r.HasFileScheme = refUrl.Scheme == "file" + if runtime.GOOS == "windows" { + // on Windows, a file URL may have an extra leading slash, and if it + // doesn't then its first component will be treated as the host by the + // Go runtime + if refUrl.Host == "" && strings.HasPrefix(refUrl.Path, "/") { + r.HasFullFilePath = filepath.IsAbs(refUrl.Path[1:]) + } else { + r.HasFullFilePath = filepath.IsAbs(refUrl.Host + refUrl.Path) + } + } else { + r.HasFullFilePath = filepath.IsAbs(refUrl.Path) + } + + // invalid json-pointer error means url has no json-pointer fragment. simply ignore error + r.referencePointer, _ = gojsonpointer.NewJsonPointer(refUrl.Fragment) + + return +} + +// Creates a new reference from a parent and a child +// If the child cannot inherit from the parent, an error is returned +func (r *JsonReference) Inherits(child JsonReference) (*JsonReference, error) { + if child.GetUrl() == nil { + return nil, errors.New("childUrl is nil!") + } + + if r.GetUrl() == nil { + return nil, errors.New("parentUrl is nil!") + } + + // Get a copy of the parent url to make sure we do not modify the original. + // URL reference resolving fails if the fragment of the child is empty, but the parent's is not. + // The fragment of the child must be used, so the fragment of the parent is manually removed. + parentUrl := *r.GetUrl() + parentUrl.Fragment = "" + + ref, err := NewJsonReference(parentUrl.ResolveReference(child.GetUrl()).String()) + if err != nil { + return nil, err + } + return &ref, err +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore new file mode 100644 index 00000000000..68e993ce3e0 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore @@ -0,0 +1,3 @@ +*.sw[nop] +*.iml +.vscode/ diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go new file mode 100644 index 00000000000..2f326a2228b --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go @@ -0,0 +1,324 @@ +package gojsonschema + +import ( + "bytes" + "sync" + "text/template" +) + +var errorTemplates errorTemplate = errorTemplate{template.New("errors-new"), sync.RWMutex{}} + +// template.Template is not thread-safe for writing, so some locking is done +// sync.RWMutex is used for efficiently locking when new templates are created +type errorTemplate struct { + *template.Template + sync.RWMutex +} + +type ( + // RequiredError. ErrorDetails: property string + RequiredError struct { + ResultErrorFields + } + + // InvalidTypeError. ErrorDetails: expected, given + InvalidTypeError struct { + ResultErrorFields + } + + // NumberAnyOfError. ErrorDetails: - + NumberAnyOfError struct { + ResultErrorFields + } + + // NumberOneOfError. ErrorDetails: - + NumberOneOfError struct { + ResultErrorFields + } + + // NumberAllOfError. ErrorDetails: - + NumberAllOfError struct { + ResultErrorFields + } + + // NumberNotError. ErrorDetails: - + NumberNotError struct { + ResultErrorFields + } + + // MissingDependencyError. ErrorDetails: dependency + MissingDependencyError struct { + ResultErrorFields + } + + // InternalError. ErrorDetails: error + InternalError struct { + ResultErrorFields + } + + // ConstError. ErrorDetails: allowed + ConstError struct { + ResultErrorFields + } + + // EnumError. ErrorDetails: allowed + EnumError struct { + ResultErrorFields + } + + // ArrayNoAdditionalItemsError. ErrorDetails: - + ArrayNoAdditionalItemsError struct { + ResultErrorFields + } + + // ArrayMinItemsError. ErrorDetails: min + ArrayMinItemsError struct { + ResultErrorFields + } + + // ArrayMaxItemsError. ErrorDetails: max + ArrayMaxItemsError struct { + ResultErrorFields + } + + // ItemsMustBeUniqueError. ErrorDetails: type + ItemsMustBeUniqueError struct { + ResultErrorFields + } + + // ArrayContainsError. ErrorDetails: + ArrayContainsError struct { + ResultErrorFields + } + + // ArrayMinPropertiesError. ErrorDetails: min + ArrayMinPropertiesError struct { + ResultErrorFields + } + + // ArrayMaxPropertiesError. ErrorDetails: max + ArrayMaxPropertiesError struct { + ResultErrorFields + } + + // AdditionalPropertyNotAllowedError. ErrorDetails: property + AdditionalPropertyNotAllowedError struct { + ResultErrorFields + } + + // InvalidPropertyPatternError. ErrorDetails: property, pattern + InvalidPropertyPatternError struct { + ResultErrorFields + } + + // InvalidPopertyNameError. ErrorDetails: property + InvalidPropertyNameError struct { + ResultErrorFields + } + + // StringLengthGTEError. ErrorDetails: min + StringLengthGTEError struct { + ResultErrorFields + } + + // StringLengthLTEError. ErrorDetails: max + StringLengthLTEError struct { + ResultErrorFields + } + + // DoesNotMatchPatternError. ErrorDetails: pattern + DoesNotMatchPatternError struct { + ResultErrorFields + } + + // DoesNotMatchFormatError. ErrorDetails: format + DoesNotMatchFormatError struct { + ResultErrorFields + } + + // MultipleOfError. ErrorDetails: multiple + MultipleOfError struct { + ResultErrorFields + } + + // NumberGTEError. ErrorDetails: min + NumberGTEError struct { + ResultErrorFields + } + + // NumberGTError. ErrorDetails: min + NumberGTError struct { + ResultErrorFields + } + + // NumberLTEError. ErrorDetails: max + NumberLTEError struct { + ResultErrorFields + } + + // NumberLTError. ErrorDetails: max + NumberLTError struct { + ResultErrorFields + } + + // ConditionThenError. ErrorDetails: - + ConditionThenError struct { + ResultErrorFields + } + + // ConditionElseError. ErrorDetails: - + ConditionElseError struct { + ResultErrorFields + } +) + +// newError takes a ResultError type and sets the type, context, description, details, value, and field +func newError(err ResultError, context *JsonContext, value interface{}, locale locale, details ErrorDetails) { + var t string + var d string + switch err.(type) { + case *RequiredError: + t = "required" + d = locale.Required() + case *InvalidTypeError: + t = "invalid_type" + d = locale.InvalidType() + case *NumberAnyOfError: + t = "number_any_of" + d = locale.NumberAnyOf() + case *NumberOneOfError: + t = "number_one_of" + d = locale.NumberOneOf() + case *NumberAllOfError: + t = "number_all_of" + d = locale.NumberAllOf() + case *NumberNotError: + t = "number_not" + d = locale.NumberNot() + case *MissingDependencyError: + t = "missing_dependency" + d = locale.MissingDependency() + case *InternalError: + t = "internal" + d = locale.Internal() + case *ConstError: + t = "const" + d = locale.Const() + case *EnumError: + t = "enum" + d = locale.Enum() + case *ArrayNoAdditionalItemsError: + t = "array_no_additional_items" + d = locale.ArrayNoAdditionalItems() + case *ArrayMinItemsError: + t = "array_min_items" + d = locale.ArrayMinItems() + case *ArrayMaxItemsError: + t = "array_max_items" + d = locale.ArrayMaxItems() + case *ItemsMustBeUniqueError: + t = "unique" + d = locale.Unique() + case *ArrayContainsError: + t = "contains" + d = locale.ArrayContains() + case *ArrayMinPropertiesError: + t = "array_min_properties" + d = locale.ArrayMinProperties() + case *ArrayMaxPropertiesError: + t = "array_max_properties" + d = locale.ArrayMaxProperties() + case *AdditionalPropertyNotAllowedError: + t = "additional_property_not_allowed" + d = locale.AdditionalPropertyNotAllowed() + case *InvalidPropertyPatternError: + t = "invalid_property_pattern" + d = locale.InvalidPropertyPattern() + case *InvalidPropertyNameError: + t = "invalid_property_name" + d = locale.InvalidPropertyName() + case *StringLengthGTEError: + t = "string_gte" + d = locale.StringGTE() + case *StringLengthLTEError: + t = "string_lte" + d = locale.StringLTE() + case *DoesNotMatchPatternError: + t = "pattern" + d = locale.DoesNotMatchPattern() + case *DoesNotMatchFormatError: + t = "format" + d = locale.DoesNotMatchFormat() + case *MultipleOfError: + t = "multiple_of" + d = locale.MultipleOf() + case *NumberGTEError: + t = "number_gte" + d = locale.NumberGTE() + case *NumberGTError: + t = "number_gt" + d = locale.NumberGT() + case *NumberLTEError: + t = "number_lte" + d = locale.NumberLTE() + case *NumberLTError: + t = "number_lt" + d = locale.NumberLT() + case *ConditionThenError: + t = "condition_then" + d = locale.ConditionThen() + case *ConditionElseError: + t = "condition_else" + d = locale.ConditionElse() + } + + err.SetType(t) + err.SetContext(context) + err.SetValue(value) + err.SetDetails(details) + err.SetDescriptionFormat(d) + details["field"] = err.Field() + + if _, exists := details["context"]; !exists && context != nil { + details["context"] = context.String() + } + + err.SetDescription(formatErrorDescription(err.DescriptionFormat(), details)) +} + +// formatErrorDescription takes a string in the default text/template +// format and converts it to a string with replacements. The fields come +// from the ErrorDetails struct and vary for each type of error. +func formatErrorDescription(s string, details ErrorDetails) string { + + var tpl *template.Template + var descrAsBuffer bytes.Buffer + var err error + + errorTemplates.RLock() + tpl = errorTemplates.Lookup(s) + errorTemplates.RUnlock() + + if tpl == nil { + errorTemplates.Lock() + tpl = errorTemplates.New(s) + + if ErrorTemplateFuncs != nil { + tpl.Funcs(ErrorTemplateFuncs) + } + + tpl, err = tpl.Parse(s) + errorTemplates.Unlock() + + if err != nil { + return err.Error() + } + } + + err = tpl.Execute(&descrAsBuffer, details) + if err != nil { + return err.Error() + } + + return descrAsBuffer.String() +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go new file mode 100644 index 00000000000..26217fca128 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go @@ -0,0 +1,343 @@ +package gojsonschema + +import ( + "net" + "net/mail" + "net/url" + "regexp" + "strings" + "sync" + "time" +) + +type ( + // FormatChecker is the interface all formatters added to FormatCheckerChain must implement + FormatChecker interface { + IsFormat(input interface{}) bool + } + + // FormatCheckerChain holds the formatters + FormatCheckerChain struct { + formatters map[string]FormatChecker + } + + // EmailFormatter verifies email address formats + EmailFormatChecker struct{} + + // IPV4FormatChecker verifies IP addresses in the ipv4 format + IPV4FormatChecker struct{} + + // IPV6FormatChecker verifies IP addresses in the ipv6 format + IPV6FormatChecker struct{} + + // DateTimeFormatChecker verifies date/time formats per RFC3339 5.6 + // + // Valid formats: + // Partial Time: HH:MM:SS + // Full Date: YYYY-MM-DD + // Full Time: HH:MM:SSZ-07:00 + // Date Time: YYYY-MM-DDTHH:MM:SSZ-0700 + // + // Where + // YYYY = 4DIGIT year + // MM = 2DIGIT month ; 01-12 + // DD = 2DIGIT day-month ; 01-28, 01-29, 01-30, 01-31 based on month/year + // HH = 2DIGIT hour ; 00-23 + // MM = 2DIGIT ; 00-59 + // SS = 2DIGIT ; 00-58, 00-60 based on leap second rules + // T = Literal + // Z = Literal + // + // Note: Nanoseconds are also suported in all formats + // + // http://tools.ietf.org/html/rfc3339#section-5.6 + DateTimeFormatChecker struct{} + + DateFormatChecker struct{} + + TimeFormatChecker struct{} + + // URIFormatChecker validates a URI with a valid Scheme per RFC3986 + URIFormatChecker struct{} + + // URIReferenceFormatChecker validates a URI or relative-reference per RFC3986 + URIReferenceFormatChecker struct{} + + // URITemplateFormatChecker validates a URI template per RFC6570 + URITemplateFormatChecker struct{} + + // HostnameFormatChecker validates a hostname is in the correct format + HostnameFormatChecker struct{} + + // UUIDFormatChecker validates a UUID is in the correct format + UUIDFormatChecker struct{} + + // RegexFormatChecker validates a regex is in the correct format + RegexFormatChecker struct{} + + // JSONPointerFormatChecker validates a JSON Pointer per RFC6901 + JSONPointerFormatChecker struct{} + + // RelativeJSONPointerFormatChecker validates a relative JSON Pointer is in the correct format + RelativeJSONPointerFormatChecker struct{} +) + +var ( + // Formatters holds the valid formatters, and is a public variable + // so library users can add custom formatters + FormatCheckers = FormatCheckerChain{ + formatters: map[string]FormatChecker{ + "date": DateFormatChecker{}, + "time": TimeFormatChecker{}, + "date-time": DateTimeFormatChecker{}, + "hostname": HostnameFormatChecker{}, + "email": EmailFormatChecker{}, + "idn-email": EmailFormatChecker{}, + "ipv4": IPV4FormatChecker{}, + "ipv6": IPV6FormatChecker{}, + "uri": URIFormatChecker{}, + "uri-reference": URIReferenceFormatChecker{}, + "iri": URIFormatChecker{}, + "iri-reference": URIReferenceFormatChecker{}, + "uri-template": URITemplateFormatChecker{}, + "uuid": UUIDFormatChecker{}, + "regex": RegexFormatChecker{}, + "json-pointer": JSONPointerFormatChecker{}, + "relative-json-pointer": RelativeJSONPointerFormatChecker{}, + }, + } + + // Regex credit: https://www.socketloop.com/tutorials/golang-validate-hostname + rxHostname = regexp.MustCompile(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`) + + // Use a regex to make sure curly brackets are balanced properly after validating it as a AURI + rxURITemplate = regexp.MustCompile("^([^{]*({[^}]*})?)*$") + + rxUUID = regexp.MustCompile("^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$") + + rxJSONPointer = regexp.MustCompile("^(?:/(?:[^~/]|~0|~1)*)*$") + + rxRelJSONPointer = regexp.MustCompile("^(?:0|[1-9][0-9]*)(?:#|(?:/(?:[^~/]|~0|~1)*)*)$") + + lock = new(sync.Mutex) +) + +// Add adds a FormatChecker to the FormatCheckerChain +// The name used will be the value used for the format key in your json schema +func (c *FormatCheckerChain) Add(name string, f FormatChecker) *FormatCheckerChain { + lock.Lock() + c.formatters[name] = f + lock.Unlock() + + return c +} + +// Remove deletes a FormatChecker from the FormatCheckerChain (if it exists) +func (c *FormatCheckerChain) Remove(name string) *FormatCheckerChain { + lock.Lock() + delete(c.formatters, name) + lock.Unlock() + + return c +} + +// Has checks to see if the FormatCheckerChain holds a FormatChecker with the given name +func (c *FormatCheckerChain) Has(name string) bool { + lock.Lock() + _, ok := c.formatters[name] + lock.Unlock() + + return ok +} + +// IsFormat will check an input against a FormatChecker with the given name +// to see if it is the correct format +func (c *FormatCheckerChain) IsFormat(name string, input interface{}) bool { + f, ok := c.formatters[name] + + if !ok { + return false + } + + return f.IsFormat(input) +} + +func (f EmailFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + _, err := mail.ParseAddress(asString) + + return err == nil +} + +// Credit: https://github.com/asaskevich/govalidator +func (f IPV4FormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + ip := net.ParseIP(asString) + return ip != nil && strings.Contains(asString, ".") +} + +// Credit: https://github.com/asaskevich/govalidator +func (f IPV6FormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + ip := net.ParseIP(asString) + return ip != nil && strings.Contains(asString, ":") +} + +func (f DateTimeFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + formats := []string{ + "15:04:05", + "15:04:05Z07:00", + "2006-01-02", + time.RFC3339, + time.RFC3339Nano, + } + + for _, format := range formats { + if _, err := time.Parse(format, asString); err == nil { + return true + } + } + + return false +} + +func (f DateFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + _, err := time.Parse("2006-01-02", asString) + return err == nil +} + +func (f TimeFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + if _, err := time.Parse("15:04:05Z07:00", asString); err == nil { + return true + } + + _, err := time.Parse("15:04:05", asString) + return err == nil +} + +func (f URIFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + u, err := url.Parse(asString) + + if err != nil || u.Scheme == "" { + return false + } + + return !strings.Contains(asString, `\`) +} + +func (f URIReferenceFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + _, err := url.Parse(asString) + return err == nil && !strings.Contains(asString, `\`) +} + +func (f URITemplateFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + u, err := url.Parse(asString) + if err != nil || strings.Contains(asString, `\`) { + return false + } + + return rxURITemplate.MatchString(u.Path) +} + +func (f HostnameFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxHostname.MatchString(asString) && len(asString) < 256 +} + +func (f UUIDFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxUUID.MatchString(asString) +} + +// IsFormat implements FormatChecker interface. +func (f RegexFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + if asString == "" { + return true + } + _, err := regexp.Compile(asString) + if err != nil { + return false + } + return true +} + +func (f JSONPointerFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + return rxJSONPointer.MatchString(asString) +} + +func (f RelativeJSONPointerFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + return rxRelJSONPointer.MatchString(asString) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go new file mode 100644 index 00000000000..4ef7a8d03e7 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go @@ -0,0 +1,37 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Very simple log wrapper. +// Used for debugging/testing purposes. +// +// created 01-01-2015 + +package gojsonschema + +import ( + "log" +) + +const internalLogEnabled = false + +func internalLog(format string, v ...interface{}) { + log.Printf(format, v...) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go new file mode 100644 index 00000000000..f40668a74c4 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go @@ -0,0 +1,72 @@ +// Copyright 2013 MongoDB, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author tolsen +// author-github https://github.com/tolsen +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Implements a persistent (immutable w/ shared structure) singly-linked list of strings for the purpose of storing a json context +// +// created 04-09-2013 + +package gojsonschema + +import "bytes" + +// JsonContext implements a persistent linked-list of strings +type JsonContext struct { + head string + tail *JsonContext +} + +func NewJsonContext(head string, tail *JsonContext) *JsonContext { + return &JsonContext{head, tail} +} + +// String displays the context in reverse. +// This plays well with the data structure's persistent nature with +// Cons and a json document's tree structure. +func (c *JsonContext) String(del ...string) string { + byteArr := make([]byte, 0, c.stringLen()) + buf := bytes.NewBuffer(byteArr) + c.writeStringToBuffer(buf, del) + + return buf.String() +} + +func (c *JsonContext) stringLen() int { + length := 0 + if c.tail != nil { + length = c.tail.stringLen() + 1 // add 1 for "." + } + + length += len(c.head) + return length +} + +func (c *JsonContext) writeStringToBuffer(buf *bytes.Buffer, del []string) { + if c.tail != nil { + c.tail.writeStringToBuffer(buf, del) + + if len(del) > 0 { + buf.WriteString(del[0]) + } else { + buf.WriteString(".") + } + } + + buf.WriteString(c.head) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go new file mode 100644 index 00000000000..cbe46649218 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go @@ -0,0 +1,362 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Different strategies to load JSON files. +// Includes References (file and HTTP), JSON strings and Go types. +// +// created 01-02-2015 + +package gojsonschema + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/xeipuuv/gojsonreference" +) + +var osFS = osFileSystem(os.Open) + +// JSON loader interface + +type JSONLoader interface { + JsonSource() interface{} + LoadJSON() (interface{}, error) + JsonReference() (gojsonreference.JsonReference, error) + LoaderFactory() JSONLoaderFactory +} + +type JSONLoaderFactory interface { + New(source string) JSONLoader +} + +type DefaultJSONLoaderFactory struct { +} + +type FileSystemJSONLoaderFactory struct { + fs http.FileSystem +} + +func (d DefaultJSONLoaderFactory) New(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: osFS, + source: source, + } +} + +func (f FileSystemJSONLoaderFactory) New(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: f.fs, + source: source, + } +} + +// osFileSystem is a functional wrapper for os.Open that implements http.FileSystem. +type osFileSystem func(string) (*os.File, error) + +func (o osFileSystem) Open(name string) (http.File, error) { + return o(name) +} + +// JSON Reference loader +// references are used to load JSONs from files and HTTP + +type jsonReferenceLoader struct { + fs http.FileSystem + source string +} + +func (l *jsonReferenceLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonReferenceLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference(l.JsonSource().(string)) +} + +func (l *jsonReferenceLoader) LoaderFactory() JSONLoaderFactory { + return &FileSystemJSONLoaderFactory{ + fs: l.fs, + } +} + +// NewReferenceLoader returns a JSON reference loader using the given source and the local OS file system. +func NewReferenceLoader(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: osFS, + source: source, + } +} + +// NewReferenceLoaderFileSystem returns a JSON reference loader using the given source and file system. +func NewReferenceLoaderFileSystem(source string, fs http.FileSystem) JSONLoader { + return &jsonReferenceLoader{ + fs: fs, + source: source, + } +} + +func (l *jsonReferenceLoader) LoadJSON() (interface{}, error) { + + var err error + + reference, err := gojsonreference.NewJsonReference(l.JsonSource().(string)) + if err != nil { + return nil, err + } + + refToUrl := reference + refToUrl.GetUrl().Fragment = "" + + var document interface{} + + if reference.HasFileScheme { + + filename := strings.TrimPrefix(refToUrl.String(), "file://") + if runtime.GOOS == "windows" { + // on Windows, a file URL may have an extra leading slash, use slashes + // instead of backslashes, and have spaces escaped + filename = strings.TrimPrefix(filename, "/") + filename = filepath.FromSlash(filename) + } + + document, err = l.loadFromFile(filename) + if err != nil { + return nil, err + } + + } else { + + document, err = l.loadFromHTTP(refToUrl.String()) + if err != nil { + return nil, err + } + + } + + return document, nil + +} + +func (l *jsonReferenceLoader) loadFromHTTP(address string) (interface{}, error) { + + resp, err := http.Get(address) + if err != nil { + return nil, err + } + + // must return HTTP Status 200 OK + if resp.StatusCode != http.StatusOK { + return nil, errors.New(formatErrorDescription(Locale.HttpBadStatus(), ErrorDetails{"status": resp.Status})) + } + + bodyBuff, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) + +} + +func (l *jsonReferenceLoader) loadFromFile(path string) (interface{}, error) { + f, err := l.fs.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + bodyBuff, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) + +} + +// JSON string loader + +type jsonStringLoader struct { + source string +} + +func (l *jsonStringLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonStringLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonStringLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewStringLoader(source string) JSONLoader { + return &jsonStringLoader{source: source} +} + +func (l *jsonStringLoader) LoadJSON() (interface{}, error) { + + return decodeJsonUsingNumber(strings.NewReader(l.JsonSource().(string))) + +} + +// JSON bytes loader + +type jsonBytesLoader struct { + source []byte +} + +func (l *jsonBytesLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonBytesLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonBytesLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewBytesLoader(source []byte) JSONLoader { + return &jsonBytesLoader{source: source} +} + +func (l *jsonBytesLoader) LoadJSON() (interface{}, error) { + return decodeJsonUsingNumber(bytes.NewReader(l.JsonSource().([]byte))) +} + +// JSON Go (types) loader +// used to load JSONs from the code as maps, interface{}, structs ... + +type jsonGoLoader struct { + source interface{} +} + +func (l *jsonGoLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonGoLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonGoLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewGoLoader(source interface{}) JSONLoader { + return &jsonGoLoader{source: source} +} + +func (l *jsonGoLoader) LoadJSON() (interface{}, error) { + + // convert it to a compliant JSON first to avoid types "mismatches" + + jsonBytes, err := json.Marshal(l.JsonSource()) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(jsonBytes)) + +} + +type jsonIOLoader struct { + buf *bytes.Buffer +} + +func NewReaderLoader(source io.Reader) (JSONLoader, io.Reader) { + buf := &bytes.Buffer{} + return &jsonIOLoader{buf: buf}, io.TeeReader(source, buf) +} + +func NewWriterLoader(source io.Writer) (JSONLoader, io.Writer) { + buf := &bytes.Buffer{} + return &jsonIOLoader{buf: buf}, io.MultiWriter(source, buf) +} + +func (l *jsonIOLoader) JsonSource() interface{} { + return l.buf.String() +} + +func (l *jsonIOLoader) LoadJSON() (interface{}, error) { + return decodeJsonUsingNumber(l.buf) +} + +func (l *jsonIOLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonIOLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +// JSON raw loader +// In case the JSON is already marshalled to interface{} use this loader +// This is used for testing as otherwise there is no guarantee the JSON is marshalled +// "properly" by using https://golang.org/pkg/encoding/json/#Decoder.UseNumber +type jsonRawLoader struct { + source interface{} +} + +func NewRawLoader(source interface{}) *jsonRawLoader { + return &jsonRawLoader{source: source} +} +func (l *jsonRawLoader) JsonSource() interface{} { + return l.source +} +func (l *jsonRawLoader) LoadJSON() (interface{}, error) { + return l.source, nil +} +func (l *jsonRawLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} +func (l *jsonRawLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func decodeJsonUsingNumber(r io.Reader) (interface{}, error) { + + var document interface{} + + decoder := json.NewDecoder(r) + decoder.UseNumber() + + err := decoder.Decode(&document) + if err != nil { + return nil, err + } + + return document, nil + +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go new file mode 100644 index 00000000000..7bd9a9dce59 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go @@ -0,0 +1,313 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Contains const string and messages. +// +// created 01-01-2015 + +package gojsonschema + +type ( + // locale is an interface for defining custom error strings + locale interface { + Required() string + InvalidType() string + NumberAnyOf() string + NumberOneOf() string + NumberAllOf() string + NumberNot() string + MissingDependency() string + Internal() string + Const() string + Enum() string + ArrayNotEnoughItems() string + ArrayNoAdditionalItems() string + ArrayMinItems() string + ArrayMaxItems() string + Unique() string + ArrayContains() string + ArrayMinProperties() string + ArrayMaxProperties() string + AdditionalPropertyNotAllowed() string + InvalidPropertyPattern() string + InvalidPropertyName() string + StringGTE() string + StringLTE() string + DoesNotMatchPattern() string + DoesNotMatchFormat() string + MultipleOf() string + NumberGTE() string + NumberGT() string + NumberLTE() string + NumberLT() string + + // Schema validations + RegexPattern() string + GreaterThanZero() string + MustBeOfA() string + MustBeOfAn() string + CannotBeUsedWithout() string + CannotBeGT() string + MustBeOfType() string + MustBeValidRegex() string + MustBeValidFormat() string + MustBeGTEZero() string + KeyCannotBeGreaterThan() string + KeyItemsMustBeOfType() string + KeyItemsMustBeUnique() string + ReferenceMustBeCanonical() string + NotAValidType() string + Duplicated() string + HttpBadStatus() string + ParseError() string + + ConditionThen() string + ConditionElse() string + + // ErrorFormat + ErrorFormat() string + } + + // DefaultLocale is the default locale for this package + DefaultLocale struct{} +) + +func (l DefaultLocale) Required() string { + return `{{.property}} is required` +} + +func (l DefaultLocale) InvalidType() string { + return `Invalid type. Expected: {{.expected}}, given: {{.given}}` +} + +func (l DefaultLocale) NumberAnyOf() string { + return `Must validate at least one schema (anyOf)` +} + +func (l DefaultLocale) NumberOneOf() string { + return `Must validate one and only one schema (oneOf)` +} + +func (l DefaultLocale) NumberAllOf() string { + return `Must validate all the schemas (allOf)` +} + +func (l DefaultLocale) NumberNot() string { + return `Must not validate the schema (not)` +} + +func (l DefaultLocale) MissingDependency() string { + return `Has a dependency on {{.dependency}}` +} + +func (l DefaultLocale) Internal() string { + return `Internal Error {{.error}}` +} + +func (l DefaultLocale) Const() string { + return `{{.field}} does not match: {{.allowed}}` +} + +func (l DefaultLocale) Enum() string { + return `{{.field}} must be one of the following: {{.allowed}}` +} + +func (l DefaultLocale) ArrayNoAdditionalItems() string { + return `No additional items allowed on array` +} + +func (l DefaultLocale) ArrayNotEnoughItems() string { + return `Not enough items on array to match positional list of schema` +} + +func (l DefaultLocale) ArrayMinItems() string { + return `Array must have at least {{.min}} items` +} + +func (l DefaultLocale) ArrayMaxItems() string { + return `Array must have at most {{.max}} items` +} + +func (l DefaultLocale) Unique() string { + return `{{.type}} items must be unique` +} + +func (l DefaultLocale) ArrayContains() string { + return `At least one of the items must match` +} + +func (l DefaultLocale) ArrayMinProperties() string { + return `Must have at least {{.min}} properties` +} + +func (l DefaultLocale) ArrayMaxProperties() string { + return `Must have at most {{.max}} properties` +} + +func (l DefaultLocale) AdditionalPropertyNotAllowed() string { + return `Additional property {{.property}} is not allowed` +} + +func (l DefaultLocale) InvalidPropertyPattern() string { + return `Property "{{.property}}" does not match pattern {{.pattern}}` +} + +func (l DefaultLocale) InvalidPropertyName() string { + return `Property name of "{{.property}}" does not match` +} + +func (l DefaultLocale) StringGTE() string { + return `String length must be greater than or equal to {{.min}}` +} + +func (l DefaultLocale) StringLTE() string { + return `String length must be less than or equal to {{.max}}` +} + +func (l DefaultLocale) DoesNotMatchPattern() string { + return `Does not match pattern '{{.pattern}}'` +} + +func (l DefaultLocale) DoesNotMatchFormat() string { + return `Does not match format '{{.format}}'` +} + +func (l DefaultLocale) MultipleOf() string { + return `Must be a multiple of {{.multiple}}` +} + +func (l DefaultLocale) NumberGTE() string { + return `Must be greater than or equal to {{.min}}` +} + +func (l DefaultLocale) NumberGT() string { + return `Must be greater than {{.min}}` +} + +func (l DefaultLocale) NumberLTE() string { + return `Must be less than or equal to {{.max}}` +} + +func (l DefaultLocale) NumberLT() string { + return `Must be less than {{.max}}` +} + +// Schema validators +func (l DefaultLocale) RegexPattern() string { + return `Invalid regex pattern '{{.pattern}}'` +} + +func (l DefaultLocale) GreaterThanZero() string { + return `{{.number}} must be strictly greater than 0` +} + +func (l DefaultLocale) MustBeOfA() string { + return `{{.x}} must be of a {{.y}}` +} + +func (l DefaultLocale) MustBeOfAn() string { + return `{{.x}} must be of an {{.y}}` +} + +func (l DefaultLocale) CannotBeUsedWithout() string { + return `{{.x}} cannot be used without {{.y}}` +} + +func (l DefaultLocale) CannotBeGT() string { + return `{{.x}} cannot be greater than {{.y}}` +} + +func (l DefaultLocale) MustBeOfType() string { + return `{{.key}} must be of type {{.type}}` +} + +func (l DefaultLocale) MustBeValidRegex() string { + return `{{.key}} must be a valid regex` +} + +func (l DefaultLocale) MustBeValidFormat() string { + return `{{.key}} must be a valid format {{.given}}` +} + +func (l DefaultLocale) MustBeGTEZero() string { + return `{{.key}} must be greater than or equal to 0` +} + +func (l DefaultLocale) KeyCannotBeGreaterThan() string { + return `{{.key}} cannot be greater than {{.y}}` +} + +func (l DefaultLocale) KeyItemsMustBeOfType() string { + return `{{.key}} items must be {{.type}}` +} + +func (l DefaultLocale) KeyItemsMustBeUnique() string { + return `{{.key}} items must be unique` +} + +func (l DefaultLocale) ReferenceMustBeCanonical() string { + return `Reference {{.reference}} must be canonical` +} + +func (l DefaultLocale) NotAValidType() string { + return `has a primitive type that is NOT VALID -- given: {{.given}} Expected valid values are:{{.expected}}` +} + +func (l DefaultLocale) Duplicated() string { + return `{{.type}} type is duplicated` +} + +func (l DefaultLocale) HttpBadStatus() string { + return `Could not read schema from HTTP, response status is {{.status}}` +} + +// Replacement options: field, description, context, value +func (l DefaultLocale) ErrorFormat() string { + return `{{.field}}: {{.description}}` +} + +//Parse error +func (l DefaultLocale) ParseError() string { + return `Expected: {{.expected}}, given: Invalid JSON` +} + +//If/Else +func (l DefaultLocale) ConditionThen() string { + return `Must validate "then" as "if" was valid` +} + +func (l DefaultLocale) ConditionElse() string { + return `Must validate "else" as "if" was not valid` +} + +const ( + STRING_NUMBER = "number" + STRING_ARRAY_OF_STRINGS = "array of strings" + STRING_ARRAY_OF_SCHEMAS = "array of schemas" + STRING_SCHEMA = "valid schema" + STRING_SCHEMA_OR_ARRAY_OF_STRINGS = "schema or array of strings" + STRING_PROPERTIES = "properties" + STRING_DEPENDENCY = "dependency" + STRING_PROPERTY = "property" + STRING_UNDEFINED = "undefined" + STRING_CONTEXT_ROOT = "(root)" + STRING_ROOT_SCHEMA_PROPERTY = "(root)" +) diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go new file mode 100644 index 00000000000..7896f312990 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go @@ -0,0 +1,195 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Result and ResultError implementations. +// +// created 01-01-2015 + +package gojsonschema + +import ( + "fmt" + "strings" +) + +type ( + // ErrorDetails is a map of details specific to each error. + // While the values will vary, every error will contain a "field" value + ErrorDetails map[string]interface{} + + // ResultError is the interface that library errors must implement + ResultError interface { + Field() string + SetType(string) + Type() string + SetContext(*JsonContext) + Context() *JsonContext + SetDescription(string) + Description() string + SetDescriptionFormat(string) + DescriptionFormat() string + SetValue(interface{}) + Value() interface{} + SetDetails(ErrorDetails) + Details() ErrorDetails + String() string + } + + // ResultErrorFields holds the fields for each ResultError implementation. + // ResultErrorFields implements the ResultError interface, so custom errors + // can be defined by just embedding this type + ResultErrorFields struct { + errorType string // A string with the type of error (i.e. invalid_type) + context *JsonContext // Tree like notation of the part that failed the validation. ex (root).a.b ... + description string // A human readable error message + descriptionFormat string // A format for human readable error message + value interface{} // Value given by the JSON file that is the source of the error + details ErrorDetails + } + + Result struct { + errors []ResultError + // Scores how well the validation matched. Useful in generating + // better error messages for anyOf and oneOf. + score int + } +) + +// Field outputs the field name without the root context +// i.e. firstName or person.firstName instead of (root).firstName or (root).person.firstName +func (v *ResultErrorFields) Field() string { + if p, ok := v.Details()["property"]; ok { + if str, isString := p.(string); isString { + return str + } + } + + return strings.TrimPrefix(v.context.String(), STRING_ROOT_SCHEMA_PROPERTY+".") +} + +func (v *ResultErrorFields) SetType(errorType string) { + v.errorType = errorType +} + +func (v *ResultErrorFields) Type() string { + return v.errorType +} + +func (v *ResultErrorFields) SetContext(context *JsonContext) { + v.context = context +} + +func (v *ResultErrorFields) Context() *JsonContext { + return v.context +} + +func (v *ResultErrorFields) SetDescription(description string) { + v.description = description +} + +func (v *ResultErrorFields) Description() string { + return v.description +} + +func (v *ResultErrorFields) SetDescriptionFormat(descriptionFormat string) { + v.descriptionFormat = descriptionFormat +} + +func (v *ResultErrorFields) DescriptionFormat() string { + return v.descriptionFormat +} + +func (v *ResultErrorFields) SetValue(value interface{}) { + v.value = value +} + +func (v *ResultErrorFields) Value() interface{} { + return v.value +} + +func (v *ResultErrorFields) SetDetails(details ErrorDetails) { + v.details = details +} + +func (v *ResultErrorFields) Details() ErrorDetails { + return v.details +} + +func (v ResultErrorFields) String() string { + // as a fallback, the value is displayed go style + valueString := fmt.Sprintf("%v", v.value) + + // marshal the go value value to json + if v.value == nil { + valueString = TYPE_NULL + } else { + if vs, err := marshalToJsonString(v.value); err == nil { + if vs == nil { + valueString = TYPE_NULL + } else { + valueString = *vs + } + } + } + + return formatErrorDescription(Locale.ErrorFormat(), ErrorDetails{ + "context": v.context.String(), + "description": v.description, + "value": valueString, + "field": v.Field(), + }) +} + +func (v *Result) Valid() bool { + return len(v.errors) == 0 +} + +func (v *Result) Errors() []ResultError { + return v.errors +} + +// Add a fully filled error to the error set +// SetDescription() will be called with the result of the parsed err.DescriptionFormat() +func (v *Result) AddError(err ResultError, details ErrorDetails) { + if _, exists := details["context"]; !exists && err.Context() != nil { + details["context"] = err.Context().String() + } + + err.SetDescription(formatErrorDescription(err.DescriptionFormat(), details)) + + v.errors = append(v.errors, err) +} + +func (v *Result) addInternalError(err ResultError, context *JsonContext, value interface{}, details ErrorDetails) { + newError(err, context, value, Locale, details) + v.errors = append(v.errors, err) + v.score -= 2 // results in a net -1 when added to the +1 we get at the end of the validation function +} + +// Used to copy errors from a sub-schema to the main one +func (v *Result) mergeErrors(otherResult *Result) { + v.errors = append(v.errors, otherResult.Errors()...) + v.score += otherResult.score +} + +func (v *Result) incrementScore() { + v.score++ +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go new file mode 100644 index 00000000000..4ae3c62047a --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go @@ -0,0 +1,1000 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines Schema, the main entry to every subSchema. +// Contains the parsing logic and error checking. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "errors" + "math/big" + "reflect" + "regexp" + "text/template" + + "github.com/xeipuuv/gojsonreference" +) + +var ( + // Locale is the default locale to use + // Library users can overwrite with their own implementation + Locale locale = DefaultLocale{} + + // ErrorTemplateFuncs allows you to define custom template funcs for use in localization. + ErrorTemplateFuncs template.FuncMap +) + +func NewSchema(l JSONLoader) (*Schema, error) { + return NewSchemaLoader().Compile(l) +} + +type Schema struct { + documentReference gojsonreference.JsonReference + rootSchema *subSchema + pool *schemaPool + referencePool *schemaReferencePool +} + +func (d *Schema) parse(document interface{}) error { + d.rootSchema = &subSchema{property: STRING_ROOT_SCHEMA_PROPERTY} + return d.parseSchema(document, d.rootSchema) +} + +func (d *Schema) SetRootSchemaName(name string) { + d.rootSchema.property = name +} + +// Parses a subSchema +// +// Pretty long function ( sorry :) )... but pretty straight forward, repetitive and boring +// Not much magic involved here, most of the job is to validate the key names and their values, +// then the values are copied into subSchema struct +// +func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) error { + + // As of draft 6 "true" is equivalent to an empty schema "{}" and false equals "{"not":{}}" + if isKind(documentNode, reflect.Bool) { + b := documentNode.(bool) + if b { + documentNode = map[string]interface{}{} + } else { + documentNode = map[string]interface{}{"not": true} + } + } + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.ParseError(), + ErrorDetails{ + "expected": STRING_SCHEMA, + }, + )) + } + + m := documentNode.(map[string]interface{}) + + if currentSchema.parent == nil { + currentSchema.ref = &d.documentReference + currentSchema.id = &d.documentReference + + if existsMapKey(m, KEY_SCHEMA) && false { + if !isKind(m[KEY_SCHEMA], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_SCHEMA, + }, + )) + } + } + } + + if currentSchema.id == nil && currentSchema.parent != nil { + currentSchema.id = currentSchema.parent.id + } + + // In draft 6 the id keyword was renamed to $id + // Use the old id by default + keyID := KEY_ID_NEW + if existsMapKey(m, KEY_ID) { + keyID = KEY_ID + } + if existsMapKey(m, keyID) && !isKind(m[keyID], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": keyID, + }, + )) + } + if k, ok := m[keyID].(string); ok { + jsonReference, err := gojsonreference.NewJsonReference(k) + if err != nil { + return err + } + if currentSchema == d.rootSchema { + currentSchema.id = &jsonReference + } else { + ref, err := currentSchema.parent.id.Inherits(jsonReference) + if err != nil { + return err + } + currentSchema.id = ref + } + } + + // definitions + if existsMapKey(m, KEY_DEFINITIONS) { + if isKind(m[KEY_DEFINITIONS], reflect.Map, reflect.Bool) { + for _, dv := range m[KEY_DEFINITIONS].(map[string]interface{}) { + if isKind(dv, reflect.Map, reflect.Bool) { + + newSchema := &subSchema{property: KEY_DEFINITIONS, parent: currentSchema} + + err := d.parseSchema(dv, newSchema) + + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_ARRAY_OF_SCHEMAS, + "given": KEY_DEFINITIONS, + }, + )) + } + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_ARRAY_OF_SCHEMAS, + "given": KEY_DEFINITIONS, + }, + )) + } + + } + + // title + if existsMapKey(m, KEY_TITLE) && !isKind(m[KEY_TITLE], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_TITLE, + }, + )) + } + if k, ok := m[KEY_TITLE].(string); ok { + currentSchema.title = &k + } + + // description + if existsMapKey(m, KEY_DESCRIPTION) && !isKind(m[KEY_DESCRIPTION], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_DESCRIPTION, + }, + )) + } + if k, ok := m[KEY_DESCRIPTION].(string); ok { + currentSchema.description = &k + } + + // $ref + if existsMapKey(m, KEY_REF) && !isKind(m[KEY_REF], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_REF, + }, + )) + } + + if k, ok := m[KEY_REF].(string); ok { + + jsonReference, err := gojsonreference.NewJsonReference(k) + if err != nil { + return err + } + + currentSchema.ref = &jsonReference + + if sch, ok := d.referencePool.Get(currentSchema.ref.String()); ok { + currentSchema.refSchema = sch + } else { + err := d.parseReference(documentNode, currentSchema) + + if err != nil { + return err + } + + return nil + } + } + + // type + if existsMapKey(m, KEY_TYPE) { + if isKind(m[KEY_TYPE], reflect.String) { + if k, ok := m[KEY_TYPE].(string); ok { + err := currentSchema.types.Add(k) + if err != nil { + return err + } + } + } else { + if isKind(m[KEY_TYPE], reflect.Slice) { + arrayOfTypes := m[KEY_TYPE].([]interface{}) + for _, typeInArray := range arrayOfTypes { + if reflect.ValueOf(typeInArray).Kind() != reflect.String { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING + "/" + STRING_ARRAY_OF_STRINGS, + "given": KEY_TYPE, + }, + )) + } else { + currentSchema.types.Add(typeInArray.(string)) + } + } + + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING + "/" + STRING_ARRAY_OF_STRINGS, + "given": KEY_TYPE, + }, + )) + } + } + } + + // properties + if existsMapKey(m, KEY_PROPERTIES) { + err := d.parseProperties(m[KEY_PROPERTIES], currentSchema) + if err != nil { + return err + } + } + + // additionalProperties + if existsMapKey(m, KEY_ADDITIONAL_PROPERTIES) { + if isKind(m[KEY_ADDITIONAL_PROPERTIES], reflect.Bool) { + currentSchema.additionalProperties = m[KEY_ADDITIONAL_PROPERTIES].(bool) + } else if isKind(m[KEY_ADDITIONAL_PROPERTIES], reflect.Map) { + newSchema := &subSchema{property: KEY_ADDITIONAL_PROPERTIES, parent: currentSchema, ref: currentSchema.ref} + currentSchema.additionalProperties = newSchema + err := d.parseSchema(m[KEY_ADDITIONAL_PROPERTIES], newSchema) + if err != nil { + return errors.New(err.Error()) + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + STRING_SCHEMA, + "given": KEY_ADDITIONAL_PROPERTIES, + }, + )) + } + } + + // patternProperties + if existsMapKey(m, KEY_PATTERN_PROPERTIES) { + if isKind(m[KEY_PATTERN_PROPERTIES], reflect.Map) { + patternPropertiesMap := m[KEY_PATTERN_PROPERTIES].(map[string]interface{}) + if len(patternPropertiesMap) > 0 { + currentSchema.patternProperties = make(map[string]*subSchema) + for k, v := range patternPropertiesMap { + _, err := regexp.MatchString(k, "") + if err != nil { + return errors.New(formatErrorDescription( + Locale.RegexPattern(), + ErrorDetails{"pattern": k}, + )) + } + newSchema := &subSchema{property: k, parent: currentSchema, ref: currentSchema.ref} + err = d.parseSchema(v, newSchema) + if err != nil { + return errors.New(err.Error()) + } + currentSchema.patternProperties[k] = newSchema + } + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA, + "given": KEY_PATTERN_PROPERTIES, + }, + )) + } + } + + // propertyNames + if existsMapKey(m, KEY_PROPERTY_NAMES) { + if isKind(m[KEY_PROPERTY_NAMES], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_PROPERTY_NAMES, parent: currentSchema, ref: currentSchema.ref} + currentSchema.propertyNames = newSchema + err := d.parseSchema(m[KEY_PROPERTY_NAMES], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA, + "given": KEY_PATTERN_PROPERTIES, + }, + )) + } + } + + // dependencies + if existsMapKey(m, KEY_DEPENDENCIES) { + err := d.parseDependencies(m[KEY_DEPENDENCIES], currentSchema) + if err != nil { + return err + } + } + + // items + if existsMapKey(m, KEY_ITEMS) { + if isKind(m[KEY_ITEMS], reflect.Slice) { + for _, itemElement := range m[KEY_ITEMS].([]interface{}) { + if isKind(itemElement, reflect.Map, reflect.Bool) { + newSchema := &subSchema{parent: currentSchema, property: KEY_ITEMS} + newSchema.ref = currentSchema.ref + currentSchema.AddItemsChild(newSchema) + err := d.parseSchema(itemElement, newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA + "/" + STRING_ARRAY_OF_SCHEMAS, + "given": KEY_ITEMS, + }, + )) + } + currentSchema.itemsChildrenIsSingleSchema = false + } + } else if isKind(m[KEY_ITEMS], reflect.Map, reflect.Bool) { + newSchema := &subSchema{parent: currentSchema, property: KEY_ITEMS} + newSchema.ref = currentSchema.ref + currentSchema.AddItemsChild(newSchema) + err := d.parseSchema(m[KEY_ITEMS], newSchema) + if err != nil { + return err + } + currentSchema.itemsChildrenIsSingleSchema = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA + "/" + STRING_ARRAY_OF_SCHEMAS, + "given": KEY_ITEMS, + }, + )) + } + } + + // additionalItems + if existsMapKey(m, KEY_ADDITIONAL_ITEMS) { + if isKind(m[KEY_ADDITIONAL_ITEMS], reflect.Bool) { + currentSchema.additionalItems = m[KEY_ADDITIONAL_ITEMS].(bool) + } else if isKind(m[KEY_ADDITIONAL_ITEMS], reflect.Map) { + newSchema := &subSchema{property: KEY_ADDITIONAL_ITEMS, parent: currentSchema, ref: currentSchema.ref} + currentSchema.additionalItems = newSchema + err := d.parseSchema(m[KEY_ADDITIONAL_ITEMS], newSchema) + if err != nil { + return errors.New(err.Error()) + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + STRING_SCHEMA, + "given": KEY_ADDITIONAL_ITEMS, + }, + )) + } + } + + // validation : number / integer + + if existsMapKey(m, KEY_MULTIPLE_OF) { + multipleOfValue := mustBeNumber(m[KEY_MULTIPLE_OF]) + if multipleOfValue == nil { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_NUMBER, + "given": KEY_MULTIPLE_OF, + }, + )) + } + if multipleOfValue.Cmp(big.NewFloat(0)) <= 0 { + return errors.New(formatErrorDescription( + Locale.GreaterThanZero(), + ErrorDetails{"number": KEY_MULTIPLE_OF}, + )) + } + currentSchema.multipleOf = multipleOfValue + } + + if existsMapKey(m, KEY_MINIMUM) { + minimumValue := mustBeNumber(m[KEY_MINIMUM]) + if minimumValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_MINIMUM, "y": STRING_NUMBER}, + )) + } + currentSchema.minimum = minimumValue + } + + if existsMapKey(m, KEY_EXCLUSIVE_MINIMUM) { + if isKind(m[KEY_EXCLUSIVE_MINIMUM], reflect.Bool) { + if currentSchema.minimum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MINIMUM, "y": KEY_MINIMUM}, + )) + } + exclusiveMinimumValue := m[KEY_EXCLUSIVE_MINIMUM].(bool) + currentSchema.exclusiveMinimum = exclusiveMinimumValue + } else if isJsonNumber(m[KEY_EXCLUSIVE_MINIMUM]) { + minimumValue := mustBeNumber(m[KEY_EXCLUSIVE_MINIMUM]) + + currentSchema.minimum = minimumValue + currentSchema.exclusiveMinimum = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{"expected": TYPE_BOOLEAN + ", " + TYPE_NUMBER, "given": KEY_EXCLUSIVE_MINIMUM}, + )) + } + } + + if existsMapKey(m, KEY_MAXIMUM) { + maximumValue := mustBeNumber(m[KEY_MAXIMUM]) + if maximumValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_MAXIMUM, "y": STRING_NUMBER}, + )) + } + currentSchema.maximum = maximumValue + } + + if existsMapKey(m, KEY_EXCLUSIVE_MAXIMUM) { + if isKind(m[KEY_EXCLUSIVE_MAXIMUM], reflect.Bool) { + if currentSchema.maximum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MAXIMUM, "y": KEY_MAXIMUM}, + )) + } + exclusiveMaximumValue := m[KEY_EXCLUSIVE_MAXIMUM].(bool) + currentSchema.exclusiveMaximum = exclusiveMaximumValue + } else if isJsonNumber(m[KEY_EXCLUSIVE_MAXIMUM]) { + maximumValue := mustBeNumber(m[KEY_EXCLUSIVE_MAXIMUM]) + + currentSchema.maximum = maximumValue + currentSchema.exclusiveMaximum = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{"expected": TYPE_BOOLEAN + ", " + TYPE_NUMBER, "given": KEY_EXCLUSIVE_MAXIMUM}, + )) + } + } + + if currentSchema.minimum != nil && currentSchema.maximum != nil { + if currentSchema.minimum.Cmp(currentSchema.maximum) == 1 { + return errors.New(formatErrorDescription( + Locale.CannotBeGT(), + ErrorDetails{"x": KEY_MINIMUM, "y": KEY_MAXIMUM}, + )) + } + } + + // validation : string + + if existsMapKey(m, KEY_MIN_LENGTH) { + minLengthIntegerValue := mustBeInteger(m[KEY_MIN_LENGTH]) + if minLengthIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_LENGTH, "y": TYPE_INTEGER}, + )) + } + if *minLengthIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_LENGTH}, + )) + } + currentSchema.minLength = minLengthIntegerValue + } + + if existsMapKey(m, KEY_MAX_LENGTH) { + maxLengthIntegerValue := mustBeInteger(m[KEY_MAX_LENGTH]) + if maxLengthIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_LENGTH, "y": TYPE_INTEGER}, + )) + } + if *maxLengthIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_LENGTH}, + )) + } + currentSchema.maxLength = maxLengthIntegerValue + } + + if currentSchema.minLength != nil && currentSchema.maxLength != nil { + if *currentSchema.minLength > *currentSchema.maxLength { + return errors.New(formatErrorDescription( + Locale.CannotBeGT(), + ErrorDetails{"x": KEY_MIN_LENGTH, "y": KEY_MAX_LENGTH}, + )) + } + } + + if existsMapKey(m, KEY_PATTERN) { + if isKind(m[KEY_PATTERN], reflect.String) { + regexpObject, err := regexp.Compile(m[KEY_PATTERN].(string)) + if err != nil { + return errors.New(formatErrorDescription( + Locale.MustBeValidRegex(), + ErrorDetails{"key": KEY_PATTERN}, + )) + } + currentSchema.pattern = regexpObject + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_PATTERN, "y": TYPE_STRING}, + )) + } + } + + if existsMapKey(m, KEY_FORMAT) { + formatString, ok := m[KEY_FORMAT].(string) + if ok && FormatCheckers.Has(formatString) { + currentSchema.format = formatString + } + } + + // validation : object + + if existsMapKey(m, KEY_MIN_PROPERTIES) { + minPropertiesIntegerValue := mustBeInteger(m[KEY_MIN_PROPERTIES]) + if minPropertiesIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_PROPERTIES, "y": TYPE_INTEGER}, + )) + } + if *minPropertiesIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_PROPERTIES}, + )) + } + currentSchema.minProperties = minPropertiesIntegerValue + } + + if existsMapKey(m, KEY_MAX_PROPERTIES) { + maxPropertiesIntegerValue := mustBeInteger(m[KEY_MAX_PROPERTIES]) + if maxPropertiesIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_PROPERTIES, "y": TYPE_INTEGER}, + )) + } + if *maxPropertiesIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_PROPERTIES}, + )) + } + currentSchema.maxProperties = maxPropertiesIntegerValue + } + + if currentSchema.minProperties != nil && currentSchema.maxProperties != nil { + if *currentSchema.minProperties > *currentSchema.maxProperties { + return errors.New(formatErrorDescription( + Locale.KeyCannotBeGreaterThan(), + ErrorDetails{"key": KEY_MIN_PROPERTIES, "y": KEY_MAX_PROPERTIES}, + )) + } + } + + if existsMapKey(m, KEY_REQUIRED) { + if isKind(m[KEY_REQUIRED], reflect.Slice) { + requiredValues := m[KEY_REQUIRED].([]interface{}) + for _, requiredValue := range requiredValues { + if isKind(requiredValue, reflect.String) { + err := currentSchema.AddRequired(requiredValue.(string)) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeOfType(), + ErrorDetails{"key": KEY_REQUIRED, "type": TYPE_STRING}, + )) + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_REQUIRED, "y": TYPE_ARRAY}, + )) + } + } + + // validation : array + + if existsMapKey(m, KEY_MIN_ITEMS) { + minItemsIntegerValue := mustBeInteger(m[KEY_MIN_ITEMS]) + if minItemsIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_ITEMS, "y": TYPE_INTEGER}, + )) + } + if *minItemsIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_ITEMS}, + )) + } + currentSchema.minItems = minItemsIntegerValue + } + + if existsMapKey(m, KEY_MAX_ITEMS) { + maxItemsIntegerValue := mustBeInteger(m[KEY_MAX_ITEMS]) + if maxItemsIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_ITEMS, "y": TYPE_INTEGER}, + )) + } + if *maxItemsIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_ITEMS}, + )) + } + currentSchema.maxItems = maxItemsIntegerValue + } + + if existsMapKey(m, KEY_UNIQUE_ITEMS) { + if isKind(m[KEY_UNIQUE_ITEMS], reflect.Bool) { + currentSchema.uniqueItems = m[KEY_UNIQUE_ITEMS].(bool) + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_UNIQUE_ITEMS, "y": TYPE_BOOLEAN}, + )) + } + } + + if existsMapKey(m, KEY_CONTAINS) { + newSchema := &subSchema{property: KEY_CONTAINS, parent: currentSchema, ref: currentSchema.ref} + currentSchema.contains = newSchema + err := d.parseSchema(m[KEY_CONTAINS], newSchema) + if err != nil { + return err + } + } + + // validation : all + + if existsMapKey(m, KEY_CONST) { + err := currentSchema.AddConst(m[KEY_CONST]) + if err != nil { + return err + } + } + + if existsMapKey(m, KEY_ENUM) { + if isKind(m[KEY_ENUM], reflect.Slice) { + for _, v := range m[KEY_ENUM].([]interface{}) { + err := currentSchema.AddEnum(v) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ENUM, "y": TYPE_ARRAY}, + )) + } + } + + // validation : subSchema + + if existsMapKey(m, KEY_ONE_OF) { + if isKind(m[KEY_ONE_OF], reflect.Slice) { + for _, v := range m[KEY_ONE_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ONE_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddOneOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ONE_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_ANY_OF) { + if isKind(m[KEY_ANY_OF], reflect.Slice) { + for _, v := range m[KEY_ANY_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ANY_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddAnyOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ANY_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_ALL_OF) { + if isKind(m[KEY_ALL_OF], reflect.Slice) { + for _, v := range m[KEY_ALL_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ALL_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddAllOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ANY_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_NOT) { + if isKind(m[KEY_NOT], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_NOT, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetNot(newSchema) + err := d.parseSchema(m[KEY_NOT], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_NOT, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_IF) { + if isKind(m[KEY_IF], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_IF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetIf(newSchema) + err := d.parseSchema(m[KEY_IF], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_IF, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_THEN) { + if isKind(m[KEY_THEN], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_THEN, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetThen(newSchema) + err := d.parseSchema(m[KEY_THEN], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_THEN, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_ELSE) { + if isKind(m[KEY_ELSE], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_ELSE, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetElse(newSchema) + err := d.parseSchema(m[KEY_ELSE], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ELSE, "y": TYPE_OBJECT}, + )) + } + } + + return nil +} + +func (d *Schema) parseReference(documentNode interface{}, currentSchema *subSchema) error { + var ( + refdDocumentNode interface{} + dsp *schemaPoolDocument + err error + ) + + newSchema := &subSchema{property: KEY_REF, parent: currentSchema, ref: currentSchema.ref} + + d.referencePool.Add(currentSchema.ref.String(), newSchema) + + dsp, err = d.pool.GetDocument(*currentSchema.ref) + if err != nil { + return err + } + newSchema.id = currentSchema.ref + + refdDocumentNode = dsp.Document + + if err != nil { + return err + } + + if !isKind(refdDocumentNode, reflect.Map, reflect.Bool) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": STRING_SCHEMA, "type": TYPE_OBJECT}, + )) + } + + err = d.parseSchema(refdDocumentNode, newSchema) + if err != nil { + return err + } + + currentSchema.refSchema = newSchema + + return nil + +} + +func (d *Schema) parseProperties(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": STRING_PROPERTIES, "type": TYPE_OBJECT}, + )) + } + + m := documentNode.(map[string]interface{}) + for k := range m { + schemaProperty := k + newSchema := &subSchema{property: schemaProperty, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddPropertiesChild(newSchema) + err := d.parseSchema(m[k], newSchema) + if err != nil { + return err + } + } + + return nil +} + +func (d *Schema) parseDependencies(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": KEY_DEPENDENCIES, "type": TYPE_OBJECT}, + )) + } + + m := documentNode.(map[string]interface{}) + currentSchema.dependencies = make(map[string]interface{}) + + for k := range m { + switch reflect.ValueOf(m[k]).Kind() { + + case reflect.Slice: + values := m[k].([]interface{}) + var valuesToRegister []string + + for _, value := range values { + if !isKind(value, reflect.String) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": STRING_DEPENDENCY, + "type": STRING_SCHEMA_OR_ARRAY_OF_STRINGS, + }, + )) + } else { + valuesToRegister = append(valuesToRegister, value.(string)) + } + currentSchema.dependencies[k] = valuesToRegister + } + + case reflect.Map, reflect.Bool: + depSchema := &subSchema{property: k, parent: currentSchema, ref: currentSchema.ref} + err := d.parseSchema(m[k], depSchema) + if err != nil { + return err + } + currentSchema.dependencies[k] = depSchema + + default: + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": STRING_DEPENDENCY, + "type": STRING_SCHEMA_OR_ARRAY_OF_STRINGS, + }, + )) + } + + } + + return nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go new file mode 100644 index 00000000000..09d3a37e255 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go @@ -0,0 +1,102 @@ +package gojsonschema + +import ( + "github.com/xeipuuv/gojsonreference" +) + +type SchemaLoader struct { + pool *schemaPool +} + +func NewSchemaLoader() *SchemaLoader { + + ps := &SchemaLoader{ + pool: &schemaPool{ + schemaPoolDocuments: make(map[string]*schemaPoolDocument), + }, + } + + return ps +} + +// AddSchemas adds an arbritrary amount of schemas to the schema cache. As this function does not require +// an explicit URL, every schema should contain an $id, so that it can be referenced by the main schema +func (sl *SchemaLoader) AddSchemas(loaders ...JSONLoader) error { + emptyRef, _ := gojsonreference.NewJsonReference("") + + for _, loader := range loaders { + doc, err := loader.LoadJSON() + if err != nil { + return err + } + // Directly use the Recursive function, so that it get only added to the schema pool by $id + // and not by the ref of the document as it's empty + if err = sl.pool.parseReferencesRecursive(doc, emptyRef); err != nil { + return err + } + } + + return nil +} + +//AddSchema adds a schema under the provided URL to the schema cache +func (sl *SchemaLoader) AddSchema(url string, loader JSONLoader) error { + + ref, err := gojsonreference.NewJsonReference(url) + + if err != nil { + return err + } + + doc, err := loader.LoadJSON() + + if err != nil { + return err + } + + return sl.pool.ParseReferences(doc, ref) +} + +func (sl *SchemaLoader) Compile(rootSchema JSONLoader) (*Schema, error) { + + ref, err := rootSchema.JsonReference() + + if err != nil { + return nil, err + } + + d := Schema{} + d.pool = sl.pool + d.pool.jsonLoaderFactory = rootSchema.LoaderFactory() + d.documentReference = ref + d.referencePool = newSchemaReferencePool() + + var doc interface{} + if ref.String() != "" { + // Get document from schema pool + spd, err := d.pool.GetDocument(d.documentReference) + if err != nil { + return nil, err + } + doc = spd.Document + } else { + // Load JSON directly + doc, err = rootSchema.LoadJSON() + if err != nil { + return nil, err + } + // References need only be parsed if loading JSON directly + // as pool.GetDocument already does this for us if loading by reference + err = d.pool.ParseReferences(doc, ref) + if err != nil { + return nil, err + } + } + + err = d.parse(doc) + if err != nil { + return nil, err + } + + return &d, nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go new file mode 100644 index 00000000000..c3de8e5ef3f --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go @@ -0,0 +1,201 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines resources pooling. +// Eases referencing and avoids downloading the same resource twice. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "errors" + "fmt" + "reflect" + + "github.com/xeipuuv/gojsonreference" +) + +type schemaPoolDocument struct { + Document interface{} +} + +type schemaPool struct { + schemaPoolDocuments map[string]*schemaPoolDocument + jsonLoaderFactory JSONLoaderFactory +} + +func newSchemaPool(f JSONLoaderFactory) *schemaPool { + + p := &schemaPool{} + p.schemaPoolDocuments = make(map[string]*schemaPoolDocument) + p.jsonLoaderFactory = f + + return p +} + +func (p *schemaPool) ParseReferences(document interface{}, ref gojsonreference.JsonReference) error { + // Only the root document should be added to the schema pool + if _, ok := p.schemaPoolDocuments[ref.String()]; ok { + return fmt.Errorf("Reference already exists: \"%s\"", ref.String()) + } + err := p.parseReferencesRecursive(document, ref) + p.schemaPoolDocuments[ref.String()] = &schemaPoolDocument{Document: document} + return err +} + +func (p *schemaPool) parseReferencesRecursive(document interface{}, ref gojsonreference.JsonReference) error { + // parseReferencesRecursive parses a JSON document and resolves all $id and $ref references. + // For $ref references it takes into account the $id scope it is in and replaces + // the reference by the absolute resolved reference + + // When encountering errors it fails silently. Error handling is done when the schema + // is syntactically parsed and any error encountered here should also come up there. + switch m := document.(type) { + case []interface{}: + for _, v := range m { + p.parseReferencesRecursive(v, ref) + } + case map[string]interface{}: + localRef := &ref + + keyID := KEY_ID_NEW + if existsMapKey(m, KEY_ID) { + keyID = KEY_ID + } + if existsMapKey(m, keyID) && isKind(m[keyID], reflect.String) { + jsonReference, err := gojsonreference.NewJsonReference(m[keyID].(string)) + if err == nil { + localRef, err = ref.Inherits(jsonReference) + if err == nil { + if _, ok := p.schemaPoolDocuments[localRef.String()]; ok { + return fmt.Errorf("Reference already exists: \"%s\"", localRef.String()) + } + p.schemaPoolDocuments[localRef.String()] = &schemaPoolDocument{Document: document} + } + } + } + + if existsMapKey(m, KEY_REF) && isKind(m[KEY_REF], reflect.String) { + jsonReference, err := gojsonreference.NewJsonReference(m[KEY_REF].(string)) + if err == nil { + absoluteRef, err := localRef.Inherits(jsonReference) + if err == nil { + m[KEY_REF] = absoluteRef.String() + } + } + } + + for k, v := range m { + // const and enums should be interpreted literally, so ignore them + if k == KEY_CONST || k == KEY_ENUM { + continue + } + // Something like a property or a dependency is not a valid schema, as it might describe properties named "$ref", "$id" or "const", etc + // Therefore don't treat it like a schema. + if k == KEY_PROPERTIES || k == KEY_DEPENDENCIES || k == KEY_PATTERN_PROPERTIES { + if child, ok := v.(map[string]interface{}); ok { + for _, v := range child { + p.parseReferencesRecursive(v, *localRef) + } + } + } else { + p.parseReferencesRecursive(v, *localRef) + } + } + } + return nil +} + +func (p *schemaPool) GetDocument(reference gojsonreference.JsonReference) (*schemaPoolDocument, error) { + + var ( + spd *schemaPoolDocument + ok bool + err error + ) + + if internalLogEnabled { + internalLog("Get Document ( %s )", reference.String()) + } + + // Create a deep copy, so we can remove the fragment part later on without altering the original + refToUrl, _ := gojsonreference.NewJsonReference(reference.String()) + + // First check if the given fragment is a location independent identifier + // http://json-schema.org/latest/json-schema-core.html#rfc.section.8.2.3 + + if spd, ok = p.schemaPoolDocuments[refToUrl.String()]; ok { + if internalLogEnabled { + internalLog(" From pool") + } + return spd, nil + } + + // If the given reference is not a location independent identifier, + // strip the fragment and look for a document with it's base URI + + refToUrl.GetUrl().Fragment = "" + + if cachedSpd, ok := p.schemaPoolDocuments[refToUrl.String()]; ok { + document, _, err := reference.GetPointer().Get(cachedSpd.Document) + + if err != nil { + return nil, err + } + + if internalLogEnabled { + internalLog(" From pool") + } + + spd = &schemaPoolDocument{Document: document} + p.schemaPoolDocuments[reference.String()] = spd + + return spd, nil + } + + // It is not possible to load anything remotely that is not canonical... + if !reference.IsCanonical() { + return nil, errors.New(formatErrorDescription( + Locale.ReferenceMustBeCanonical(), + ErrorDetails{"reference": reference.String()}, + )) + } + + jsonReferenceLoader := p.jsonLoaderFactory.New(reference.String()) + document, err := jsonReferenceLoader.LoadJSON() + + if err != nil { + return nil, err + } + + // add the whole document to the pool for potential re-use + p.ParseReferences(document, refToUrl) + + // resolve the potential fragment and also cache it + document, _, err = reference.GetPointer().Get(document) + + if err != nil { + return nil, err + } + + return &schemaPoolDocument{Document: document}, nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go new file mode 100644 index 00000000000..6e5e1b5cdb3 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go @@ -0,0 +1,68 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Pool of referenced schemas. +// +// created 25-06-2013 + +package gojsonschema + +import ( + "fmt" +) + +type schemaReferencePool struct { + documents map[string]*subSchema +} + +func newSchemaReferencePool() *schemaReferencePool { + + p := &schemaReferencePool{} + p.documents = make(map[string]*subSchema) + + return p +} + +func (p *schemaReferencePool) Get(ref string) (r *subSchema, o bool) { + + if internalLogEnabled { + internalLog(fmt.Sprintf("Schema Reference ( %s )", ref)) + } + + if sch, ok := p.documents[ref]; ok { + if internalLogEnabled { + internalLog(fmt.Sprintf(" From pool")) + } + return sch, true + } + + return nil, false +} + +func (p *schemaReferencePool) Add(ref string, sch *subSchema) { + + if internalLogEnabled { + internalLog(fmt.Sprintf("Add Schema Reference %s to pool", ref)) + } + if _, ok := p.documents[ref]; !ok { + p.documents[ref] = sch + } +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go new file mode 100644 index 00000000000..36b447a2915 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go @@ -0,0 +1,83 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Helper structure to handle schema types, and the combination of them. +// +// created 28-02-2013 + +package gojsonschema + +import ( + "errors" + "fmt" + "strings" +) + +type jsonSchemaType struct { + types []string +} + +// Is the schema typed ? that is containing at least one type +// When not typed, the schema does not need any type validation +func (t *jsonSchemaType) IsTyped() bool { + return len(t.types) > 0 +} + +func (t *jsonSchemaType) Add(etype string) error { + + if !isStringInSlice(JSON_TYPES, etype) { + return errors.New(formatErrorDescription(Locale.NotAValidType(), ErrorDetails{"given": "/" + etype + "/", "expected": JSON_TYPES})) + } + + if t.Contains(etype) { + return errors.New(formatErrorDescription(Locale.Duplicated(), ErrorDetails{"type": etype})) + } + + t.types = append(t.types, etype) + + return nil +} + +func (t *jsonSchemaType) Contains(etype string) bool { + + for _, v := range t.types { + if v == etype { + return true + } + } + + return false +} + +func (t *jsonSchemaType) String() string { + + if len(t.types) == 0 { + return STRING_UNDEFINED // should never happen + } + + // Displayed as a list [type1,type2,...] + if len(t.types) > 1 { + return fmt.Sprintf("[%s]", strings.Join(t.types, ",")) + } + + // Only one type: name only + return t.types[0] +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go new file mode 100644 index 00000000000..ea792a111a2 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go @@ -0,0 +1,255 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines the structure of a sub-subSchema. +// A sub-subSchema can contain other sub-schemas. +// +// created 27-02-2013 + +package gojsonschema + +import ( + "errors" + "math/big" + "regexp" + "strings" + + "github.com/xeipuuv/gojsonreference" +) + +const ( + KEY_SCHEMA = "$schema" + KEY_ID = "id" + KEY_ID_NEW = "$id" + KEY_REF = "$ref" + KEY_TITLE = "title" + KEY_DESCRIPTION = "description" + KEY_TYPE = "type" + KEY_ITEMS = "items" + KEY_ADDITIONAL_ITEMS = "additionalItems" + KEY_PROPERTIES = "properties" + KEY_PATTERN_PROPERTIES = "patternProperties" + KEY_ADDITIONAL_PROPERTIES = "additionalProperties" + KEY_PROPERTY_NAMES = "propertyNames" + KEY_DEFINITIONS = "definitions" + KEY_MULTIPLE_OF = "multipleOf" + KEY_MINIMUM = "minimum" + KEY_MAXIMUM = "maximum" + KEY_EXCLUSIVE_MINIMUM = "exclusiveMinimum" + KEY_EXCLUSIVE_MAXIMUM = "exclusiveMaximum" + KEY_MIN_LENGTH = "minLength" + KEY_MAX_LENGTH = "maxLength" + KEY_PATTERN = "pattern" + KEY_FORMAT = "format" + KEY_MIN_PROPERTIES = "minProperties" + KEY_MAX_PROPERTIES = "maxProperties" + KEY_DEPENDENCIES = "dependencies" + KEY_REQUIRED = "required" + KEY_MIN_ITEMS = "minItems" + KEY_MAX_ITEMS = "maxItems" + KEY_UNIQUE_ITEMS = "uniqueItems" + KEY_CONTAINS = "contains" + KEY_CONST = "const" + KEY_ENUM = "enum" + KEY_ONE_OF = "oneOf" + KEY_ANY_OF = "anyOf" + KEY_ALL_OF = "allOf" + KEY_NOT = "not" + KEY_IF = "if" + KEY_THEN = "then" + KEY_ELSE = "else" +) + +type subSchema struct { + + // basic subSchema meta properties + id *gojsonreference.JsonReference + title *string + description *string + + property string + + // Types associated with the subSchema + types jsonSchemaType + + // Reference url + ref *gojsonreference.JsonReference + // Schema referenced + refSchema *subSchema + + // hierarchy + parent *subSchema + itemsChildren []*subSchema + itemsChildrenIsSingleSchema bool + propertiesChildren []*subSchema + + // validation : number / integer + multipleOf *big.Float + maximum *big.Float + exclusiveMaximum bool + minimum *big.Float + exclusiveMinimum bool + + // validation : string + minLength *int + maxLength *int + pattern *regexp.Regexp + format string + + // validation : object + minProperties *int + maxProperties *int + required []string + + dependencies map[string]interface{} + additionalProperties interface{} + patternProperties map[string]*subSchema + propertyNames *subSchema + + // validation : array + minItems *int + maxItems *int + uniqueItems bool + contains *subSchema + + additionalItems interface{} + + // validation : all + _const *string //const is a golang keyword + enum []string + + // validation : subSchema + oneOf []*subSchema + anyOf []*subSchema + allOf []*subSchema + not *subSchema + _if *subSchema // if/else are golang keywords + _then *subSchema + _else *subSchema +} + +func (s *subSchema) AddConst(i interface{}) error { + + is, err := marshalWithoutNumber(i) + if err != nil { + return err + } + s._const = is + return nil +} + +func (s *subSchema) AddEnum(i interface{}) error { + + is, err := marshalWithoutNumber(i) + if err != nil { + return err + } + + if isStringInSlice(s.enum, *is) { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeUnique(), + ErrorDetails{"key": KEY_ENUM}, + )) + } + + s.enum = append(s.enum, *is) + + return nil +} + +func (s *subSchema) ContainsEnum(i interface{}) (bool, error) { + + is, err := marshalWithoutNumber(i) + if err != nil { + return false, err + } + + return isStringInSlice(s.enum, *is), nil +} + +func (s *subSchema) AddOneOf(subSchema *subSchema) { + s.oneOf = append(s.oneOf, subSchema) +} + +func (s *subSchema) AddAllOf(subSchema *subSchema) { + s.allOf = append(s.allOf, subSchema) +} + +func (s *subSchema) AddAnyOf(subSchema *subSchema) { + s.anyOf = append(s.anyOf, subSchema) +} + +func (s *subSchema) SetNot(subSchema *subSchema) { + s.not = subSchema +} + +func (s *subSchema) SetIf(subSchema *subSchema) { + s._if = subSchema +} + +func (s *subSchema) SetThen(subSchema *subSchema) { + s._then = subSchema +} + +func (s *subSchema) SetElse(subSchema *subSchema) { + s._else = subSchema +} + +func (s *subSchema) AddRequired(value string) error { + + if isStringInSlice(s.required, value) { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeUnique(), + ErrorDetails{"key": KEY_REQUIRED}, + )) + } + + s.required = append(s.required, value) + + return nil +} + +func (s *subSchema) AddItemsChild(child *subSchema) { + s.itemsChildren = append(s.itemsChildren, child) +} + +func (s *subSchema) AddPropertiesChild(child *subSchema) { + s.propertiesChildren = append(s.propertiesChildren, child) +} + +func (s *subSchema) PatternPropertiesString() string { + + if s.patternProperties == nil || len(s.patternProperties) == 0 { + return STRING_UNDEFINED // should never happen + } + + patternPropertiesKeySlice := []string{} + for pk := range s.patternProperties { + patternPropertiesKeySlice = append(patternPropertiesKeySlice, `"`+pk+`"`) + } + + if len(patternPropertiesKeySlice) == 1 { + return patternPropertiesKeySlice[0] + } + + return "[" + strings.Join(patternPropertiesKeySlice, ",") + "]" + +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go new file mode 100644 index 00000000000..952d22ef65e --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go @@ -0,0 +1,58 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Contains const types for schema and JSON. +// +// created 28-02-2013 + +package gojsonschema + +const ( + TYPE_ARRAY = `array` + TYPE_BOOLEAN = `boolean` + TYPE_INTEGER = `integer` + TYPE_NUMBER = `number` + TYPE_NULL = `null` + TYPE_OBJECT = `object` + TYPE_STRING = `string` +) + +var JSON_TYPES []string +var SCHEMA_TYPES []string + +func init() { + JSON_TYPES = []string{ + TYPE_ARRAY, + TYPE_BOOLEAN, + TYPE_INTEGER, + TYPE_NUMBER, + TYPE_NULL, + TYPE_OBJECT, + TYPE_STRING} + + SCHEMA_TYPES = []string{ + TYPE_ARRAY, + TYPE_BOOLEAN, + TYPE_INTEGER, + TYPE_NUMBER, + TYPE_OBJECT, + TYPE_STRING} +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go new file mode 100644 index 00000000000..cc3100a78dd --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go @@ -0,0 +1,226 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Various utility functions. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "encoding/json" + "fmt" + "math" + "math/big" + "reflect" +) + +func isKind(what interface{}, kinds ...reflect.Kind) bool { + target := what + if isJsonNumber(what) { + // JSON Numbers are strings! + target = *mustBeNumber(what) + } + targetKind := reflect.ValueOf(target).Kind() + for _, kind := range kinds { + if targetKind == kind { + return true + } + } + return false +} + +func existsMapKey(m map[string]interface{}, k string) bool { + _, ok := m[k] + return ok +} + +func isStringInSlice(s []string, what string) bool { + for i := range s { + if s[i] == what { + return true + } + } + return false +} + +func marshalToJsonString(value interface{}) (*string, error) { + + mBytes, err := json.Marshal(value) + if err != nil { + return nil, err + } + + sBytes := string(mBytes) + return &sBytes, nil +} + +func marshalWithoutNumber(value interface{}) (*string, error) { + + // The JSON is decoded using https://golang.org/pkg/encoding/json/#Decoder.UseNumber + // This means the numbers are internally still represented as strings and therefore 1.00 is unequal to 1 + // One way to eliminate these differences is to decode and encode the JSON one more time without Decoder.UseNumber + // so that these differences in representation are removed + + jsonString, err := marshalToJsonString(value) + if err != nil { + return nil, err + } + + var document interface{} + + err = json.Unmarshal([]byte(*jsonString), &document) + if err != nil { + return nil, err + } + + return marshalToJsonString(document) +} + +func isJsonNumber(what interface{}) bool { + + switch what.(type) { + + case json.Number: + return true + } + + return false +} + +func checkJsonInteger(what interface{}) (isInt bool) { + + jsonNumber := what.(json.Number) + + bigFloat, isValidNumber := new(big.Float).SetString(string(jsonNumber)) + + return isValidNumber && bigFloat.IsInt() + +} + +// same as ECMA Number.MAX_SAFE_INTEGER and Number.MIN_SAFE_INTEGER +const ( + max_json_float = float64(1<<53 - 1) // 9007199254740991.0 2^53 - 1 + min_json_float = -float64(1<<53 - 1) //-9007199254740991.0 -2^53 - 1 +) + +func isFloat64AnInteger(f float64) bool { + + if math.IsNaN(f) || math.IsInf(f, 0) || f < min_json_float || f > max_json_float { + return false + } + + return f == float64(int64(f)) || f == float64(uint64(f)) +} + +func mustBeInteger(what interface{}) *int { + + if isJsonNumber(what) { + + number := what.(json.Number) + + isInt := checkJsonInteger(number) + + if isInt { + + int64Value, err := number.Int64() + if err != nil { + return nil + } + + int32Value := int(int64Value) + return &int32Value + + } else { + return nil + } + + } + + return nil +} + +func mustBeNumber(what interface{}) *big.Float { + + if isJsonNumber(what) { + number := what.(json.Number) + float64Value, success := new(big.Float).SetString(string(number)) + if success { + return float64Value + } else { + return nil + } + + } + + return nil + +} + +// formats a number so that it is displayed as the smallest string possible +func resultErrorFormatJsonNumber(n json.Number) string { + + if int64Value, err := n.Int64(); err == nil { + return fmt.Sprintf("%d", int64Value) + } + + float64Value, _ := n.Float64() + + return fmt.Sprintf("%g", float64Value) +} + +// formats a number so that it is displayed as the smallest string possible +func resultErrorFormatNumber(n float64) string { + + if isFloat64AnInteger(n) { + return fmt.Sprintf("%d", int64(n)) + } + + return fmt.Sprintf("%g", n) +} + +func convertDocumentNode(val interface{}) interface{} { + + if lval, ok := val.([]interface{}); ok { + + res := []interface{}{} + for _, v := range lval { + res = append(res, convertDocumentNode(v)) + } + + return res + + } + + if mval, ok := val.(map[interface{}]interface{}); ok { + + res := map[string]interface{}{} + + for k, v := range mval { + res[k.(string)] = convertDocumentNode(v) + } + + return res + + } + + return val +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go new file mode 100644 index 00000000000..2f1e7df36c1 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go @@ -0,0 +1,928 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Extends Schema and subSchema, implements the validation phase. +// +// created 28-02-2013 + +package gojsonschema + +import ( + "encoding/json" + "math/big" + "reflect" + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +func Validate(ls JSONLoader, ld JSONLoader) (*Result, error) { + + var err error + + // load schema + + schema, err := NewSchema(ls) + if err != nil { + return nil, err + } + + // begine validation + + return schema.Validate(ld) + +} + +func (v *Schema) Validate(l JSONLoader) (*Result, error) { + + // load document + + root, err := l.LoadJSON() + if err != nil { + return nil, err + } + + return v.validateDocument(root), nil +} + +func (v *Schema) validateDocument(root interface{}) *Result { + // begin validation + + result := &Result{} + context := NewJsonContext(STRING_CONTEXT_ROOT, nil) + v.rootSchema.validateRecursive(v.rootSchema, root, result, context) + + return result +} + +func (v *subSchema) subValidateWithContext(document interface{}, context *JsonContext) *Result { + result := &Result{} + v.validateRecursive(v, document, result, context) + return result +} + +// Walker function to validate the json recursively against the subSchema +func (v *subSchema) validateRecursive(currentSubSchema *subSchema, currentNode interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateRecursive %s", context.String()) + internalLog(" %v", currentNode) + } + + // Handle referenced schemas, returns directly when a $ref is found + if currentSubSchema.refSchema != nil { + v.validateRecursive(currentSubSchema.refSchema, currentNode, result, context) + return + } + + // Check for null value + if currentNode == nil { + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_NULL) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_NULL, + }, + ) + return + } + + currentSubSchema.validateSchema(currentSubSchema, currentNode, result, context) + v.validateCommon(currentSubSchema, currentNode, result, context) + + } else { // Not a null value + + if isJsonNumber(currentNode) { + + value := currentNode.(json.Number) + + isInt := checkJsonInteger(value) + + validType := currentSubSchema.types.Contains(TYPE_NUMBER) || (isInt && currentSubSchema.types.Contains(TYPE_INTEGER)) + + if currentSubSchema.types.IsTyped() && !validType { + + givenType := TYPE_INTEGER + if !isInt { + givenType = TYPE_NUMBER + } + + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": givenType, + }, + ) + return + } + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + } else { + + rValue := reflect.ValueOf(currentNode) + rKind := rValue.Kind() + + switch rKind { + + // Slice => JSON array + + case reflect.Slice: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_ARRAY) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_ARRAY, + }, + ) + return + } + + castCurrentNode := currentNode.([]interface{}) + + currentSubSchema.validateSchema(currentSubSchema, castCurrentNode, result, context) + + v.validateArray(currentSubSchema, castCurrentNode, result, context) + v.validateCommon(currentSubSchema, castCurrentNode, result, context) + + // Map => JSON object + + case reflect.Map: + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_OBJECT) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_OBJECT, + }, + ) + return + } + + castCurrentNode, ok := currentNode.(map[string]interface{}) + if !ok { + castCurrentNode = convertDocumentNode(currentNode).(map[string]interface{}) + } + + currentSubSchema.validateSchema(currentSubSchema, castCurrentNode, result, context) + + v.validateObject(currentSubSchema, castCurrentNode, result, context) + v.validateCommon(currentSubSchema, castCurrentNode, result, context) + + for _, pSchema := range currentSubSchema.propertiesChildren { + nextNode, ok := castCurrentNode[pSchema.property] + if ok { + subContext := NewJsonContext(pSchema.property, context) + v.validateRecursive(pSchema, nextNode, result, subContext) + } + } + + // Simple JSON values : string, number, boolean + + case reflect.Bool: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_BOOLEAN) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_BOOLEAN, + }, + ) + return + } + + value := currentNode.(bool) + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + case reflect.String: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_STRING) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_STRING, + }, + ) + return + } + + value := currentNode.(string) + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + } + + } + + } + + result.incrementScore() +} + +// Different kinds of validation there, subSchema / common / array / object / string... +func (v *subSchema) validateSchema(currentSubSchema *subSchema, currentNode interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateSchema %s", context.String()) + internalLog(" %v", currentNode) + } + + if len(currentSubSchema.anyOf) > 0 { + + validatedAnyOf := false + var bestValidationResult *Result + + for _, anyOfSchema := range currentSubSchema.anyOf { + if !validatedAnyOf { + validationResult := anyOfSchema.subValidateWithContext(currentNode, context) + validatedAnyOf = validationResult.Valid() + + if !validatedAnyOf && (bestValidationResult == nil || validationResult.score > bestValidationResult.score) { + bestValidationResult = validationResult + } + } + } + if !validatedAnyOf { + + result.addInternalError(new(NumberAnyOfError), context, currentNode, ErrorDetails{}) + + if bestValidationResult != nil { + // add error messages of closest matching subSchema as + // that's probably the one the user was trying to match + result.mergeErrors(bestValidationResult) + } + } + } + + if len(currentSubSchema.oneOf) > 0 { + + nbValidated := 0 + var bestValidationResult *Result + + for _, oneOfSchema := range currentSubSchema.oneOf { + validationResult := oneOfSchema.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + nbValidated++ + } else if nbValidated == 0 && (bestValidationResult == nil || validationResult.score > bestValidationResult.score) { + bestValidationResult = validationResult + } + } + + if nbValidated != 1 { + + result.addInternalError(new(NumberOneOfError), context, currentNode, ErrorDetails{}) + + if nbValidated == 0 { + // add error messages of closest matching subSchema as + // that's probably the one the user was trying to match + result.mergeErrors(bestValidationResult) + } + } + + } + + if len(currentSubSchema.allOf) > 0 { + nbValidated := 0 + + for _, allOfSchema := range currentSubSchema.allOf { + validationResult := allOfSchema.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + nbValidated++ + } + result.mergeErrors(validationResult) + } + + if nbValidated != len(currentSubSchema.allOf) { + result.addInternalError(new(NumberAllOfError), context, currentNode, ErrorDetails{}) + } + } + + if currentSubSchema.not != nil { + validationResult := currentSubSchema.not.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + result.addInternalError(new(NumberNotError), context, currentNode, ErrorDetails{}) + } + } + + if currentSubSchema.dependencies != nil && len(currentSubSchema.dependencies) > 0 { + if isKind(currentNode, reflect.Map) { + for elementKey := range currentNode.(map[string]interface{}) { + if dependency, ok := currentSubSchema.dependencies[elementKey]; ok { + switch dependency := dependency.(type) { + + case []string: + for _, dependOnKey := range dependency { + if _, dependencyResolved := currentNode.(map[string]interface{})[dependOnKey]; !dependencyResolved { + result.addInternalError( + new(MissingDependencyError), + context, + currentNode, + ErrorDetails{"dependency": dependOnKey}, + ) + } + } + + case *subSchema: + dependency.validateRecursive(dependency, currentNode, result, context) + } + } + } + } + } + + if currentSubSchema._if != nil { + validationResultIf := currentSubSchema._if.subValidateWithContext(currentNode, context) + if currentSubSchema._then != nil && validationResultIf.Valid() { + validationResultThen := currentSubSchema._then.subValidateWithContext(currentNode, context) + if !validationResultThen.Valid() { + result.addInternalError(new(ConditionThenError), context, currentNode, ErrorDetails{}) + result.mergeErrors(validationResultThen) + } + } + if currentSubSchema._else != nil && !validationResultIf.Valid() { + validationResultElse := currentSubSchema._else.subValidateWithContext(currentNode, context) + if !validationResultElse.Valid() { + result.addInternalError(new(ConditionElseError), context, currentNode, ErrorDetails{}) + result.mergeErrors(validationResultElse) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validateCommon(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateCommon %s", context.String()) + internalLog(" %v", value) + } + + // const: + if currentSubSchema._const != nil { + vString, err := marshalWithoutNumber(value) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"error": err}) + } + if *vString != *currentSubSchema._const { + result.addInternalError(new(ConstError), + context, + value, + ErrorDetails{ + "allowed": *currentSubSchema._const, + }, + ) + } + } + + // enum: + if len(currentSubSchema.enum) > 0 { + has, err := currentSubSchema.ContainsEnum(value) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"error": err}) + } + if !has { + result.addInternalError( + new(EnumError), + context, + value, + ErrorDetails{ + "allowed": strings.Join(currentSubSchema.enum, ", "), + }, + ) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateArray(currentSubSchema *subSchema, value []interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateArray %s", context.String()) + internalLog(" %v", value) + } + + nbValues := len(value) + + // TODO explain + if currentSubSchema.itemsChildrenIsSingleSchema { + for i := range value { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := currentSubSchema.itemsChildren[0].subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + } else { + if currentSubSchema.itemsChildren != nil && len(currentSubSchema.itemsChildren) > 0 { + + nbItems := len(currentSubSchema.itemsChildren) + + // while we have both schemas and values, check them against each other + for i := 0; i != nbItems && i != nbValues; i++ { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := currentSubSchema.itemsChildren[i].subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + + if nbItems < nbValues { + // we have less schemas than elements in the instance array, + // but that might be ok if "additionalItems" is specified. + + switch currentSubSchema.additionalItems.(type) { + case bool: + if !currentSubSchema.additionalItems.(bool) { + result.addInternalError(new(ArrayNoAdditionalItemsError), context, value, ErrorDetails{}) + } + case *subSchema: + additionalItemSchema := currentSubSchema.additionalItems.(*subSchema) + for i := nbItems; i != nbValues; i++ { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := additionalItemSchema.subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + } + } + } + } + + // minItems & maxItems + if currentSubSchema.minItems != nil { + if nbValues < int(*currentSubSchema.minItems) { + result.addInternalError( + new(ArrayMinItemsError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minItems}, + ) + } + } + if currentSubSchema.maxItems != nil { + if nbValues > int(*currentSubSchema.maxItems) { + result.addInternalError( + new(ArrayMaxItemsError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxItems}, + ) + } + } + + // uniqueItems: + if currentSubSchema.uniqueItems { + var stringifiedItems []string + for _, v := range value { + vString, err := marshalWithoutNumber(v) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"err": err}) + } + if isStringInSlice(stringifiedItems, *vString) { + result.addInternalError( + new(ItemsMustBeUniqueError), + context, + value, + ErrorDetails{"type": TYPE_ARRAY}, + ) + } + stringifiedItems = append(stringifiedItems, *vString) + } + } + + // contains: + + if currentSubSchema.contains != nil { + validatedOne := false + var bestValidationResult *Result + + for i, v := range value { + subContext := NewJsonContext(strconv.Itoa(i), context) + + validationResult := currentSubSchema.contains.subValidateWithContext(v, subContext) + if validationResult.Valid() { + validatedOne = true + break + } else { + if bestValidationResult == nil || validationResult.score > bestValidationResult.score { + bestValidationResult = validationResult + } + } + } + if !validatedOne { + result.addInternalError( + new(ArrayContainsError), + context, + value, + ErrorDetails{}, + ) + if bestValidationResult != nil { + result.mergeErrors(bestValidationResult) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validateObject(currentSubSchema *subSchema, value map[string]interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateObject %s", context.String()) + internalLog(" %v", value) + } + + // minProperties & maxProperties: + if currentSubSchema.minProperties != nil { + if len(value) < int(*currentSubSchema.minProperties) { + result.addInternalError( + new(ArrayMinPropertiesError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minProperties}, + ) + } + } + if currentSubSchema.maxProperties != nil { + if len(value) > int(*currentSubSchema.maxProperties) { + result.addInternalError( + new(ArrayMaxPropertiesError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxProperties}, + ) + } + } + + // required: + for _, requiredProperty := range currentSubSchema.required { + _, ok := value[requiredProperty] + if ok { + result.incrementScore() + } else { + result.addInternalError( + new(RequiredError), + context, + value, + ErrorDetails{"property": requiredProperty}, + ) + } + } + + // additionalProperty & patternProperty: + if currentSubSchema.additionalProperties != nil { + + switch currentSubSchema.additionalProperties.(type) { + case bool: + + if !currentSubSchema.additionalProperties.(bool) { + + for pk := range value { + + found := false + for _, spValue := range currentSubSchema.propertiesChildren { + if pk == spValue.property { + found = true + } + } + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if found { + + if pp_has && !pp_match { + result.addInternalError( + new(AdditionalPropertyNotAllowedError), + context, + value[pk], + ErrorDetails{"property": pk}, + ) + } + + } else { + + if !pp_has || !pp_match { + result.addInternalError( + new(AdditionalPropertyNotAllowedError), + context, + value[pk], + ErrorDetails{"property": pk}, + ) + } + + } + } + } + + case *subSchema: + + additionalPropertiesSchema := currentSubSchema.additionalProperties.(*subSchema) + for pk := range value { + + found := false + for _, spValue := range currentSubSchema.propertiesChildren { + if pk == spValue.property { + found = true + } + } + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if found { + + if pp_has && !pp_match { + validationResult := additionalPropertiesSchema.subValidateWithContext(value[pk], context) + result.mergeErrors(validationResult) + } + + } else { + + if !pp_has || !pp_match { + validationResult := additionalPropertiesSchema.subValidateWithContext(value[pk], context) + result.mergeErrors(validationResult) + } + + } + + } + } + } else { + + for pk := range value { + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if pp_has && !pp_match { + + result.addInternalError( + new(InvalidPropertyPatternError), + context, + value[pk], + ErrorDetails{ + "property": pk, + "pattern": currentSubSchema.PatternPropertiesString(), + }, + ) + } + + } + } + + // propertyNames: + if currentSubSchema.propertyNames != nil { + for pk := range value { + validationResult := currentSubSchema.propertyNames.subValidateWithContext(pk, context) + if !validationResult.Valid() { + result.addInternalError(new(InvalidPropertyNameError), + context, + value, ErrorDetails{ + "property": pk, + }) + result.mergeErrors(validationResult) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validatePatternProperty(currentSubSchema *subSchema, key string, value interface{}, result *Result, context *JsonContext) (has bool, matched bool) { + + if internalLogEnabled { + internalLog("validatePatternProperty %s", context.String()) + internalLog(" %s %v", key, value) + } + + has = false + + validatedkey := false + + for pk, pv := range currentSubSchema.patternProperties { + if matches, _ := regexp.MatchString(pk, key); matches { + has = true + subContext := NewJsonContext(key, context) + validationResult := pv.subValidateWithContext(value, subContext) + result.mergeErrors(validationResult) + validatedkey = true + } + } + + if !validatedkey { + return has, false + } + + result.incrementScore() + + return has, true +} + +func (v *subSchema) validateString(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + // Ignore JSON numbers + if isJsonNumber(value) { + return + } + + // Ignore non strings + if !isKind(value, reflect.String) { + return + } + + if internalLogEnabled { + internalLog("validateString %s", context.String()) + internalLog(" %v", value) + } + + stringValue := value.(string) + + // minLength & maxLength: + if currentSubSchema.minLength != nil { + if utf8.RuneCount([]byte(stringValue)) < int(*currentSubSchema.minLength) { + result.addInternalError( + new(StringLengthGTEError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minLength}, + ) + } + } + if currentSubSchema.maxLength != nil { + if utf8.RuneCount([]byte(stringValue)) > int(*currentSubSchema.maxLength) { + result.addInternalError( + new(StringLengthLTEError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxLength}, + ) + } + } + + // pattern: + if currentSubSchema.pattern != nil { + if !currentSubSchema.pattern.MatchString(stringValue) { + result.addInternalError( + new(DoesNotMatchPatternError), + context, + value, + ErrorDetails{"pattern": currentSubSchema.pattern}, + ) + + } + } + + // format + if currentSubSchema.format != "" { + if !FormatCheckers.IsFormat(currentSubSchema.format, stringValue) { + result.addInternalError( + new(DoesNotMatchFormatError), + context, + value, + ErrorDetails{"format": currentSubSchema.format}, + ) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateNumber(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + // Ignore non numbers + if !isJsonNumber(value) { + return + } + + if internalLogEnabled { + internalLog("validateNumber %s", context.String()) + internalLog(" %v", value) + } + + number := value.(json.Number) + float64Value, _ := new(big.Float).SetString(string(number)) + + // multipleOf: + if currentSubSchema.multipleOf != nil { + + if q := new(big.Float).Quo(float64Value, currentSubSchema.multipleOf); !q.IsInt() { + result.addInternalError( + new(MultipleOfError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{"multiple": currentSubSchema.multipleOf}, + ) + } + } + + //maximum & exclusiveMaximum: + if currentSubSchema.maximum != nil { + if currentSubSchema.exclusiveMaximum { + if float64Value.Cmp(currentSubSchema.maximum) >= 0 { + result.addInternalError( + new(NumberLTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": currentSubSchema.maximum, + }, + ) + } + } else { + if float64Value.Cmp(currentSubSchema.maximum) == 1 { + result.addInternalError( + new(NumberLTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": currentSubSchema.maximum, + }, + ) + } + } + } + + //minimum & exclusiveMinimum: + if currentSubSchema.minimum != nil { + if currentSubSchema.exclusiveMinimum { + if float64Value.Cmp(currentSubSchema.minimum) <= 0 { + // if float64Value <= *currentSubSchema.minimum { + result.addInternalError( + new(NumberGTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": currentSubSchema.minimum, + }, + ) + } + } else { + if float64Value.Cmp(currentSubSchema.minimum) == -1 { + result.addInternalError( + new(NumberGTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": currentSubSchema.minimum, + }, + ) + } + } + } + + // format + if currentSubSchema.format != "" { + if !FormatCheckers.IsFormat(currentSubSchema.format, float64Value) { + result.addInternalError( + new(DoesNotMatchFormatError), + context, + value, + ErrorDetails{"format": currentSubSchema.format}, + ) + } + } + + result.incrementScore() +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE new file mode 100644 index 00000000000..8dada3edaf5 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml new file mode 100644 index 00000000000..8da58fbf6f8 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml @@ -0,0 +1,31 @@ +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original copyright and license: + + apic.go + emitterc.go + parserc.go + readerc.go + scannerc.go + writerc.go + yamlh.go + yamlprivateh.go + +Copyright (c) 2006 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go new file mode 100644 index 00000000000..1f7e87e6727 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go @@ -0,0 +1,739 @@ +package yaml + +import ( + "io" +) + +func yaml_insert_token(parser *yaml_parser_t, pos int, token *yaml_token_t) { + //fmt.Println("yaml_insert_token", "pos:", pos, "typ:", token.typ, "head:", parser.tokens_head, "len:", len(parser.tokens)) + + // Check if we can move the queue at the beginning of the buffer. + if parser.tokens_head > 0 && len(parser.tokens) == cap(parser.tokens) { + if parser.tokens_head != len(parser.tokens) { + copy(parser.tokens, parser.tokens[parser.tokens_head:]) + } + parser.tokens = parser.tokens[:len(parser.tokens)-parser.tokens_head] + parser.tokens_head = 0 + } + parser.tokens = append(parser.tokens, *token) + if pos < 0 { + return + } + copy(parser.tokens[parser.tokens_head+pos+1:], parser.tokens[parser.tokens_head+pos:]) + parser.tokens[parser.tokens_head+pos] = *token +} + +// Create a new parser object. +func yaml_parser_initialize(parser *yaml_parser_t) bool { + *parser = yaml_parser_t{ + raw_buffer: make([]byte, 0, input_raw_buffer_size), + buffer: make([]byte, 0, input_buffer_size), + } + return true +} + +// Destroy a parser object. +func yaml_parser_delete(parser *yaml_parser_t) { + *parser = yaml_parser_t{} +} + +// String read handler. +func yaml_string_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + if parser.input_pos == len(parser.input) { + return 0, io.EOF + } + n = copy(buffer, parser.input[parser.input_pos:]) + parser.input_pos += n + return n, nil +} + +// Reader read handler. +func yaml_reader_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + return parser.input_reader.Read(buffer) +} + +// Set a string input. +func yaml_parser_set_input_string(parser *yaml_parser_t, input []byte) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_string_read_handler + parser.input = input + parser.input_pos = 0 +} + +// Set a file input. +func yaml_parser_set_input_reader(parser *yaml_parser_t, r io.Reader) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_reader_read_handler + parser.input_reader = r +} + +// Set the source encoding. +func yaml_parser_set_encoding(parser *yaml_parser_t, encoding yaml_encoding_t) { + if parser.encoding != yaml_ANY_ENCODING { + panic("must set the encoding only once") + } + parser.encoding = encoding +} + +// Create a new emitter object. +func yaml_emitter_initialize(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{ + buffer: make([]byte, output_buffer_size), + raw_buffer: make([]byte, 0, output_raw_buffer_size), + states: make([]yaml_emitter_state_t, 0, initial_stack_size), + events: make([]yaml_event_t, 0, initial_queue_size), + } +} + +// Destroy an emitter object. +func yaml_emitter_delete(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{} +} + +// String write handler. +func yaml_string_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + *emitter.output_buffer = append(*emitter.output_buffer, buffer...) + return nil +} + +// yaml_writer_write_handler uses emitter.output_writer to write the +// emitted text. +func yaml_writer_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + _, err := emitter.output_writer.Write(buffer) + return err +} + +// Set a string output. +func yaml_emitter_set_output_string(emitter *yaml_emitter_t, output_buffer *[]byte) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_string_write_handler + emitter.output_buffer = output_buffer +} + +// Set a file output. +func yaml_emitter_set_output_writer(emitter *yaml_emitter_t, w io.Writer) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_writer_write_handler + emitter.output_writer = w +} + +// Set the output encoding. +func yaml_emitter_set_encoding(emitter *yaml_emitter_t, encoding yaml_encoding_t) { + if emitter.encoding != yaml_ANY_ENCODING { + panic("must set the output encoding only once") + } + emitter.encoding = encoding +} + +// Set the canonical output style. +func yaml_emitter_set_canonical(emitter *yaml_emitter_t, canonical bool) { + emitter.canonical = canonical +} + +//// Set the indentation increment. +func yaml_emitter_set_indent(emitter *yaml_emitter_t, indent int) { + if indent < 2 || indent > 9 { + indent = 2 + } + emitter.best_indent = indent +} + +// Set the preferred line width. +func yaml_emitter_set_width(emitter *yaml_emitter_t, width int) { + if width < 0 { + width = -1 + } + emitter.best_width = width +} + +// Set if unescaped non-ASCII characters are allowed. +func yaml_emitter_set_unicode(emitter *yaml_emitter_t, unicode bool) { + emitter.unicode = unicode +} + +// Set the preferred line break character. +func yaml_emitter_set_break(emitter *yaml_emitter_t, line_break yaml_break_t) { + emitter.line_break = line_break +} + +///* +// * Destroy a token object. +// */ +// +//YAML_DECLARE(void) +//yaml_token_delete(yaml_token_t *token) +//{ +// assert(token); // Non-NULL token object expected. +// +// switch (token.type) +// { +// case YAML_TAG_DIRECTIVE_TOKEN: +// yaml_free(token.data.tag_directive.handle); +// yaml_free(token.data.tag_directive.prefix); +// break; +// +// case YAML_ALIAS_TOKEN: +// yaml_free(token.data.alias.value); +// break; +// +// case YAML_ANCHOR_TOKEN: +// yaml_free(token.data.anchor.value); +// break; +// +// case YAML_TAG_TOKEN: +// yaml_free(token.data.tag.handle); +// yaml_free(token.data.tag.suffix); +// break; +// +// case YAML_SCALAR_TOKEN: +// yaml_free(token.data.scalar.value); +// break; +// +// default: +// break; +// } +// +// memset(token, 0, sizeof(yaml_token_t)); +//} +// +///* +// * Check if a string is a valid UTF-8 sequence. +// * +// * Check 'reader.c' for more details on UTF-8 encoding. +// */ +// +//static int +//yaml_check_utf8(yaml_char_t *start, size_t length) +//{ +// yaml_char_t *end = start+length; +// yaml_char_t *pointer = start; +// +// while (pointer < end) { +// unsigned char octet; +// unsigned int width; +// unsigned int value; +// size_t k; +// +// octet = pointer[0]; +// width = (octet & 0x80) == 0x00 ? 1 : +// (octet & 0xE0) == 0xC0 ? 2 : +// (octet & 0xF0) == 0xE0 ? 3 : +// (octet & 0xF8) == 0xF0 ? 4 : 0; +// value = (octet & 0x80) == 0x00 ? octet & 0x7F : +// (octet & 0xE0) == 0xC0 ? octet & 0x1F : +// (octet & 0xF0) == 0xE0 ? octet & 0x0F : +// (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; +// if (!width) return 0; +// if (pointer+width > end) return 0; +// for (k = 1; k < width; k ++) { +// octet = pointer[k]; +// if ((octet & 0xC0) != 0x80) return 0; +// value = (value << 6) + (octet & 0x3F); +// } +// if (!((width == 1) || +// (width == 2 && value >= 0x80) || +// (width == 3 && value >= 0x800) || +// (width == 4 && value >= 0x10000))) return 0; +// +// pointer += width; +// } +// +// return 1; +//} +// + +// Create STREAM-START. +func yaml_stream_start_event_initialize(event *yaml_event_t, encoding yaml_encoding_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + encoding: encoding, + } +} + +// Create STREAM-END. +func yaml_stream_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + } +} + +// Create DOCUMENT-START. +func yaml_document_start_event_initialize( + event *yaml_event_t, + version_directive *yaml_version_directive_t, + tag_directives []yaml_tag_directive_t, + implicit bool, +) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: implicit, + } +} + +// Create DOCUMENT-END. +func yaml_document_end_event_initialize(event *yaml_event_t, implicit bool) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + implicit: implicit, + } +} + +///* +// * Create ALIAS. +// */ +// +//YAML_DECLARE(int) +//yaml_alias_event_initialize(event *yaml_event_t, anchor *yaml_char_t) +//{ +// mark yaml_mark_t = { 0, 0, 0 } +// anchor_copy *yaml_char_t = NULL +// +// assert(event) // Non-NULL event object is expected. +// assert(anchor) // Non-NULL anchor is expected. +// +// if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0 +// +// anchor_copy = yaml_strdup(anchor) +// if (!anchor_copy) +// return 0 +// +// ALIAS_EVENT_INIT(*event, anchor_copy, mark, mark) +// +// return 1 +//} + +// Create SCALAR. +func yaml_scalar_event_initialize(event *yaml_event_t, anchor, tag, value []byte, plain_implicit, quoted_implicit bool, style yaml_scalar_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + anchor: anchor, + tag: tag, + value: value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-START. +func yaml_sequence_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_sequence_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-END. +func yaml_sequence_end_event_initialize(event *yaml_event_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + } + return true +} + +// Create MAPPING-START. +func yaml_mapping_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_mapping_style_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } +} + +// Create MAPPING-END. +func yaml_mapping_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + } +} + +// Destroy an event object. +func yaml_event_delete(event *yaml_event_t) { + *event = yaml_event_t{} +} + +///* +// * Create a document object. +// */ +// +//YAML_DECLARE(int) +//yaml_document_initialize(document *yaml_document_t, +// version_directive *yaml_version_directive_t, +// tag_directives_start *yaml_tag_directive_t, +// tag_directives_end *yaml_tag_directive_t, +// start_implicit int, end_implicit int) +//{ +// struct { +// error yaml_error_type_t +// } context +// struct { +// start *yaml_node_t +// end *yaml_node_t +// top *yaml_node_t +// } nodes = { NULL, NULL, NULL } +// version_directive_copy *yaml_version_directive_t = NULL +// struct { +// start *yaml_tag_directive_t +// end *yaml_tag_directive_t +// top *yaml_tag_directive_t +// } tag_directives_copy = { NULL, NULL, NULL } +// value yaml_tag_directive_t = { NULL, NULL } +// mark yaml_mark_t = { 0, 0, 0 } +// +// assert(document) // Non-NULL document object is expected. +// assert((tag_directives_start && tag_directives_end) || +// (tag_directives_start == tag_directives_end)) +// // Valid tag directives are expected. +// +// if (!STACK_INIT(&context, nodes, INITIAL_STACK_SIZE)) goto error +// +// if (version_directive) { +// version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)) +// if (!version_directive_copy) goto error +// version_directive_copy.major = version_directive.major +// version_directive_copy.minor = version_directive.minor +// } +// +// if (tag_directives_start != tag_directives_end) { +// tag_directive *yaml_tag_directive_t +// if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) +// goto error +// for (tag_directive = tag_directives_start +// tag_directive != tag_directives_end; tag_directive ++) { +// assert(tag_directive.handle) +// assert(tag_directive.prefix) +// if (!yaml_check_utf8(tag_directive.handle, +// strlen((char *)tag_directive.handle))) +// goto error +// if (!yaml_check_utf8(tag_directive.prefix, +// strlen((char *)tag_directive.prefix))) +// goto error +// value.handle = yaml_strdup(tag_directive.handle) +// value.prefix = yaml_strdup(tag_directive.prefix) +// if (!value.handle || !value.prefix) goto error +// if (!PUSH(&context, tag_directives_copy, value)) +// goto error +// value.handle = NULL +// value.prefix = NULL +// } +// } +// +// DOCUMENT_INIT(*document, nodes.start, nodes.end, version_directive_copy, +// tag_directives_copy.start, tag_directives_copy.top, +// start_implicit, end_implicit, mark, mark) +// +// return 1 +// +//error: +// STACK_DEL(&context, nodes) +// yaml_free(version_directive_copy) +// while (!STACK_EMPTY(&context, tag_directives_copy)) { +// value yaml_tag_directive_t = POP(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// } +// STACK_DEL(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// +// return 0 +//} +// +///* +// * Destroy a document object. +// */ +// +//YAML_DECLARE(void) +//yaml_document_delete(document *yaml_document_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// tag_directive *yaml_tag_directive_t +// +// context.error = YAML_NO_ERROR // Eliminate a compiler warning. +// +// assert(document) // Non-NULL document object is expected. +// +// while (!STACK_EMPTY(&context, document.nodes)) { +// node yaml_node_t = POP(&context, document.nodes) +// yaml_free(node.tag) +// switch (node.type) { +// case YAML_SCALAR_NODE: +// yaml_free(node.data.scalar.value) +// break +// case YAML_SEQUENCE_NODE: +// STACK_DEL(&context, node.data.sequence.items) +// break +// case YAML_MAPPING_NODE: +// STACK_DEL(&context, node.data.mapping.pairs) +// break +// default: +// assert(0) // Should not happen. +// } +// } +// STACK_DEL(&context, document.nodes) +// +// yaml_free(document.version_directive) +// for (tag_directive = document.tag_directives.start +// tag_directive != document.tag_directives.end +// tag_directive++) { +// yaml_free(tag_directive.handle) +// yaml_free(tag_directive.prefix) +// } +// yaml_free(document.tag_directives.start) +// +// memset(document, 0, sizeof(yaml_document_t)) +//} +// +///** +// * Get a document node. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_node(document *yaml_document_t, index int) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (index > 0 && document.nodes.start + index <= document.nodes.top) { +// return document.nodes.start + index - 1 +// } +// return NULL +//} +// +///** +// * Get the root object. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_root_node(document *yaml_document_t) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (document.nodes.top != document.nodes.start) { +// return document.nodes.start +// } +// return NULL +//} +// +///* +// * Add a scalar node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_scalar(document *yaml_document_t, +// tag *yaml_char_t, value *yaml_char_t, length int, +// style yaml_scalar_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// value_copy *yaml_char_t = NULL +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// assert(value) // Non-NULL value is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (length < 0) { +// length = strlen((char *)value) +// } +// +// if (!yaml_check_utf8(value, length)) goto error +// value_copy = yaml_malloc(length+1) +// if (!value_copy) goto error +// memcpy(value_copy, value, length) +// value_copy[length] = '\0' +// +// SCALAR_NODE_INIT(node, tag_copy, value_copy, length, style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// yaml_free(tag_copy) +// yaml_free(value_copy) +// +// return 0 +//} +// +///* +// * Add a sequence node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_sequence(document *yaml_document_t, +// tag *yaml_char_t, style yaml_sequence_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_item_t +// end *yaml_node_item_t +// top *yaml_node_item_t +// } items = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, items, INITIAL_STACK_SIZE)) goto error +// +// SEQUENCE_NODE_INIT(node, tag_copy, items.start, items.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, items) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Add a mapping node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_mapping(document *yaml_document_t, +// tag *yaml_char_t, style yaml_mapping_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_pair_t +// end *yaml_node_pair_t +// top *yaml_node_pair_t +// } pairs = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, pairs, INITIAL_STACK_SIZE)) goto error +// +// MAPPING_NODE_INIT(node, tag_copy, pairs.start, pairs.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, pairs) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Append an item to a sequence node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_sequence_item(document *yaml_document_t, +// sequence int, item int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// assert(document) // Non-NULL document is required. +// assert(sequence > 0 +// && document.nodes.start + sequence <= document.nodes.top) +// // Valid sequence id is required. +// assert(document.nodes.start[sequence-1].type == YAML_SEQUENCE_NODE) +// // A sequence node is required. +// assert(item > 0 && document.nodes.start + item <= document.nodes.top) +// // Valid item id is required. +// +// if (!PUSH(&context, +// document.nodes.start[sequence-1].data.sequence.items, item)) +// return 0 +// +// return 1 +//} +// +///* +// * Append a pair of a key and a value to a mapping node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_mapping_pair(document *yaml_document_t, +// mapping int, key int, value int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// pair yaml_node_pair_t +// +// assert(document) // Non-NULL document is required. +// assert(mapping > 0 +// && document.nodes.start + mapping <= document.nodes.top) +// // Valid mapping id is required. +// assert(document.nodes.start[mapping-1].type == YAML_MAPPING_NODE) +// // A mapping node is required. +// assert(key > 0 && document.nodes.start + key <= document.nodes.top) +// // Valid key id is required. +// assert(value > 0 && document.nodes.start + value <= document.nodes.top) +// // Valid value id is required. +// +// pair.key = key +// pair.value = value +// +// if (!PUSH(&context, +// document.nodes.start[mapping-1].data.mapping.pairs, pair)) +// return 0 +// +// return 1 +//} +// +// diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go new file mode 100644 index 00000000000..e4e56e28e0e --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go @@ -0,0 +1,775 @@ +package yaml + +import ( + "encoding" + "encoding/base64" + "fmt" + "io" + "math" + "reflect" + "strconv" + "time" +) + +const ( + documentNode = 1 << iota + mappingNode + sequenceNode + scalarNode + aliasNode +) + +type node struct { + kind int + line, column int + tag string + // For an alias node, alias holds the resolved alias. + alias *node + value string + implicit bool + children []*node + anchors map[string]*node +} + +// ---------------------------------------------------------------------------- +// Parser, produces a node tree out of a libyaml event stream. + +type parser struct { + parser yaml_parser_t + event yaml_event_t + doc *node + doneInit bool +} + +func newParser(b []byte) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + if len(b) == 0 { + b = []byte{'\n'} + } + yaml_parser_set_input_string(&p.parser, b) + return &p +} + +func newParserFromReader(r io.Reader) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + yaml_parser_set_input_reader(&p.parser, r) + return &p +} + +func (p *parser) init() { + if p.doneInit { + return + } + p.expect(yaml_STREAM_START_EVENT) + p.doneInit = true +} + +func (p *parser) destroy() { + if p.event.typ != yaml_NO_EVENT { + yaml_event_delete(&p.event) + } + yaml_parser_delete(&p.parser) +} + +// expect consumes an event from the event stream and +// checks that it's of the expected type. +func (p *parser) expect(e yaml_event_type_t) { + if p.event.typ == yaml_NO_EVENT { + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + } + if p.event.typ == yaml_STREAM_END_EVENT { + failf("attempted to go past the end of stream; corrupted value?") + } + if p.event.typ != e { + p.parser.problem = fmt.Sprintf("expected %s event but got %s", e, p.event.typ) + p.fail() + } + yaml_event_delete(&p.event) + p.event.typ = yaml_NO_EVENT +} + +// peek peeks at the next event in the event stream, +// puts the results into p.event and returns the event type. +func (p *parser) peek() yaml_event_type_t { + if p.event.typ != yaml_NO_EVENT { + return p.event.typ + } + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + return p.event.typ +} + +func (p *parser) fail() { + var where string + var line int + if p.parser.problem_mark.line != 0 { + line = p.parser.problem_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } else if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + } + if line != 0 { + where = "line " + strconv.Itoa(line) + ": " + } + var msg string + if len(p.parser.problem) > 0 { + msg = p.parser.problem + } else { + msg = "unknown problem parsing YAML content" + } + failf("%s%s", where, msg) +} + +func (p *parser) anchor(n *node, anchor []byte) { + if anchor != nil { + p.doc.anchors[string(anchor)] = n + } +} + +func (p *parser) parse() *node { + p.init() + switch p.peek() { + case yaml_SCALAR_EVENT: + return p.scalar() + case yaml_ALIAS_EVENT: + return p.alias() + case yaml_MAPPING_START_EVENT: + return p.mapping() + case yaml_SEQUENCE_START_EVENT: + return p.sequence() + case yaml_DOCUMENT_START_EVENT: + return p.document() + case yaml_STREAM_END_EVENT: + // Happens when attempting to decode an empty buffer. + return nil + default: + panic("attempted to parse unknown event: " + p.event.typ.String()) + } +} + +func (p *parser) node(kind int) *node { + return &node{ + kind: kind, + line: p.event.start_mark.line, + column: p.event.start_mark.column, + } +} + +func (p *parser) document() *node { + n := p.node(documentNode) + n.anchors = make(map[string]*node) + p.doc = n + p.expect(yaml_DOCUMENT_START_EVENT) + n.children = append(n.children, p.parse()) + p.expect(yaml_DOCUMENT_END_EVENT) + return n +} + +func (p *parser) alias() *node { + n := p.node(aliasNode) + n.value = string(p.event.anchor) + n.alias = p.doc.anchors[n.value] + if n.alias == nil { + failf("unknown anchor '%s' referenced", n.value) + } + p.expect(yaml_ALIAS_EVENT) + return n +} + +func (p *parser) scalar() *node { + n := p.node(scalarNode) + n.value = string(p.event.value) + n.tag = string(p.event.tag) + n.implicit = p.event.implicit + p.anchor(n, p.event.anchor) + p.expect(yaml_SCALAR_EVENT) + return n +} + +func (p *parser) sequence() *node { + n := p.node(sequenceNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_SEQUENCE_START_EVENT) + for p.peek() != yaml_SEQUENCE_END_EVENT { + n.children = append(n.children, p.parse()) + } + p.expect(yaml_SEQUENCE_END_EVENT) + return n +} + +func (p *parser) mapping() *node { + n := p.node(mappingNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_MAPPING_START_EVENT) + for p.peek() != yaml_MAPPING_END_EVENT { + n.children = append(n.children, p.parse(), p.parse()) + } + p.expect(yaml_MAPPING_END_EVENT) + return n +} + +// ---------------------------------------------------------------------------- +// Decoder, unmarshals a node into a provided value. + +type decoder struct { + doc *node + aliases map[*node]bool + mapType reflect.Type + terrors []string + strict bool +} + +var ( + mapItemType = reflect.TypeOf(MapItem{}) + durationType = reflect.TypeOf(time.Duration(0)) + defaultMapType = reflect.TypeOf(map[interface{}]interface{}{}) + ifaceType = defaultMapType.Elem() + timeType = reflect.TypeOf(time.Time{}) + ptrTimeType = reflect.TypeOf(&time.Time{}) +) + +func newDecoder(strict bool) *decoder { + d := &decoder{mapType: defaultMapType, strict: strict} + d.aliases = make(map[*node]bool) + return d +} + +func (d *decoder) terror(n *node, tag string, out reflect.Value) { + if n.tag != "" { + tag = n.tag + } + value := n.value + if tag != yaml_SEQ_TAG && tag != yaml_MAP_TAG { + if len(value) > 10 { + value = " `" + value[:7] + "...`" + } else { + value = " `" + value + "`" + } + } + d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.line+1, shortTag(tag), value, out.Type())) +} + +func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) { + terrlen := len(d.terrors) + err := u.UnmarshalYAML(func(v interface{}) (err error) { + defer handleErr(&err) + d.unmarshal(n, reflect.ValueOf(v)) + if len(d.terrors) > terrlen { + issues := d.terrors[terrlen:] + d.terrors = d.terrors[:terrlen] + return &TypeError{issues} + } + return nil + }) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +// d.prepare initializes and dereferences pointers and calls UnmarshalYAML +// if a value is found to implement it. +// It returns the initialized and dereferenced out value, whether +// unmarshalling was already done by UnmarshalYAML, and if so whether +// its types unmarshalled appropriately. +// +// If n holds a null value, prepare returns before doing anything. +func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { + if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "~" || n.value == "" && n.implicit) { + return out, false, false + } + again := true + for again { + again = false + if out.Kind() == reflect.Ptr { + if out.IsNil() { + out.Set(reflect.New(out.Type().Elem())) + } + out = out.Elem() + again = true + } + if out.CanAddr() { + if u, ok := out.Addr().Interface().(Unmarshaler); ok { + good = d.callUnmarshaler(n, u) + return out, true, good + } + } + } + return out, false, false +} + +func (d *decoder) unmarshal(n *node, out reflect.Value) (good bool) { + switch n.kind { + case documentNode: + return d.document(n, out) + case aliasNode: + return d.alias(n, out) + } + out, unmarshaled, good := d.prepare(n, out) + if unmarshaled { + return good + } + switch n.kind { + case scalarNode: + good = d.scalar(n, out) + case mappingNode: + good = d.mapping(n, out) + case sequenceNode: + good = d.sequence(n, out) + default: + panic("internal error: unknown node kind: " + strconv.Itoa(n.kind)) + } + return good +} + +func (d *decoder) document(n *node, out reflect.Value) (good bool) { + if len(n.children) == 1 { + d.doc = n + d.unmarshal(n.children[0], out) + return true + } + return false +} + +func (d *decoder) alias(n *node, out reflect.Value) (good bool) { + if d.aliases[n] { + // TODO this could actually be allowed in some circumstances. + failf("anchor '%s' value contains itself", n.value) + } + d.aliases[n] = true + good = d.unmarshal(n.alias, out) + delete(d.aliases, n) + return good +} + +var zeroValue reflect.Value + +func resetMap(out reflect.Value) { + for _, k := range out.MapKeys() { + out.SetMapIndex(k, zeroValue) + } +} + +func (d *decoder) scalar(n *node, out reflect.Value) bool { + var tag string + var resolved interface{} + if n.tag == "" && !n.implicit { + tag = yaml_STR_TAG + resolved = n.value + } else { + tag, resolved = resolve(n.tag, n.value) + if tag == yaml_BINARY_TAG { + data, err := base64.StdEncoding.DecodeString(resolved.(string)) + if err != nil { + failf("!!binary value contains invalid base64 data") + } + resolved = string(data) + } + } + if resolved == nil { + if out.Kind() == reflect.Map && !out.CanAddr() { + resetMap(out) + } else { + out.Set(reflect.Zero(out.Type())) + } + return true + } + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + // We've resolved to exactly the type we want, so use that. + out.Set(resolvedv) + return true + } + // Perhaps we can use the value as a TextUnmarshaler to + // set its value. + if out.CanAddr() { + u, ok := out.Addr().Interface().(encoding.TextUnmarshaler) + if ok { + var text []byte + if tag == yaml_BINARY_TAG { + text = []byte(resolved.(string)) + } else { + // We let any value be unmarshaled into TextUnmarshaler. + // That might be more lax than we'd like, but the + // TextUnmarshaler itself should bowl out any dubious values. + text = []byte(n.value) + } + err := u.UnmarshalText(text) + if err != nil { + fail(err) + } + return true + } + } + switch out.Kind() { + case reflect.String: + if tag == yaml_BINARY_TAG { + out.SetString(resolved.(string)) + return true + } + if resolved != nil { + out.SetString(n.value) + return true + } + case reflect.Interface: + if resolved == nil { + out.Set(reflect.Zero(out.Type())) + } else if tag == yaml_TIMESTAMP_TAG { + // It looks like a timestamp but for backward compatibility + // reasons we set it as a string, so that code that unmarshals + // timestamp-like values into interface{} will continue to + // see a string and not a time.Time. + // TODO(v3) Drop this. + out.Set(reflect.ValueOf(n.value)) + } else { + out.Set(reflect.ValueOf(resolved)) + } + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch resolved := resolved.(type) { + case int: + if !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case int64: + if !out.OverflowInt(resolved) { + out.SetInt(resolved) + return true + } + case uint64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case float64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case string: + if out.Type() == durationType { + d, err := time.ParseDuration(resolved) + if err == nil { + out.SetInt(int64(d)) + return true + } + } + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch resolved := resolved.(type) { + case int: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case int64: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case uint64: + if !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case float64: + if resolved <= math.MaxUint64 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + } + case reflect.Bool: + switch resolved := resolved.(type) { + case bool: + out.SetBool(resolved) + return true + } + case reflect.Float32, reflect.Float64: + switch resolved := resolved.(type) { + case int: + out.SetFloat(float64(resolved)) + return true + case int64: + out.SetFloat(float64(resolved)) + return true + case uint64: + out.SetFloat(float64(resolved)) + return true + case float64: + out.SetFloat(resolved) + return true + } + case reflect.Struct: + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + out.Set(resolvedv) + return true + } + case reflect.Ptr: + if out.Type().Elem() == reflect.TypeOf(resolved) { + // TODO DOes this make sense? When is out a Ptr except when decoding a nil value? + elem := reflect.New(out.Type().Elem()) + elem.Elem().Set(reflect.ValueOf(resolved)) + out.Set(elem) + return true + } + } + d.terror(n, tag, out) + return false +} + +func settableValueOf(i interface{}) reflect.Value { + v := reflect.ValueOf(i) + sv := reflect.New(v.Type()).Elem() + sv.Set(v) + return sv +} + +func (d *decoder) sequence(n *node, out reflect.Value) (good bool) { + l := len(n.children) + + var iface reflect.Value + switch out.Kind() { + case reflect.Slice: + out.Set(reflect.MakeSlice(out.Type(), l, l)) + case reflect.Array: + if l != out.Len() { + failf("invalid array: want %d elements but got %d", out.Len(), l) + } + case reflect.Interface: + // No type hints. Will have to use a generic sequence. + iface = out + out = settableValueOf(make([]interface{}, l)) + default: + d.terror(n, yaml_SEQ_TAG, out) + return false + } + et := out.Type().Elem() + + j := 0 + for i := 0; i < l; i++ { + e := reflect.New(et).Elem() + if ok := d.unmarshal(n.children[i], e); ok { + out.Index(j).Set(e) + j++ + } + } + if out.Kind() != reflect.Array { + out.Set(out.Slice(0, j)) + } + if iface.IsValid() { + iface.Set(out) + } + return true +} + +func (d *decoder) mapping(n *node, out reflect.Value) (good bool) { + switch out.Kind() { + case reflect.Struct: + return d.mappingStruct(n, out) + case reflect.Slice: + return d.mappingSlice(n, out) + case reflect.Map: + // okay + case reflect.Interface: + if d.mapType.Kind() == reflect.Map { + iface := out + out = reflect.MakeMap(d.mapType) + iface.Set(out) + } else { + slicev := reflect.New(d.mapType).Elem() + if !d.mappingSlice(n, slicev) { + return false + } + out.Set(slicev) + return true + } + default: + d.terror(n, yaml_MAP_TAG, out) + return false + } + outt := out.Type() + kt := outt.Key() + et := outt.Elem() + + mapType := d.mapType + if outt.Key() == ifaceType && outt.Elem() == ifaceType { + d.mapType = outt + } + + if out.IsNil() { + out.Set(reflect.MakeMap(outt)) + } + l := len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + k := reflect.New(kt).Elem() + if d.unmarshal(n.children[i], k) { + kkind := k.Kind() + if kkind == reflect.Interface { + kkind = k.Elem().Kind() + } + if kkind == reflect.Map || kkind == reflect.Slice { + failf("invalid map key: %#v", k.Interface()) + } + e := reflect.New(et).Elem() + if d.unmarshal(n.children[i+1], e) { + d.setMapIndex(n.children[i+1], out, k, e) + } + } + } + d.mapType = mapType + return true +} + +func (d *decoder) setMapIndex(n *node, out, k, v reflect.Value) { + if d.strict && out.MapIndex(k) != zeroValue { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: key %#v already set in map", n.line+1, k.Interface())) + return + } + out.SetMapIndex(k, v) +} + +func (d *decoder) mappingSlice(n *node, out reflect.Value) (good bool) { + outt := out.Type() + if outt.Elem() != mapItemType { + d.terror(n, yaml_MAP_TAG, out) + return false + } + + mapType := d.mapType + d.mapType = outt + + var slice []MapItem + var l = len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + item := MapItem{} + k := reflect.ValueOf(&item.Key).Elem() + if d.unmarshal(n.children[i], k) { + v := reflect.ValueOf(&item.Value).Elem() + if d.unmarshal(n.children[i+1], v) { + slice = append(slice, item) + } + } + } + out.Set(reflect.ValueOf(slice)) + d.mapType = mapType + return true +} + +func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) { + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + name := settableValueOf("") + l := len(n.children) + + var inlineMap reflect.Value + var elemType reflect.Type + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + inlineMap.Set(reflect.New(inlineMap.Type()).Elem()) + elemType = inlineMap.Type().Elem() + } + + var doneFields []bool + if d.strict { + doneFields = make([]bool, len(sinfo.FieldsList)) + } + for i := 0; i < l; i += 2 { + ni := n.children[i] + if isMerge(ni) { + d.merge(n.children[i+1], out) + continue + } + if !d.unmarshal(ni, name) { + continue + } + if info, ok := sinfo.FieldsMap[name.String()]; ok { + if d.strict { + if doneFields[info.Id] { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.line+1, name.String(), out.Type())) + continue + } + doneFields[info.Id] = true + } + var field reflect.Value + if info.Inline == nil { + field = out.Field(info.Num) + } else { + field = out.FieldByIndex(info.Inline) + } + d.unmarshal(n.children[i+1], field) + } else if sinfo.InlineMap != -1 { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + value := reflect.New(elemType).Elem() + d.unmarshal(n.children[i+1], value) + d.setMapIndex(n.children[i+1], inlineMap, name, value) + } else if d.strict { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.line+1, name.String(), out.Type())) + } + } + return true +} + +func failWantMap() { + failf("map merge requires map or sequence of maps as the value") +} + +func (d *decoder) merge(n *node, out reflect.Value) { + switch n.kind { + case mappingNode: + d.unmarshal(n, out) + case aliasNode: + an, ok := d.doc.anchors[n.value] + if ok && an.kind != mappingNode { + failWantMap() + } + d.unmarshal(n, out) + case sequenceNode: + // Step backwards as earlier nodes take precedence. + for i := len(n.children) - 1; i >= 0; i-- { + ni := n.children[i] + if ni.kind == aliasNode { + an, ok := d.doc.anchors[ni.value] + if ok && an.kind != mappingNode { + failWantMap() + } + } else if ni.kind != mappingNode { + failWantMap() + } + d.unmarshal(ni, out) + } + default: + failWantMap() + } +} + +func isMerge(n *node) bool { + return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == yaml_MERGE_TAG) +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go new file mode 100644 index 00000000000..a1c2cc52627 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go @@ -0,0 +1,1685 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Flush the buffer if needed. +func flush(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) { + return yaml_emitter_flush(emitter) + } + return true +} + +// Put a character to the output buffer. +func put(emitter *yaml_emitter_t, value byte) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + emitter.buffer[emitter.buffer_pos] = value + emitter.buffer_pos++ + emitter.column++ + return true +} + +// Put a line break to the output buffer. +func put_break(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + switch emitter.line_break { + case yaml_CR_BREAK: + emitter.buffer[emitter.buffer_pos] = '\r' + emitter.buffer_pos += 1 + case yaml_LN_BREAK: + emitter.buffer[emitter.buffer_pos] = '\n' + emitter.buffer_pos += 1 + case yaml_CRLN_BREAK: + emitter.buffer[emitter.buffer_pos+0] = '\r' + emitter.buffer[emitter.buffer_pos+1] = '\n' + emitter.buffer_pos += 2 + default: + panic("unknown line break setting") + } + emitter.column = 0 + emitter.line++ + return true +} + +// Copy a character from a string into buffer. +func write(emitter *yaml_emitter_t, s []byte, i *int) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + p := emitter.buffer_pos + w := width(s[*i]) + switch w { + case 4: + emitter.buffer[p+3] = s[*i+3] + fallthrough + case 3: + emitter.buffer[p+2] = s[*i+2] + fallthrough + case 2: + emitter.buffer[p+1] = s[*i+1] + fallthrough + case 1: + emitter.buffer[p+0] = s[*i+0] + default: + panic("unknown character width") + } + emitter.column++ + emitter.buffer_pos += w + *i += w + return true +} + +// Write a whole string into buffer. +func write_all(emitter *yaml_emitter_t, s []byte) bool { + for i := 0; i < len(s); { + if !write(emitter, s, &i) { + return false + } + } + return true +} + +// Copy a line break character from a string into buffer. +func write_break(emitter *yaml_emitter_t, s []byte, i *int) bool { + if s[*i] == '\n' { + if !put_break(emitter) { + return false + } + *i++ + } else { + if !write(emitter, s, i) { + return false + } + emitter.column = 0 + emitter.line++ + } + return true +} + +// Set an emitter error and return false. +func yaml_emitter_set_emitter_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_EMITTER_ERROR + emitter.problem = problem + return false +} + +// Emit an event. +func yaml_emitter_emit(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.events = append(emitter.events, *event) + for !yaml_emitter_need_more_events(emitter) { + event := &emitter.events[emitter.events_head] + if !yaml_emitter_analyze_event(emitter, event) { + return false + } + if !yaml_emitter_state_machine(emitter, event) { + return false + } + yaml_event_delete(event) + emitter.events_head++ + } + return true +} + +// Check if we need to accumulate more events before emitting. +// +// We accumulate extra +// - 1 event for DOCUMENT-START +// - 2 events for SEQUENCE-START +// - 3 events for MAPPING-START +// +func yaml_emitter_need_more_events(emitter *yaml_emitter_t) bool { + if emitter.events_head == len(emitter.events) { + return true + } + var accumulate int + switch emitter.events[emitter.events_head].typ { + case yaml_DOCUMENT_START_EVENT: + accumulate = 1 + break + case yaml_SEQUENCE_START_EVENT: + accumulate = 2 + break + case yaml_MAPPING_START_EVENT: + accumulate = 3 + break + default: + return false + } + if len(emitter.events)-emitter.events_head > accumulate { + return false + } + var level int + for i := emitter.events_head; i < len(emitter.events); i++ { + switch emitter.events[i].typ { + case yaml_STREAM_START_EVENT, yaml_DOCUMENT_START_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT: + level++ + case yaml_STREAM_END_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_END_EVENT, yaml_MAPPING_END_EVENT: + level-- + } + if level == 0 { + return false + } + } + return true +} + +// Append a directive to the directives stack. +func yaml_emitter_append_tag_directive(emitter *yaml_emitter_t, value *yaml_tag_directive_t, allow_duplicates bool) bool { + for i := 0; i < len(emitter.tag_directives); i++ { + if bytes.Equal(value.handle, emitter.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_emitter_set_emitter_error(emitter, "duplicate %TAG directive") + } + } + + // [Go] Do we actually need to copy this given garbage collection + // and the lack of deallocating destructors? + tag_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(tag_copy.handle, value.handle) + copy(tag_copy.prefix, value.prefix) + emitter.tag_directives = append(emitter.tag_directives, tag_copy) + return true +} + +// Increase the indentation level. +func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool) bool { + emitter.indents = append(emitter.indents, emitter.indent) + if emitter.indent < 0 { + if flow { + emitter.indent = emitter.best_indent + } else { + emitter.indent = 0 + } + } else if !indentless { + emitter.indent += emitter.best_indent + } + return true +} + +// State dispatcher. +func yaml_emitter_state_machine(emitter *yaml_emitter_t, event *yaml_event_t) bool { + switch emitter.state { + default: + case yaml_EMIT_STREAM_START_STATE: + return yaml_emitter_emit_stream_start(emitter, event) + + case yaml_EMIT_FIRST_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, true) + + case yaml_EMIT_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, false) + + case yaml_EMIT_DOCUMENT_CONTENT_STATE: + return yaml_emitter_emit_document_content(emitter, event) + + case yaml_EMIT_DOCUMENT_END_STATE: + return yaml_emitter_emit_document_end(emitter, event) + + case yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, true) + + case yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, false) + + case yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, true) + + case yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, false) + + case yaml_EMIT_END_STATE: + return yaml_emitter_set_emitter_error(emitter, "expected nothing after STREAM-END") + } + panic("invalid emitter state") +} + +// Expect STREAM-START. +func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_STREAM_START_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected STREAM-START") + } + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = event.encoding + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = yaml_UTF8_ENCODING + } + } + if emitter.best_indent < 2 || emitter.best_indent > 9 { + emitter.best_indent = 2 + } + if emitter.best_width >= 0 && emitter.best_width <= emitter.best_indent*2 { + emitter.best_width = 80 + } + if emitter.best_width < 0 { + emitter.best_width = 1<<31 - 1 + } + if emitter.line_break == yaml_ANY_BREAK { + emitter.line_break = yaml_LN_BREAK + } + + emitter.indent = -1 + emitter.line = 0 + emitter.column = 0 + emitter.whitespace = true + emitter.indention = true + + if emitter.encoding != yaml_UTF8_ENCODING { + if !yaml_emitter_write_bom(emitter) { + return false + } + } + emitter.state = yaml_EMIT_FIRST_DOCUMENT_START_STATE + return true +} + +// Expect DOCUMENT-START or STREAM-END. +func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + + if event.typ == yaml_DOCUMENT_START_EVENT { + + if event.version_directive != nil { + if !yaml_emitter_analyze_version_directive(emitter, event.version_directive) { + return false + } + } + + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_analyze_tag_directive(emitter, tag_directive) { + return false + } + if !yaml_emitter_append_tag_directive(emitter, tag_directive, false) { + return false + } + } + + for i := 0; i < len(default_tag_directives); i++ { + tag_directive := &default_tag_directives[i] + if !yaml_emitter_append_tag_directive(emitter, tag_directive, true) { + return false + } + } + + implicit := event.implicit + if !first || emitter.canonical { + implicit = false + } + + if emitter.open_ended && (event.version_directive != nil || len(event.tag_directives) > 0) { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if event.version_directive != nil { + implicit = false + if !yaml_emitter_write_indicator(emitter, []byte("%YAML"), true, false, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("1.1"), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if len(event.tag_directives) > 0 { + implicit = false + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_write_indicator(emitter, []byte("%TAG"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_handle(emitter, tag_directive.handle) { + return false + } + if !yaml_emitter_write_tag_content(emitter, tag_directive.prefix, true) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if yaml_emitter_check_empty_document(emitter) { + implicit = false + } + if !implicit { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("---"), true, false, false) { + return false + } + if emitter.canonical { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + emitter.state = yaml_EMIT_DOCUMENT_CONTENT_STATE + return true + } + + if event.typ == yaml_STREAM_END_EVENT { + if emitter.open_ended { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_END_STATE + return true + } + + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-START or STREAM-END") +} + +// Expect the root node. +func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.states = append(emitter.states, yaml_EMIT_DOCUMENT_END_STATE) + return yaml_emitter_emit_node(emitter, event, true, false, false, false) +} + +// Expect DOCUMENT-END. +func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_DOCUMENT_END_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-END") + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !event.implicit { + // [Go] Allocate the slice elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_DOCUMENT_START_STATE + emitter.tag_directives = emitter.tag_directives[:0] + return true +} + +// Expect a flow item node. +func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a flow key node. +func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_MAPPING_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if !emitter.canonical && yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, false) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a flow value node. +func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, false) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block item node. +func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, emitter.mapping_context && !emitter.indention) { + return false + } + } + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'-'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a block key node. +func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, false) { + return false + } + } + if event.typ == yaml_MAPPING_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block value node. +func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, true) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a node. +func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, + root bool, sequence bool, mapping bool, simple_key bool) bool { + + emitter.root_context = root + emitter.sequence_context = sequence + emitter.mapping_context = mapping + emitter.simple_key_context = simple_key + + switch event.typ { + case yaml_ALIAS_EVENT: + return yaml_emitter_emit_alias(emitter, event) + case yaml_SCALAR_EVENT: + return yaml_emitter_emit_scalar(emitter, event) + case yaml_SEQUENCE_START_EVENT: + return yaml_emitter_emit_sequence_start(emitter, event) + case yaml_MAPPING_START_EVENT: + return yaml_emitter_emit_mapping_start(emitter, event) + default: + return yaml_emitter_set_emitter_error(emitter, + fmt.Sprintf("expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS, but got %v", event.typ)) + } +} + +// Expect ALIAS. +func yaml_emitter_emit_alias(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SCALAR. +func yaml_emitter_emit_scalar(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_select_scalar_style(emitter, event) { + return false + } + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + if !yaml_emitter_process_scalar(emitter) { + return false + } + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SEQUENCE-START. +func yaml_emitter_emit_sequence_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || + yaml_emitter_check_empty_sequence(emitter) { + emitter.state = yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE + } + return true +} + +// Expect MAPPING-START. +func yaml_emitter_emit_mapping_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.mapping_style() == yaml_FLOW_MAPPING_STYLE || + yaml_emitter_check_empty_mapping(emitter) { + emitter.state = yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE + } + return true +} + +// Check if the document content is an empty scalar. +func yaml_emitter_check_empty_document(emitter *yaml_emitter_t) bool { + return false // [Go] Huh? +} + +// Check if the next events represent an empty sequence. +func yaml_emitter_check_empty_sequence(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_SEQUENCE_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_SEQUENCE_END_EVENT +} + +// Check if the next events represent an empty mapping. +func yaml_emitter_check_empty_mapping(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_MAPPING_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_MAPPING_END_EVENT +} + +// Check if the next node can be expressed as a simple key. +func yaml_emitter_check_simple_key(emitter *yaml_emitter_t) bool { + length := 0 + switch emitter.events[emitter.events_head].typ { + case yaml_ALIAS_EVENT: + length += len(emitter.anchor_data.anchor) + case yaml_SCALAR_EVENT: + if emitter.scalar_data.multiline { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + + len(emitter.scalar_data.value) + case yaml_SEQUENCE_START_EVENT: + if !yaml_emitter_check_empty_sequence(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + case yaml_MAPPING_START_EVENT: + if !yaml_emitter_check_empty_mapping(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + default: + return false + } + return length <= 128 +} + +// Determine an acceptable scalar style. +func yaml_emitter_select_scalar_style(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + no_tag := len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 + if no_tag && !event.implicit && !event.quoted_implicit { + return yaml_emitter_set_emitter_error(emitter, "neither tag nor implicit flags are specified") + } + + style := event.scalar_style() + if style == yaml_ANY_SCALAR_STYLE { + style = yaml_PLAIN_SCALAR_STYLE + } + if emitter.canonical { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + if emitter.simple_key_context && emitter.scalar_data.multiline { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + if style == yaml_PLAIN_SCALAR_STYLE { + if emitter.flow_level > 0 && !emitter.scalar_data.flow_plain_allowed || + emitter.flow_level == 0 && !emitter.scalar_data.block_plain_allowed { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if len(emitter.scalar_data.value) == 0 && (emitter.flow_level > 0 || emitter.simple_key_context) { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if no_tag && !event.implicit { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_SINGLE_QUOTED_SCALAR_STYLE { + if !emitter.scalar_data.single_quoted_allowed { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_LITERAL_SCALAR_STYLE || style == yaml_FOLDED_SCALAR_STYLE { + if !emitter.scalar_data.block_allowed || emitter.flow_level > 0 || emitter.simple_key_context { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + + if no_tag && !event.quoted_implicit && style != yaml_PLAIN_SCALAR_STYLE { + emitter.tag_data.handle = []byte{'!'} + } + emitter.scalar_data.style = style + return true +} + +// Write an anchor. +func yaml_emitter_process_anchor(emitter *yaml_emitter_t) bool { + if emitter.anchor_data.anchor == nil { + return true + } + c := []byte{'&'} + if emitter.anchor_data.alias { + c[0] = '*' + } + if !yaml_emitter_write_indicator(emitter, c, true, false, false) { + return false + } + return yaml_emitter_write_anchor(emitter, emitter.anchor_data.anchor) +} + +// Write a tag. +func yaml_emitter_process_tag(emitter *yaml_emitter_t) bool { + if len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 { + return true + } + if len(emitter.tag_data.handle) > 0 { + if !yaml_emitter_write_tag_handle(emitter, emitter.tag_data.handle) { + return false + } + if len(emitter.tag_data.suffix) > 0 { + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + } + } else { + // [Go] Allocate these slices elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("!<"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, false, false, false) { + return false + } + } + return true +} + +// Write a scalar. +func yaml_emitter_process_scalar(emitter *yaml_emitter_t) bool { + switch emitter.scalar_data.style { + case yaml_PLAIN_SCALAR_STYLE: + return yaml_emitter_write_plain_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_SINGLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_single_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_DOUBLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_double_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_LITERAL_SCALAR_STYLE: + return yaml_emitter_write_literal_scalar(emitter, emitter.scalar_data.value) + + case yaml_FOLDED_SCALAR_STYLE: + return yaml_emitter_write_folded_scalar(emitter, emitter.scalar_data.value) + } + panic("unknown scalar style") +} + +// Check if a %YAML directive is valid. +func yaml_emitter_analyze_version_directive(emitter *yaml_emitter_t, version_directive *yaml_version_directive_t) bool { + if version_directive.major != 1 || version_directive.minor != 1 { + return yaml_emitter_set_emitter_error(emitter, "incompatible %YAML directive") + } + return true +} + +// Check if a %TAG directive is valid. +func yaml_emitter_analyze_tag_directive(emitter *yaml_emitter_t, tag_directive *yaml_tag_directive_t) bool { + handle := tag_directive.handle + prefix := tag_directive.prefix + if len(handle) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag handle must not be empty") + } + if handle[0] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must start with '!'") + } + if handle[len(handle)-1] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must end with '!'") + } + for i := 1; i < len(handle)-1; i += width(handle[i]) { + if !is_alpha(handle, i) { + return yaml_emitter_set_emitter_error(emitter, "tag handle must contain alphanumerical characters only") + } + } + if len(prefix) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag prefix must not be empty") + } + return true +} + +// Check if an anchor is valid. +func yaml_emitter_analyze_anchor(emitter *yaml_emitter_t, anchor []byte, alias bool) bool { + if len(anchor) == 0 { + problem := "anchor value must not be empty" + if alias { + problem = "alias value must not be empty" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + for i := 0; i < len(anchor); i += width(anchor[i]) { + if !is_alpha(anchor, i) { + problem := "anchor value must contain alphanumerical characters only" + if alias { + problem = "alias value must contain alphanumerical characters only" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + } + emitter.anchor_data.anchor = anchor + emitter.anchor_data.alias = alias + return true +} + +// Check if a tag is valid. +func yaml_emitter_analyze_tag(emitter *yaml_emitter_t, tag []byte) bool { + if len(tag) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag value must not be empty") + } + for i := 0; i < len(emitter.tag_directives); i++ { + tag_directive := &emitter.tag_directives[i] + if bytes.HasPrefix(tag, tag_directive.prefix) { + emitter.tag_data.handle = tag_directive.handle + emitter.tag_data.suffix = tag[len(tag_directive.prefix):] + return true + } + } + emitter.tag_data.suffix = tag + return true +} + +// Check if a scalar is valid. +func yaml_emitter_analyze_scalar(emitter *yaml_emitter_t, value []byte) bool { + var ( + block_indicators = false + flow_indicators = false + line_breaks = false + special_characters = false + + leading_space = false + leading_break = false + trailing_space = false + trailing_break = false + break_space = false + space_break = false + + preceded_by_whitespace = false + followed_by_whitespace = false + previous_space = false + previous_break = false + ) + + emitter.scalar_data.value = value + + if len(value) == 0 { + emitter.scalar_data.multiline = false + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = false + return true + } + + if len(value) >= 3 && ((value[0] == '-' && value[1] == '-' && value[2] == '-') || (value[0] == '.' && value[1] == '.' && value[2] == '.')) { + block_indicators = true + flow_indicators = true + } + + preceded_by_whitespace = true + for i, w := 0, 0; i < len(value); i += w { + w = width(value[i]) + followed_by_whitespace = i+w >= len(value) || is_blank(value, i+w) + + if i == 0 { + switch value[i] { + case '#', ',', '[', ']', '{', '}', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`': + flow_indicators = true + block_indicators = true + case '?', ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '-': + if followed_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } else { + switch value[i] { + case ',', '?', '[', ']', '{', '}': + flow_indicators = true + case ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '#': + if preceded_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } + + if !is_printable(value, i) || !is_ascii(value, i) && !emitter.unicode { + special_characters = true + } + if is_space(value, i) { + if i == 0 { + leading_space = true + } + if i+width(value[i]) == len(value) { + trailing_space = true + } + if previous_break { + break_space = true + } + previous_space = true + previous_break = false + } else if is_break(value, i) { + line_breaks = true + if i == 0 { + leading_break = true + } + if i+width(value[i]) == len(value) { + trailing_break = true + } + if previous_space { + space_break = true + } + previous_space = false + previous_break = true + } else { + previous_space = false + previous_break = false + } + + // [Go]: Why 'z'? Couldn't be the end of the string as that's the loop condition. + preceded_by_whitespace = is_blankz(value, i) + } + + emitter.scalar_data.multiline = line_breaks + emitter.scalar_data.flow_plain_allowed = true + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = true + + if leading_space || leading_break || trailing_space || trailing_break { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if trailing_space { + emitter.scalar_data.block_allowed = false + } + if break_space { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || special_characters { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + emitter.scalar_data.block_allowed = false + } + if line_breaks { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if flow_indicators { + emitter.scalar_data.flow_plain_allowed = false + } + if block_indicators { + emitter.scalar_data.block_plain_allowed = false + } + return true +} + +// Check if the event data is valid. +func yaml_emitter_analyze_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + emitter.anchor_data.anchor = nil + emitter.tag_data.handle = nil + emitter.tag_data.suffix = nil + emitter.scalar_data.value = nil + + switch event.typ { + case yaml_ALIAS_EVENT: + if !yaml_emitter_analyze_anchor(emitter, event.anchor, true) { + return false + } + + case yaml_SCALAR_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || (!event.implicit && !event.quoted_implicit)) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + if !yaml_emitter_analyze_scalar(emitter, event.value) { + return false + } + + case yaml_SEQUENCE_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + + case yaml_MAPPING_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + } + return true +} + +// Write the BOM character. +func yaml_emitter_write_bom(emitter *yaml_emitter_t) bool { + if !flush(emitter) { + return false + } + pos := emitter.buffer_pos + emitter.buffer[pos+0] = '\xEF' + emitter.buffer[pos+1] = '\xBB' + emitter.buffer[pos+2] = '\xBF' + emitter.buffer_pos += 3 + return true +} + +func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool { + indent := emitter.indent + if indent < 0 { + indent = 0 + } + if !emitter.indention || emitter.column > indent || (emitter.column == indent && !emitter.whitespace) { + if !put_break(emitter) { + return false + } + } + for emitter.column < indent { + if !put(emitter, ' ') { + return false + } + } + emitter.whitespace = true + emitter.indention = true + return true +} + +func yaml_emitter_write_indicator(emitter *yaml_emitter_t, indicator []byte, need_whitespace, is_whitespace, is_indention bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, indicator) { + return false + } + emitter.whitespace = is_whitespace + emitter.indention = (emitter.indention && is_indention) + emitter.open_ended = false + return true +} + +func yaml_emitter_write_anchor(emitter *yaml_emitter_t, value []byte) bool { + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_handle(emitter *yaml_emitter_t, value []byte) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_whitespace bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + for i := 0; i < len(value); { + var must_write bool + switch value[i] { + case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')', '[', ']': + must_write = true + default: + must_write = is_alpha(value, i) + } + if must_write { + if !write(emitter, value, &i) { + return false + } + } else { + w := width(value[i]) + for k := 0; k < w; k++ { + octet := value[i] + i++ + if !put(emitter, '%') { + return false + } + + c := octet >> 4 + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + + c = octet & 0x0f + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + } + } + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + + emitter.whitespace = false + emitter.indention = false + if emitter.root_context { + emitter.open_ended = true + } + + return true +} + +func yaml_emitter_write_single_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, true, false, false) { + return false + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if value[i] == '\'' { + if !put(emitter, '\'') { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_double_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + spaces := false + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, true, false, false) { + return false + } + + for i := 0; i < len(value); { + if !is_printable(value, i) || (!emitter.unicode && !is_ascii(value, i)) || + is_bom(value, i) || is_break(value, i) || + value[i] == '"' || value[i] == '\\' { + + octet := value[i] + + var w int + var v rune + switch { + case octet&0x80 == 0x00: + w, v = 1, rune(octet&0x7F) + case octet&0xE0 == 0xC0: + w, v = 2, rune(octet&0x1F) + case octet&0xF0 == 0xE0: + w, v = 3, rune(octet&0x0F) + case octet&0xF8 == 0xF0: + w, v = 4, rune(octet&0x07) + } + for k := 1; k < w; k++ { + octet = value[i+k] + v = (v << 6) + (rune(octet) & 0x3F) + } + i += w + + if !put(emitter, '\\') { + return false + } + + var ok bool + switch v { + case 0x00: + ok = put(emitter, '0') + case 0x07: + ok = put(emitter, 'a') + case 0x08: + ok = put(emitter, 'b') + case 0x09: + ok = put(emitter, 't') + case 0x0A: + ok = put(emitter, 'n') + case 0x0b: + ok = put(emitter, 'v') + case 0x0c: + ok = put(emitter, 'f') + case 0x0d: + ok = put(emitter, 'r') + case 0x1b: + ok = put(emitter, 'e') + case 0x22: + ok = put(emitter, '"') + case 0x5c: + ok = put(emitter, '\\') + case 0x85: + ok = put(emitter, 'N') + case 0xA0: + ok = put(emitter, '_') + case 0x2028: + ok = put(emitter, 'L') + case 0x2029: + ok = put(emitter, 'P') + default: + if v <= 0xFF { + ok = put(emitter, 'x') + w = 2 + } else if v <= 0xFFFF { + ok = put(emitter, 'u') + w = 4 + } else { + ok = put(emitter, 'U') + w = 8 + } + for k := (w - 1) * 4; ok && k >= 0; k -= 4 { + digit := byte((v >> uint(k)) & 0x0F) + if digit < 10 { + ok = put(emitter, digit+'0') + } else { + ok = put(emitter, digit+'A'-10) + } + } + } + if !ok { + return false + } + spaces = false + } else if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if is_space(value, i+1) { + if !put(emitter, '\\') { + return false + } + } + i += width(value[i]) + } else if !write(emitter, value, &i) { + return false + } + spaces = true + } else { + if !write(emitter, value, &i) { + return false + } + spaces = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_block_scalar_hints(emitter *yaml_emitter_t, value []byte) bool { + if is_space(value, 0) || is_break(value, 0) { + indent_hint := []byte{'0' + byte(emitter.best_indent)} + if !yaml_emitter_write_indicator(emitter, indent_hint, false, false, false) { + return false + } + } + + emitter.open_ended = false + + var chomp_hint [1]byte + if len(value) == 0 { + chomp_hint[0] = '-' + } else { + i := len(value) - 1 + for value[i]&0xC0 == 0x80 { + i-- + } + if !is_break(value, i) { + chomp_hint[0] = '-' + } else if i == 0 { + chomp_hint[0] = '+' + emitter.open_ended = true + } else { + i-- + for value[i]&0xC0 == 0x80 { + i-- + } + if is_break(value, i) { + chomp_hint[0] = '+' + emitter.open_ended = true + } + } + } + if chomp_hint[0] != 0 { + if !yaml_emitter_write_indicator(emitter, chomp_hint[:], false, false, false) { + return false + } + } + return true +} + +func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'|'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + breaks := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + + return true +} + +func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + + breaks := true + leading_spaces := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !breaks && !leading_spaces && value[i] == '\n' { + k := 0 + for is_break(value, k) { + k += width(value[k]) + } + if !is_blankz(value, k) { + if !put_break(emitter) { + return false + } + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + leading_spaces = is_blank(value, i) + } + if !breaks && is_space(value, i) && !is_space(value, i+1) && emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + emitter.indention = false + breaks = false + } + } + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go new file mode 100644 index 00000000000..0ee738e11b6 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go @@ -0,0 +1,390 @@ +package yaml + +import ( + "encoding" + "fmt" + "io" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// jsonNumber is the interface of the encoding/json.Number datatype. +// Repeating the interface here avoids a dependency on encoding/json, and also +// supports other libraries like jsoniter, which use a similar datatype with +// the same interface. Detecting this interface is useful when dealing with +// structures containing json.Number, which is a string under the hood. The +// encoder should prefer the use of Int64(), Float64() and string(), in that +// order, when encoding this type. +type jsonNumber interface { + Float64() (float64, error) + Int64() (int64, error) + String() string +} + +type encoder struct { + emitter yaml_emitter_t + event yaml_event_t + out []byte + flow bool + // doneInit holds whether the initial stream_start_event has been + // emitted. + doneInit bool +} + +func newEncoder() *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_string(&e.emitter, &e.out) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func newEncoderWithWriter(w io.Writer) *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_writer(&e.emitter, w) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func (e *encoder) init() { + if e.doneInit { + return + } + yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING) + e.emit() + e.doneInit = true +} + +func (e *encoder) finish() { + e.emitter.open_ended = false + yaml_stream_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) destroy() { + yaml_emitter_delete(&e.emitter) +} + +func (e *encoder) emit() { + // This will internally delete the e.event value. + e.must(yaml_emitter_emit(&e.emitter, &e.event)) +} + +func (e *encoder) must(ok bool) { + if !ok { + msg := e.emitter.problem + if msg == "" { + msg = "unknown problem generating YAML content" + } + failf("%s", msg) + } +} + +func (e *encoder) marshalDoc(tag string, in reflect.Value) { + e.init() + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.emit() + e.marshal(tag, in) + yaml_document_end_event_initialize(&e.event, true) + e.emit() +} + +func (e *encoder) marshal(tag string, in reflect.Value) { + if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() { + e.nilv() + return + } + iface := in.Interface() + switch m := iface.(type) { + case jsonNumber: + integer, err := m.Int64() + if err == nil { + // In this case the json.Number is a valid int64 + in = reflect.ValueOf(integer) + break + } + float, err := m.Float64() + if err == nil { + // In this case the json.Number is a valid float64 + in = reflect.ValueOf(float) + break + } + // fallback case - no number could be obtained + in = reflect.ValueOf(m.String()) + case time.Time, *time.Time: + // Although time.Time implements TextMarshaler, + // we don't want to treat it as a string for YAML + // purposes because YAML has special support for + // timestamps. + case Marshaler: + v, err := m.MarshalYAML() + if err != nil { + fail(err) + } + if v == nil { + e.nilv() + return + } + in = reflect.ValueOf(v) + case encoding.TextMarshaler: + text, err := m.MarshalText() + if err != nil { + fail(err) + } + in = reflect.ValueOf(string(text)) + case nil: + e.nilv() + return + } + switch in.Kind() { + case reflect.Interface: + e.marshal(tag, in.Elem()) + case reflect.Map: + e.mapv(tag, in) + case reflect.Ptr: + if in.Type() == ptrTimeType { + e.timev(tag, in.Elem()) + } else { + e.marshal(tag, in.Elem()) + } + case reflect.Struct: + if in.Type() == timeType { + e.timev(tag, in) + } else { + e.structv(tag, in) + } + case reflect.Slice, reflect.Array: + if in.Type().Elem() == mapItemType { + e.itemsv(tag, in) + } else { + e.slicev(tag, in) + } + case reflect.String: + e.stringv(tag, in) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if in.Type() == durationType { + e.stringv(tag, reflect.ValueOf(iface.(time.Duration).String())) + } else { + e.intv(tag, in) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + e.uintv(tag, in) + case reflect.Float32, reflect.Float64: + e.floatv(tag, in) + case reflect.Bool: + e.boolv(tag, in) + default: + panic("cannot marshal type: " + in.Type().String()) + } +} + +func (e *encoder) mapv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + keys := keyList(in.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + e.marshal("", k) + e.marshal("", in.MapIndex(k)) + } + }) +} + +func (e *encoder) itemsv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + slice := in.Convert(reflect.TypeOf([]MapItem{})).Interface().([]MapItem) + for _, item := range slice { + e.marshal("", reflect.ValueOf(item.Key)) + e.marshal("", reflect.ValueOf(item.Value)) + } + }) +} + +func (e *encoder) structv(tag string, in reflect.Value) { + sinfo, err := getStructInfo(in.Type()) + if err != nil { + panic(err) + } + e.mappingv(tag, func() { + for _, info := range sinfo.FieldsList { + var value reflect.Value + if info.Inline == nil { + value = in.Field(info.Num) + } else { + value = in.FieldByIndex(info.Inline) + } + if info.OmitEmpty && isZero(value) { + continue + } + e.marshal("", reflect.ValueOf(info.Key)) + e.flow = info.Flow + e.marshal("", value) + } + if sinfo.InlineMap >= 0 { + m := in.Field(sinfo.InlineMap) + if m.Len() > 0 { + e.flow = false + keys := keyList(m.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + if _, found := sinfo.FieldsMap[k.String()]; found { + panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", k.String())) + } + e.marshal("", k) + e.flow = false + e.marshal("", m.MapIndex(k)) + } + } + } + }) +} + +func (e *encoder) mappingv(tag string, f func()) { + implicit := tag == "" + style := yaml_BLOCK_MAPPING_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, nil, []byte(tag), implicit, style) + e.emit() + f() + yaml_mapping_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) slicev(tag string, in reflect.Value) { + implicit := tag == "" + style := yaml_BLOCK_SEQUENCE_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, nil, []byte(tag), implicit, style)) + e.emit() + n := in.Len() + for i := 0; i < n; i++ { + e.marshal("", in.Index(i)) + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.emit() +} + +// isBase60 returns whether s is in base 60 notation as defined in YAML 1.1. +// +// The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported +// in YAML 1.2 and by this package, but these should be marshalled quoted for +// the time being for compatibility with other parsers. +func isBase60Float(s string) (result bool) { + // Fast path. + if s == "" { + return false + } + c := s[0] + if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 { + return false + } + // Do the full match. + return base60float.MatchString(s) +} + +// From http://yaml.org/type/float.html, except the regular expression there +// is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. +var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) + +func (e *encoder) stringv(tag string, in reflect.Value) { + var style yaml_scalar_style_t + s := in.String() + canUsePlain := true + switch { + case !utf8.ValidString(s): + if tag == yaml_BINARY_TAG { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = yaml_BINARY_TAG + s = encodeBase64(s) + case tag == "": + // Check to see if it would resolve to a specific + // tag when encoded unquoted. If it doesn't, + // there's no need to quote it. + rtag, _ := resolve("", s) + canUsePlain = rtag == yaml_STR_TAG && !isBase60Float(s) + } + // Note: it's possible for user code to emit invalid YAML + // if they explicitly specify a tag and a string containing + // text that's incompatible with that tag. + switch { + case strings.Contains(s, "\n"): + style = yaml_LITERAL_SCALAR_STYLE + case canUsePlain: + style = yaml_PLAIN_SCALAR_STYLE + default: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + e.emitScalar(s, "", tag, style) +} + +func (e *encoder) boolv(tag string, in reflect.Value) { + var s string + if in.Bool() { + s = "true" + } else { + s = "false" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) intv(tag string, in reflect.Value) { + s := strconv.FormatInt(in.Int(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) uintv(tag string, in reflect.Value) { + s := strconv.FormatUint(in.Uint(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) timev(tag string, in reflect.Value) { + t := in.Interface().(time.Time) + s := t.Format(time.RFC3339Nano) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) floatv(tag string, in reflect.Value) { + // Issue #352: When formatting, use the precision of the underlying value + precision := 64 + if in.Kind() == reflect.Float32 { + precision = 32 + } + + s := strconv.FormatFloat(in.Float(), 'g', -1, precision) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) nilv() { + e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t) { + implicit := tag == "" + e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style)) + e.emit() +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go new file mode 100644 index 00000000000..81d05dfe573 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go @@ -0,0 +1,1095 @@ +package yaml + +import ( + "bytes" +) + +// The parser implements the following grammar: +// +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// implicit_document ::= block_node DOCUMENT-END* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// block_node_or_indentless_sequence ::= +// ALIAS +// | properties (block_content | indentless_block_sequence)? +// | block_content +// | indentless_block_sequence +// block_node ::= ALIAS +// | properties block_content? +// | block_content +// flow_node ::= ALIAS +// | properties flow_content? +// | flow_content +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// block_content ::= block_collection | flow_collection | SCALAR +// flow_content ::= flow_collection | SCALAR +// block_collection ::= block_sequence | block_mapping +// flow_collection ::= flow_sequence | flow_mapping +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// block_mapping ::= BLOCK-MAPPING_START +// ((KEY block_node_or_indentless_sequence?)? +// (VALUE block_node_or_indentless_sequence?)?)* +// BLOCK-END +// flow_sequence ::= FLOW-SEQUENCE-START +// (flow_sequence_entry FLOW-ENTRY)* +// flow_sequence_entry? +// FLOW-SEQUENCE-END +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// flow_mapping ::= FLOW-MAPPING-START +// (flow_mapping_entry FLOW-ENTRY)* +// flow_mapping_entry? +// FLOW-MAPPING-END +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +// Peek the next token in the token queue. +func peek_token(parser *yaml_parser_t) *yaml_token_t { + if parser.token_available || yaml_parser_fetch_more_tokens(parser) { + return &parser.tokens[parser.tokens_head] + } + return nil +} + +// Remove the next token from the queue (must be called after peek_token). +func skip_token(parser *yaml_parser_t) { + parser.token_available = false + parser.tokens_parsed++ + parser.stream_end_produced = parser.tokens[parser.tokens_head].typ == yaml_STREAM_END_TOKEN + parser.tokens_head++ +} + +// Get the next event. +func yaml_parser_parse(parser *yaml_parser_t, event *yaml_event_t) bool { + // Erase the event object. + *event = yaml_event_t{} + + // No events after the end of the stream or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR || parser.state == yaml_PARSE_END_STATE { + return true + } + + // Generate the next event. + return yaml_parser_state_machine(parser, event) +} + +// Set parser error. +func yaml_parser_set_parser_error(parser *yaml_parser_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +func yaml_parser_set_parser_error_context(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +// State dispatcher. +func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool { + //trace("yaml_parser_state_machine", "state:", parser.state.String()) + + switch parser.state { + case yaml_PARSE_STREAM_START_STATE: + return yaml_parser_parse_stream_start(parser, event) + + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, true) + + case yaml_PARSE_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, false) + + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return yaml_parser_parse_document_content(parser, event) + + case yaml_PARSE_DOCUMENT_END_STATE: + return yaml_parser_parse_document_end(parser, event) + + case yaml_PARSE_BLOCK_NODE_STATE: + return yaml_parser_parse_node(parser, event, true, false) + + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return yaml_parser_parse_node(parser, event, true, true) + + case yaml_PARSE_FLOW_NODE_STATE: + return yaml_parser_parse_node(parser, event, false, false) + + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, true) + + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, false) + + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_indentless_sequence_entry(parser, event) + + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, true) + + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, false) + + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return yaml_parser_parse_block_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, true) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, false) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_key(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_end(parser, event) + + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, true) + + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, true) + + default: + panic("invalid parser state") + } +} + +// Parse the production: +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ +func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_STREAM_START_TOKEN { + return yaml_parser_set_parser_error(parser, "did not find expected ", token.start_mark) + } + parser.state = yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + encoding: token.encoding, + } + skip_token(parser) + return true +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* +func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { + + token := peek_token(parser) + if token == nil { + return false + } + + // Parse extra document end indicators. + if !implicit { + for token.typ == yaml_DOCUMENT_END_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + if implicit && token.typ != yaml_VERSION_DIRECTIVE_TOKEN && + token.typ != yaml_TAG_DIRECTIVE_TOKEN && + token.typ != yaml_DOCUMENT_START_TOKEN && + token.typ != yaml_STREAM_END_TOKEN { + // Parse an implicit document. + if !yaml_parser_process_directives(parser, nil, nil) { + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_BLOCK_NODE_STATE + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + } else if token.typ != yaml_STREAM_END_TOKEN { + // Parse an explicit document. + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + start_mark := token.start_mark + if !yaml_parser_process_directives(parser, &version_directive, &tag_directives) { + return false + } + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_DOCUMENT_START_TOKEN { + yaml_parser_set_parser_error(parser, + "did not find expected ", token.start_mark) + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_DOCUMENT_CONTENT_STATE + end_mark := token.end_mark + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: false, + } + skip_token(parser) + + } else { + // Parse the stream end. + parser.state = yaml_PARSE_END_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + } + + return true +} + +// Parse the productions: +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** +// +func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN || + token.typ == yaml_TAG_DIRECTIVE_TOKEN || + token.typ == yaml_DOCUMENT_START_TOKEN || + token.typ == yaml_DOCUMENT_END_TOKEN || + token.typ == yaml_STREAM_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + return yaml_parser_process_empty_scalar(parser, event, + token.start_mark) + } + return yaml_parser_parse_node(parser, event, true, false) +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// +func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + start_mark := token.start_mark + end_mark := token.start_mark + + implicit := true + if token.typ == yaml_DOCUMENT_END_TOKEN { + end_mark = token.end_mark + skip_token(parser) + implicit = false + } + + parser.tag_directives = parser.tag_directives[:0] + + parser.state = yaml_PARSE_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + start_mark: start_mark, + end_mark: end_mark, + implicit: implicit, + } + return true +} + +// Parse the productions: +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** +func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { + //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_ALIAS_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + anchor: token.value, + } + skip_token(parser) + return true + } + + start_mark := token.start_mark + end_mark := token.start_mark + + var tag_token bool + var tag_handle, tag_suffix, anchor []byte + var tag_mark yaml_mark_t + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + start_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } else if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + start_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + var tag []byte + if tag_token { + if len(tag_handle) == 0 { + tag = tag_suffix + tag_suffix = nil + } else { + for i := range parser.tag_directives { + if bytes.Equal(parser.tag_directives[i].handle, tag_handle) { + tag = append([]byte(nil), parser.tag_directives[i].prefix...) + tag = append(tag, tag_suffix...) + break + } + } + if len(tag) == 0 { + yaml_parser_set_parser_error_context(parser, + "while parsing a node", start_mark, + "found undefined tag handle", tag_mark) + return false + } + } + } + + implicit := len(tag) == 0 + if indentless_sequence && token.typ == yaml_BLOCK_ENTRY_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_SCALAR_TOKEN { + var plain_implicit, quoted_implicit bool + end_mark = token.end_mark + if (len(tag) == 0 && token.style == yaml_PLAIN_SCALAR_STYLE) || (len(tag) == 1 && tag[0] == '!') { + plain_implicit = true + } else if len(tag) == 0 { + quoted_implicit = true + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + value: token.value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(token.style), + } + skip_token(parser) + return true + } + if token.typ == yaml_FLOW_SEQUENCE_START_TOKEN { + // [Go] Some of the events below can be merged as they differ only on style. + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_FLOW_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), + } + return true + } + if len(anchor) > 0 || len(tag) > 0 { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + quoted_implicit: false, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true + } + + context := "while parsing a flow node" + if block { + context = "while parsing a block node" + } + yaml_parser_set_parser_error_context(parser, context, start_mark, + "did not find expected node content", token.start_mark) + return false +} + +// Parse the productions: +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* +// +func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } else { + parser.state = yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } + if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block collection", context_mark, + "did not find expected '-' indicator", token.start_mark) +} + +// Parse the productions: +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * +func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && + token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be token.end_mark? + } + return true +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* +// +// BLOCK-END +// ********* +// +func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_KEY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } else { + parser.state = yaml_PARSE_BLOCK_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } else if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block mapping", context_mark, + "did not find expected key", token.start_mark) +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? +// +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END +// +// +func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow sequence", context_mark, + "did not find expected ',' or ']'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + implicit: true, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + skip_token(parser) + return true + } else if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true +} + +// +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + mark := token.end_mark + skip_token(parser) + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be end_mark? + } + return true +} + +// Parse the productions: +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * +// +func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow mapping", context_mark, + "did not find expected ',' or '}'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } else { + parser.state = yaml_PARSE_FLOW_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + } else if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true +} + +// Parse the productions: +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * +// +func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { + token := peek_token(parser) + if token == nil { + return false + } + if empty { + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Generate an empty scalar event. +func yaml_parser_process_empty_scalar(parser *yaml_parser_t, event *yaml_event_t, mark yaml_mark_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: mark, + end_mark: mark, + value: nil, // Empty + implicit: true, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true +} + +var default_tag_directives = []yaml_tag_directive_t{ + {[]byte("!"), []byte("!")}, + {[]byte("!!"), []byte("tag:yaml.org,2002:")}, +} + +// Parse directives. +func yaml_parser_process_directives(parser *yaml_parser_t, + version_directive_ref **yaml_version_directive_t, + tag_directives_ref *[]yaml_tag_directive_t) bool { + + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + + token := peek_token(parser) + if token == nil { + return false + } + + for token.typ == yaml_VERSION_DIRECTIVE_TOKEN || token.typ == yaml_TAG_DIRECTIVE_TOKEN { + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN { + if version_directive != nil { + yaml_parser_set_parser_error(parser, + "found duplicate %YAML directive", token.start_mark) + return false + } + if token.major != 1 || token.minor != 1 { + yaml_parser_set_parser_error(parser, + "found incompatible YAML document", token.start_mark) + return false + } + version_directive = &yaml_version_directive_t{ + major: token.major, + minor: token.minor, + } + } else if token.typ == yaml_TAG_DIRECTIVE_TOKEN { + value := yaml_tag_directive_t{ + handle: token.value, + prefix: token.prefix, + } + if !yaml_parser_append_tag_directive(parser, value, false, token.start_mark) { + return false + } + tag_directives = append(tag_directives, value) + } + + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + + for i := range default_tag_directives { + if !yaml_parser_append_tag_directive(parser, default_tag_directives[i], true, token.start_mark) { + return false + } + } + + if version_directive_ref != nil { + *version_directive_ref = version_directive + } + if tag_directives_ref != nil { + *tag_directives_ref = tag_directives + } + return true +} + +// Append a tag directive to the directives stack. +func yaml_parser_append_tag_directive(parser *yaml_parser_t, value yaml_tag_directive_t, allow_duplicates bool, mark yaml_mark_t) bool { + for i := range parser.tag_directives { + if bytes.Equal(value.handle, parser.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_parser_set_parser_error(parser, "found duplicate %TAG directive", mark) + } + } + + // [Go] I suspect the copy is unnecessary. This was likely done + // because there was no way to track ownership of the data. + value_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(value_copy.handle, value.handle) + copy(value_copy.prefix, value.prefix) + parser.tag_directives = append(parser.tag_directives, value_copy) + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go new file mode 100644 index 00000000000..7c1f5fac3db --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go @@ -0,0 +1,412 @@ +package yaml + +import ( + "io" +) + +// Set the reader error and return 0. +func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool { + parser.error = yaml_READER_ERROR + parser.problem = problem + parser.problem_offset = offset + parser.problem_value = value + return false +} + +// Byte order marks. +const ( + bom_UTF8 = "\xef\xbb\xbf" + bom_UTF16LE = "\xff\xfe" + bom_UTF16BE = "\xfe\xff" +) + +// Determine the input stream encoding by checking the BOM symbol. If no BOM is +// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure. +func yaml_parser_determine_encoding(parser *yaml_parser_t) bool { + // Ensure that we had enough bytes in the raw buffer. + for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 { + if !yaml_parser_update_raw_buffer(parser) { + return false + } + } + + // Determine the encoding. + buf := parser.raw_buffer + pos := parser.raw_buffer_pos + avail := len(buf) - pos + if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] { + parser.encoding = yaml_UTF16LE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] { + parser.encoding = yaml_UTF16BE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] { + parser.encoding = yaml_UTF8_ENCODING + parser.raw_buffer_pos += 3 + parser.offset += 3 + } else { + parser.encoding = yaml_UTF8_ENCODING + } + return true +} + +// Update the raw buffer. +func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool { + size_read := 0 + + // Return if the raw buffer is full. + if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) { + return true + } + + // Return on EOF. + if parser.eof { + return true + } + + // Move the remaining bytes in the raw buffer to the beginning. + if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) { + copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:]) + } + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos] + parser.raw_buffer_pos = 0 + + // Call the read handler to fill the buffer. + size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)]) + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read] + if err == io.EOF { + parser.eof = true + } else if err != nil { + return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1) + } + return true +} + +// Ensure that the buffer contains at least `length` characters. +// Return true on success, false on failure. +// +// The length is supposed to be significantly less that the buffer size. +func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool { + if parser.read_handler == nil { + panic("read handler must be set") + } + + // [Go] This function was changed to guarantee the requested length size at EOF. + // The fact we need to do this is pretty awful, but the description above implies + // for that to be the case, and there are tests + + // If the EOF flag is set and the raw buffer is empty, do nothing. + if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) { + // [Go] ACTUALLY! Read the documentation of this function above. + // This is just broken. To return true, we need to have the + // given length in the buffer. Not doing that means every single + // check that calls this function to make sure the buffer has a + // given length is Go) panicking; or C) accessing invalid memory. + //return true + } + + // Return if the buffer contains enough characters. + if parser.unread >= length { + return true + } + + // Determine the input encoding if it is not known yet. + if parser.encoding == yaml_ANY_ENCODING { + if !yaml_parser_determine_encoding(parser) { + return false + } + } + + // Move the unread characters to the beginning of the buffer. + buffer_len := len(parser.buffer) + if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len { + copy(parser.buffer, parser.buffer[parser.buffer_pos:]) + buffer_len -= parser.buffer_pos + parser.buffer_pos = 0 + } else if parser.buffer_pos == buffer_len { + buffer_len = 0 + parser.buffer_pos = 0 + } + + // Open the whole buffer for writing, and cut it before returning. + parser.buffer = parser.buffer[:cap(parser.buffer)] + + // Fill the buffer until it has enough characters. + first := true + for parser.unread < length { + + // Fill the raw buffer if necessary. + if !first || parser.raw_buffer_pos == len(parser.raw_buffer) { + if !yaml_parser_update_raw_buffer(parser) { + parser.buffer = parser.buffer[:buffer_len] + return false + } + } + first = false + + // Decode the raw buffer. + inner: + for parser.raw_buffer_pos != len(parser.raw_buffer) { + var value rune + var width int + + raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos + + // Decode the next character. + switch parser.encoding { + case yaml_UTF8_ENCODING: + // Decode a UTF-8 character. Check RFC 3629 + // (http://www.ietf.org/rfc/rfc3629.txt) for more details. + // + // The following table (taken from the RFC) is used for + // decoding. + // + // Char. number range | UTF-8 octet sequence + // (hexadecimal) | (binary) + // --------------------+------------------------------------ + // 0000 0000-0000 007F | 0xxxxxxx + // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx + // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx + // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Additionally, the characters in the range 0xD800-0xDFFF + // are prohibited as they are reserved for use with UTF-16 + // surrogate pairs. + + // Determine the length of the UTF-8 sequence. + octet := parser.raw_buffer[parser.raw_buffer_pos] + switch { + case octet&0x80 == 0x00: + width = 1 + case octet&0xE0 == 0xC0: + width = 2 + case octet&0xF0 == 0xE0: + width = 3 + case octet&0xF8 == 0xF0: + width = 4 + default: + // The leading octet is invalid. + return yaml_parser_set_reader_error(parser, + "invalid leading UTF-8 octet", + parser.offset, int(octet)) + } + + // Check if the raw buffer contains an incomplete character. + if width > raw_unread { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-8 octet sequence", + parser.offset, -1) + } + break inner + } + + // Decode the leading octet. + switch { + case octet&0x80 == 0x00: + value = rune(octet & 0x7F) + case octet&0xE0 == 0xC0: + value = rune(octet & 0x1F) + case octet&0xF0 == 0xE0: + value = rune(octet & 0x0F) + case octet&0xF8 == 0xF0: + value = rune(octet & 0x07) + default: + value = 0 + } + + // Check and decode the trailing octets. + for k := 1; k < width; k++ { + octet = parser.raw_buffer[parser.raw_buffer_pos+k] + + // Check if the octet is valid. + if (octet & 0xC0) != 0x80 { + return yaml_parser_set_reader_error(parser, + "invalid trailing UTF-8 octet", + parser.offset+k, int(octet)) + } + + // Decode the octet. + value = (value << 6) + rune(octet&0x3F) + } + + // Check the length of the sequence against the value. + switch { + case width == 1: + case width == 2 && value >= 0x80: + case width == 3 && value >= 0x800: + case width == 4 && value >= 0x10000: + default: + return yaml_parser_set_reader_error(parser, + "invalid length of a UTF-8 sequence", + parser.offset, -1) + } + + // Check the range of the value. + if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF { + return yaml_parser_set_reader_error(parser, + "invalid Unicode character", + parser.offset, int(value)) + } + + case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING: + var low, high int + if parser.encoding == yaml_UTF16LE_ENCODING { + low, high = 0, 1 + } else { + low, high = 1, 0 + } + + // The UTF-16 encoding is not as simple as one might + // naively think. Check RFC 2781 + // (http://www.ietf.org/rfc/rfc2781.txt). + // + // Normally, two subsequent bytes describe a Unicode + // character. However a special technique (called a + // surrogate pair) is used for specifying character + // values larger than 0xFFFF. + // + // A surrogate pair consists of two pseudo-characters: + // high surrogate area (0xD800-0xDBFF) + // low surrogate area (0xDC00-0xDFFF) + // + // The following formulas are used for decoding + // and encoding characters using surrogate pairs: + // + // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF) + // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF) + // W1 = 110110yyyyyyyyyy + // W2 = 110111xxxxxxxxxx + // + // where U is the character value, W1 is the high surrogate + // area, W2 is the low surrogate area. + + // Check for incomplete UTF-16 character. + if raw_unread < 2 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 character", + parser.offset, -1) + } + break inner + } + + // Get the character. + value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8) + + // Check for unexpected low surrogate area. + if value&0xFC00 == 0xDC00 { + return yaml_parser_set_reader_error(parser, + "unexpected low surrogate area", + parser.offset, int(value)) + } + + // Check for a high surrogate area. + if value&0xFC00 == 0xD800 { + width = 4 + + // Check for incomplete surrogate pair. + if raw_unread < 4 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 surrogate pair", + parser.offset, -1) + } + break inner + } + + // Get the next character. + value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8) + + // Check for a low surrogate area. + if value2&0xFC00 != 0xDC00 { + return yaml_parser_set_reader_error(parser, + "expected low surrogate area", + parser.offset+2, int(value2)) + } + + // Generate the value of the surrogate pair. + value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF) + } else { + width = 2 + } + + default: + panic("impossible") + } + + // Check if the character is in the allowed range: + // #x9 | #xA | #xD | [#x20-#x7E] (8 bit) + // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) + // | [#x10000-#x10FFFF] (32 bit) + switch { + case value == 0x09: + case value == 0x0A: + case value == 0x0D: + case value >= 0x20 && value <= 0x7E: + case value == 0x85: + case value >= 0xA0 && value <= 0xD7FF: + case value >= 0xE000 && value <= 0xFFFD: + case value >= 0x10000 && value <= 0x10FFFF: + default: + return yaml_parser_set_reader_error(parser, + "control characters are not allowed", + parser.offset, int(value)) + } + + // Move the raw pointers. + parser.raw_buffer_pos += width + parser.offset += width + + // Finally put the character into the buffer. + if value <= 0x7F { + // 0000 0000-0000 007F . 0xxxxxxx + parser.buffer[buffer_len+0] = byte(value) + buffer_len += 1 + } else if value <= 0x7FF { + // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6)) + parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F)) + buffer_len += 2 + } else if value <= 0xFFFF { + // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F)) + buffer_len += 3 + } else { + // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F)) + buffer_len += 4 + } + + parser.unread++ + } + + // On EOF, put NUL into the buffer and return. + if parser.eof { + parser.buffer[buffer_len] = 0 + buffer_len++ + parser.unread++ + break + } + } + // [Go] Read the documentation of this function above. To return true, + // we need to have the given length in the buffer. Not doing that means + // every single check that calls this function to make sure the buffer + // has a given length is Go) panicking; or C) accessing invalid memory. + // This happens here due to the EOF above breaking early. + for buffer_len < length { + parser.buffer[buffer_len] = 0 + buffer_len++ + } + parser.buffer = parser.buffer[:buffer_len] + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go new file mode 100644 index 00000000000..4120e0c9160 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go @@ -0,0 +1,258 @@ +package yaml + +import ( + "encoding/base64" + "math" + "regexp" + "strconv" + "strings" + "time" +) + +type resolveMapItem struct { + value interface{} + tag string +} + +var resolveTable = make([]byte, 256) +var resolveMap = make(map[string]resolveMapItem) + +func init() { + t := resolveTable + t[int('+')] = 'S' // Sign + t[int('-')] = 'S' + for _, c := range "0123456789" { + t[int(c)] = 'D' // Digit + } + for _, c := range "yYnNtTfFoO~" { + t[int(c)] = 'M' // In map + } + t[int('.')] = '.' // Float (potentially in map) + + var resolveMapList = []struct { + v interface{} + tag string + l []string + }{ + {true, yaml_BOOL_TAG, []string{"y", "Y", "yes", "Yes", "YES"}}, + {true, yaml_BOOL_TAG, []string{"true", "True", "TRUE"}}, + {true, yaml_BOOL_TAG, []string{"on", "On", "ON"}}, + {false, yaml_BOOL_TAG, []string{"n", "N", "no", "No", "NO"}}, + {false, yaml_BOOL_TAG, []string{"false", "False", "FALSE"}}, + {false, yaml_BOOL_TAG, []string{"off", "Off", "OFF"}}, + {nil, yaml_NULL_TAG, []string{"", "~", "null", "Null", "NULL"}}, + {math.NaN(), yaml_FLOAT_TAG, []string{".nan", ".NaN", ".NAN"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{".inf", ".Inf", ".INF"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{"+.inf", "+.Inf", "+.INF"}}, + {math.Inf(-1), yaml_FLOAT_TAG, []string{"-.inf", "-.Inf", "-.INF"}}, + {"<<", yaml_MERGE_TAG, []string{"<<"}}, + } + + m := resolveMap + for _, item := range resolveMapList { + for _, s := range item.l { + m[s] = resolveMapItem{item.v, item.tag} + } + } +} + +const longTagPrefix = "tag:yaml.org,2002:" + +func shortTag(tag string) string { + // TODO This can easily be made faster and produce less garbage. + if strings.HasPrefix(tag, longTagPrefix) { + return "!!" + tag[len(longTagPrefix):] + } + return tag +} + +func longTag(tag string) string { + if strings.HasPrefix(tag, "!!") { + return longTagPrefix + tag[2:] + } + return tag +} + +func resolvableTag(tag string) bool { + switch tag { + case "", yaml_STR_TAG, yaml_BOOL_TAG, yaml_INT_TAG, yaml_FLOAT_TAG, yaml_NULL_TAG, yaml_TIMESTAMP_TAG: + return true + } + return false +} + +var yamlStyleFloat = regexp.MustCompile(`^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$`) + +func resolve(tag string, in string) (rtag string, out interface{}) { + if !resolvableTag(tag) { + return tag, in + } + + defer func() { + switch tag { + case "", rtag, yaml_STR_TAG, yaml_BINARY_TAG: + return + case yaml_FLOAT_TAG: + if rtag == yaml_INT_TAG { + switch v := out.(type) { + case int64: + rtag = yaml_FLOAT_TAG + out = float64(v) + return + case int: + rtag = yaml_FLOAT_TAG + out = float64(v) + return + } + } + } + failf("cannot decode %s `%s` as a %s", shortTag(rtag), in, shortTag(tag)) + }() + + // Any data is accepted as a !!str or !!binary. + // Otherwise, the prefix is enough of a hint about what it might be. + hint := byte('N') + if in != "" { + hint = resolveTable[in[0]] + } + if hint != 0 && tag != yaml_STR_TAG && tag != yaml_BINARY_TAG { + // Handle things we can lookup in a map. + if item, ok := resolveMap[in]; ok { + return item.tag, item.value + } + + // Base 60 floats are a bad idea, were dropped in YAML 1.2, and + // are purposefully unsupported here. They're still quoted on + // the way out for compatibility with other parser, though. + + switch hint { + case 'M': + // We've already checked the map above. + + case '.': + // Not in the map, so maybe a normal float. + floatv, err := strconv.ParseFloat(in, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + + case 'D', 'S': + // Int, float, or timestamp. + // Only try values as a timestamp if the value is unquoted or there's an explicit + // !!timestamp tag. + if tag == "" || tag == yaml_TIMESTAMP_TAG { + t, ok := parseTimestamp(in) + if ok { + return yaml_TIMESTAMP_TAG, t + } + } + + plain := strings.Replace(in, "_", "", -1) + intv, err := strconv.ParseInt(plain, 0, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain, 0, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + if yamlStyleFloat.MatchString(plain) { + floatv, err := strconv.ParseFloat(plain, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + } + if strings.HasPrefix(plain, "0b") { + intv, err := strconv.ParseInt(plain[2:], 2, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 2, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + } else if strings.HasPrefix(plain, "-0b") { + intv, err := strconv.ParseInt("-" + plain[3:], 2, 64) + if err == nil { + if true || intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + } + default: + panic("resolveTable item not yet handled: " + string(rune(hint)) + " (with " + in + ")") + } + } + return yaml_STR_TAG, in +} + +// encodeBase64 encodes s as base64 that is broken up into multiple lines +// as appropriate for the resulting length. +func encodeBase64(s string) string { + const lineLen = 70 + encLen := base64.StdEncoding.EncodedLen(len(s)) + lines := encLen/lineLen + 1 + buf := make([]byte, encLen*2+lines) + in := buf[0:encLen] + out := buf[encLen:] + base64.StdEncoding.Encode(in, []byte(s)) + k := 0 + for i := 0; i < len(in); i += lineLen { + j := i + lineLen + if j > len(in) { + j = len(in) + } + k += copy(out[k:], in[i:j]) + if lines > 1 { + out[k] = '\n' + k++ + } + } + return string(out[:k]) +} + +// This is a subset of the formats allowed by the regular expression +// defined at http://yaml.org/type/timestamp.html. +var allowedTimestampFormats = []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + // Notable exception: time.Parse cannot handle: "2001-12-14 21:59:43.10 -5" + // from the set of examples. +} + +// parseTimestamp parses s as a timestamp string and +// returns the timestamp and reports whether it succeeded. +// Timestamp formats are defined at http://yaml.org/type/timestamp.html +func parseTimestamp(s string) (time.Time, bool) { + // TODO write code to check all the formats supported by + // http://yaml.org/type/timestamp.html instead of using time.Parse. + + // Quick check: all date formats start with YYYY-. + i := 0 + for ; i < len(s); i++ { + if c := s[i]; c < '0' || c > '9' { + break + } + } + if i != 4 || i == len(s) || s[i] != '-' { + return time.Time{}, false + } + for _, format := range allowedTimestampFormats { + if t, err := time.Parse(format, s); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go new file mode 100644 index 00000000000..077fd1dd2d4 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go @@ -0,0 +1,2696 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Introduction +// ************ +// +// The following notes assume that you are familiar with the YAML specification +// (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in +// some cases we are less restrictive that it requires. +// +// The process of transforming a YAML stream into a sequence of events is +// divided on two steps: Scanning and Parsing. +// +// The Scanner transforms the input stream into a sequence of tokens, while the +// parser transform the sequence of tokens produced by the Scanner into a +// sequence of parsing events. +// +// The Scanner is rather clever and complicated. The Parser, on the contrary, +// is a straightforward implementation of a recursive-descendant parser (or, +// LL(1) parser, as it is usually called). +// +// Actually there are two issues of Scanning that might be called "clever", the +// rest is quite straightforward. The issues are "block collection start" and +// "simple keys". Both issues are explained below in details. +// +// Here the Scanning step is explained and implemented. We start with the list +// of all the tokens produced by the Scanner together with short descriptions. +// +// Now, tokens: +// +// STREAM-START(encoding) # The stream start. +// STREAM-END # The stream end. +// VERSION-DIRECTIVE(major,minor) # The '%YAML' directive. +// TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive. +// DOCUMENT-START # '---' +// DOCUMENT-END # '...' +// BLOCK-SEQUENCE-START # Indentation increase denoting a block +// BLOCK-MAPPING-START # sequence or a block mapping. +// BLOCK-END # Indentation decrease. +// FLOW-SEQUENCE-START # '[' +// FLOW-SEQUENCE-END # ']' +// BLOCK-SEQUENCE-START # '{' +// BLOCK-SEQUENCE-END # '}' +// BLOCK-ENTRY # '-' +// FLOW-ENTRY # ',' +// KEY # '?' or nothing (simple keys). +// VALUE # ':' +// ALIAS(anchor) # '*anchor' +// ANCHOR(anchor) # '&anchor' +// TAG(handle,suffix) # '!handle!suffix' +// SCALAR(value,style) # A scalar. +// +// The following two tokens are "virtual" tokens denoting the beginning and the +// end of the stream: +// +// STREAM-START(encoding) +// STREAM-END +// +// We pass the information about the input stream encoding with the +// STREAM-START token. +// +// The next two tokens are responsible for tags: +// +// VERSION-DIRECTIVE(major,minor) +// TAG-DIRECTIVE(handle,prefix) +// +// Example: +// +// %YAML 1.1 +// %TAG ! !foo +// %TAG !yaml! tag:yaml.org,2002: +// --- +// +// The correspoding sequence of tokens: +// +// STREAM-START(utf-8) +// VERSION-DIRECTIVE(1,1) +// TAG-DIRECTIVE("!","!foo") +// TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:") +// DOCUMENT-START +// STREAM-END +// +// Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole +// line. +// +// The document start and end indicators are represented by: +// +// DOCUMENT-START +// DOCUMENT-END +// +// Note that if a YAML stream contains an implicit document (without '---' +// and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be +// produced. +// +// In the following examples, we present whole documents together with the +// produced tokens. +// +// 1. An implicit document: +// +// 'a scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// STREAM-END +// +// 2. An explicit document: +// +// --- +// 'a scalar' +// ... +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// SCALAR("a scalar",single-quoted) +// DOCUMENT-END +// STREAM-END +// +// 3. Several documents in a stream: +// +// 'a scalar' +// --- +// 'another scalar' +// --- +// 'yet another scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// DOCUMENT-START +// SCALAR("another scalar",single-quoted) +// DOCUMENT-START +// SCALAR("yet another scalar",single-quoted) +// STREAM-END +// +// We have already introduced the SCALAR token above. The following tokens are +// used to describe aliases, anchors, tag, and scalars: +// +// ALIAS(anchor) +// ANCHOR(anchor) +// TAG(handle,suffix) +// SCALAR(value,style) +// +// The following series of examples illustrate the usage of these tokens: +// +// 1. A recursive sequence: +// +// &A [ *A ] +// +// Tokens: +// +// STREAM-START(utf-8) +// ANCHOR("A") +// FLOW-SEQUENCE-START +// ALIAS("A") +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A tagged scalar: +// +// !!float "3.14" # A good approximation. +// +// Tokens: +// +// STREAM-START(utf-8) +// TAG("!!","float") +// SCALAR("3.14",double-quoted) +// STREAM-END +// +// 3. Various scalar styles: +// +// --- # Implicit empty plain scalars do not produce tokens. +// --- a plain scalar +// --- 'a single-quoted scalar' +// --- "a double-quoted scalar" +// --- |- +// a literal scalar +// --- >- +// a folded +// scalar +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// DOCUMENT-START +// SCALAR("a plain scalar",plain) +// DOCUMENT-START +// SCALAR("a single-quoted scalar",single-quoted) +// DOCUMENT-START +// SCALAR("a double-quoted scalar",double-quoted) +// DOCUMENT-START +// SCALAR("a literal scalar",literal) +// DOCUMENT-START +// SCALAR("a folded scalar",folded) +// STREAM-END +// +// Now it's time to review collection-related tokens. We will start with +// flow collections: +// +// FLOW-SEQUENCE-START +// FLOW-SEQUENCE-END +// FLOW-MAPPING-START +// FLOW-MAPPING-END +// FLOW-ENTRY +// KEY +// VALUE +// +// The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and +// FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}' +// correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the +// indicators '?' and ':', which are used for denoting mapping keys and values, +// are represented by the KEY and VALUE tokens. +// +// The following examples show flow collections: +// +// 1. A flow sequence: +// +// [item 1, item 2, item 3] +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-SEQUENCE-START +// SCALAR("item 1",plain) +// FLOW-ENTRY +// SCALAR("item 2",plain) +// FLOW-ENTRY +// SCALAR("item 3",plain) +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A flow mapping: +// +// { +// a simple key: a value, # Note that the KEY token is produced. +// ? a complex key: another value, +// } +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// FLOW-ENTRY +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// FLOW-ENTRY +// FLOW-MAPPING-END +// STREAM-END +// +// A simple key is a key which is not denoted by the '?' indicator. Note that +// the Scanner still produce the KEY token whenever it encounters a simple key. +// +// For scanning block collections, the following tokens are used (note that we +// repeat KEY and VALUE here): +// +// BLOCK-SEQUENCE-START +// BLOCK-MAPPING-START +// BLOCK-END +// BLOCK-ENTRY +// KEY +// VALUE +// +// The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation +// increase that precedes a block collection (cf. the INDENT token in Python). +// The token BLOCK-END denote indentation decrease that ends a block collection +// (cf. the DEDENT token in Python). However YAML has some syntax pecularities +// that makes detections of these tokens more complex. +// +// The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators +// '-', '?', and ':' correspondingly. +// +// The following examples show how the tokens BLOCK-SEQUENCE-START, +// BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner: +// +// 1. Block sequences: +// +// - item 1 +// - item 2 +// - +// - item 3.1 +// - item 3.2 +// - +// key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 3.1",plain) +// BLOCK-ENTRY +// SCALAR("item 3.2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Block mappings: +// +// a simple key: a value # The KEY token is produced here. +// ? a complex key +// : another value +// a mapping: +// key 1: value 1 +// key 2: value 2 +// a sequence: +// - item 1 +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// KEY +// SCALAR("a mapping",plain) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML does not always require to start a new block collection from a new +// line. If the current line contains only '-', '?', and ':' indicators, a new +// block collection may start at the current line. The following examples +// illustrate this case: +// +// 1. Collections in a sequence: +// +// - - item 1 +// - item 2 +// - key 1: value 1 +// key 2: value 2 +// - ? complex key +// : complex value +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("complex key") +// VALUE +// SCALAR("complex value") +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Collections in a mapping: +// +// ? a sequence +// : - item 1 +// - item 2 +// ? a mapping +// : key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// KEY +// SCALAR("a mapping",plain) +// VALUE +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML also permits non-indented sequences if they are included into a block +// mapping. In this case, the token BLOCK-SEQUENCE-START is not produced: +// +// key: +// - item 1 # BLOCK-SEQUENCE-START is NOT produced here. +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key",plain) +// VALUE +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// + +// Ensure that the buffer contains the required number of characters. +// Return true on success, false on failure (reader error or memory error). +func cache(parser *yaml_parser_t, length int) bool { + // [Go] This was inlined: !cache(A, B) -> unread < B && !update(A, B) + return parser.unread >= length || yaml_parser_update_buffer(parser, length) +} + +// Advance the buffer pointer. +func skip(parser *yaml_parser_t) { + parser.mark.index++ + parser.mark.column++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) +} + +func skip_line(parser *yaml_parser_t) { + if is_crlf(parser.buffer, parser.buffer_pos) { + parser.mark.index += 2 + parser.mark.column = 0 + parser.mark.line++ + parser.unread -= 2 + parser.buffer_pos += 2 + } else if is_break(parser.buffer, parser.buffer_pos) { + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) + } +} + +// Copy a character to a string buffer and advance pointers. +func read(parser *yaml_parser_t, s []byte) []byte { + w := width(parser.buffer[parser.buffer_pos]) + if w == 0 { + panic("invalid character sequence") + } + if len(s) == 0 { + s = make([]byte, 0, 32) + } + if w == 1 && len(s)+w <= cap(s) { + s = s[:len(s)+1] + s[len(s)-1] = parser.buffer[parser.buffer_pos] + parser.buffer_pos++ + } else { + s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...) + parser.buffer_pos += w + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + return s +} + +// Copy a line break character to a string buffer and advance pointers. +func read_line(parser *yaml_parser_t, s []byte) []byte { + buf := parser.buffer + pos := parser.buffer_pos + switch { + case buf[pos] == '\r' && buf[pos+1] == '\n': + // CR LF . LF + s = append(s, '\n') + parser.buffer_pos += 2 + parser.mark.index++ + parser.unread-- + case buf[pos] == '\r' || buf[pos] == '\n': + // CR|LF . LF + s = append(s, '\n') + parser.buffer_pos += 1 + case buf[pos] == '\xC2' && buf[pos+1] == '\x85': + // NEL . LF + s = append(s, '\n') + parser.buffer_pos += 2 + case buf[pos] == '\xE2' && buf[pos+1] == '\x80' && (buf[pos+2] == '\xA8' || buf[pos+2] == '\xA9'): + // LS|PS . LS|PS + s = append(s, buf[parser.buffer_pos:pos+3]...) + parser.buffer_pos += 3 + default: + return s + } + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + return s +} + +// Get the next token. +func yaml_parser_scan(parser *yaml_parser_t, token *yaml_token_t) bool { + // Erase the token object. + *token = yaml_token_t{} // [Go] Is this necessary? + + // No tokens after STREAM-END or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR { + return true + } + + // Ensure that the tokens queue contains enough tokens. + if !parser.token_available { + if !yaml_parser_fetch_more_tokens(parser) { + return false + } + } + + // Fetch the next token from the queue. + *token = parser.tokens[parser.tokens_head] + parser.tokens_head++ + parser.tokens_parsed++ + parser.token_available = false + + if token.typ == yaml_STREAM_END_TOKEN { + parser.stream_end_produced = true + } + return true +} + +// Set the scanner error and return false. +func yaml_parser_set_scanner_error(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string) bool { + parser.error = yaml_SCANNER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = parser.mark + return false +} + +func yaml_parser_set_scanner_tag_error(parser *yaml_parser_t, directive bool, context_mark yaml_mark_t, problem string) bool { + context := "while parsing a tag" + if directive { + context = "while parsing a %TAG directive" + } + return yaml_parser_set_scanner_error(parser, context, context_mark, problem) +} + +func trace(args ...interface{}) func() { + pargs := append([]interface{}{"+++"}, args...) + fmt.Println(pargs...) + pargs = append([]interface{}{"---"}, args...) + return func() { fmt.Println(pargs...) } +} + +// Ensure that the tokens queue contains at least one token which can be +// returned to the Parser. +func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { + // While we need more tokens to fetch, do it. + for { + // Check if we really need to fetch more tokens. + need_more_tokens := false + + if parser.tokens_head == len(parser.tokens) { + // Queue is empty. + need_more_tokens = true + } else { + // Check if any potential simple key may occupy the head position. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + if simple_key.possible && simple_key.token_number == parser.tokens_parsed { + need_more_tokens = true + break + } + } + } + + // We are finished. + if !need_more_tokens { + break + } + // Fetch the next token. + if !yaml_parser_fetch_next_token(parser) { + return false + } + } + + parser.token_available = true + return true +} + +// The dispatcher for token fetchers. +func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool { + // Ensure that the buffer is initialized. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we just started scanning. Fetch STREAM-START then. + if !parser.stream_start_produced { + return yaml_parser_fetch_stream_start(parser) + } + + // Eat whitespaces and comments until we reach the next token. + if !yaml_parser_scan_to_next_token(parser) { + return false + } + + // Remove obsolete potential simple keys. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + + // Check the indentation level against the current column. + if !yaml_parser_unroll_indent(parser, parser.mark.column) { + return false + } + + // Ensure that the buffer contains at least 4 characters. 4 is the length + // of the longest indicators ('--- ' and '... '). + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + // Is it the end of the stream? + if is_z(parser.buffer, parser.buffer_pos) { + return yaml_parser_fetch_stream_end(parser) + } + + // Is it a directive? + if parser.mark.column == 0 && parser.buffer[parser.buffer_pos] == '%' { + return yaml_parser_fetch_directive(parser) + } + + buf := parser.buffer + pos := parser.buffer_pos + + // Is it the document start indicator? + if parser.mark.column == 0 && buf[pos] == '-' && buf[pos+1] == '-' && buf[pos+2] == '-' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_START_TOKEN) + } + + // Is it the document end indicator? + if parser.mark.column == 0 && buf[pos] == '.' && buf[pos+1] == '.' && buf[pos+2] == '.' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN) + } + + // Is it the flow sequence start indicator? + if buf[pos] == '[' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN) + } + + // Is it the flow mapping start indicator? + if parser.buffer[parser.buffer_pos] == '{' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_MAPPING_START_TOKEN) + } + + // Is it the flow sequence end indicator? + if parser.buffer[parser.buffer_pos] == ']' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_SEQUENCE_END_TOKEN) + } + + // Is it the flow mapping end indicator? + if parser.buffer[parser.buffer_pos] == '}' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_MAPPING_END_TOKEN) + } + + // Is it the flow entry indicator? + if parser.buffer[parser.buffer_pos] == ',' { + return yaml_parser_fetch_flow_entry(parser) + } + + // Is it the block entry indicator? + if parser.buffer[parser.buffer_pos] == '-' && is_blankz(parser.buffer, parser.buffer_pos+1) { + return yaml_parser_fetch_block_entry(parser) + } + + // Is it the key indicator? + if parser.buffer[parser.buffer_pos] == '?' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_key(parser) + } + + // Is it the value indicator? + if parser.buffer[parser.buffer_pos] == ':' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_value(parser) + } + + // Is it an alias? + if parser.buffer[parser.buffer_pos] == '*' { + return yaml_parser_fetch_anchor(parser, yaml_ALIAS_TOKEN) + } + + // Is it an anchor? + if parser.buffer[parser.buffer_pos] == '&' { + return yaml_parser_fetch_anchor(parser, yaml_ANCHOR_TOKEN) + } + + // Is it a tag? + if parser.buffer[parser.buffer_pos] == '!' { + return yaml_parser_fetch_tag(parser) + } + + // Is it a literal scalar? + if parser.buffer[parser.buffer_pos] == '|' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, true) + } + + // Is it a folded scalar? + if parser.buffer[parser.buffer_pos] == '>' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, false) + } + + // Is it a single-quoted scalar? + if parser.buffer[parser.buffer_pos] == '\'' { + return yaml_parser_fetch_flow_scalar(parser, true) + } + + // Is it a double-quoted scalar? + if parser.buffer[parser.buffer_pos] == '"' { + return yaml_parser_fetch_flow_scalar(parser, false) + } + + // Is it a plain scalar? + // + // A plain scalar may start with any non-blank characters except + // + // '-', '?', ':', ',', '[', ']', '{', '}', + // '#', '&', '*', '!', '|', '>', '\'', '\"', + // '%', '@', '`'. + // + // In the block context (and, for the '-' indicator, in the flow context + // too), it may also start with the characters + // + // '-', '?', ':' + // + // if it is followed by a non-space character. + // + // The last rule is more restrictive than the specification requires. + // [Go] Make this logic more reasonable. + //switch parser.buffer[parser.buffer_pos] { + //case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`': + //} + if !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '-' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}' || parser.buffer[parser.buffer_pos] == '#' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '*' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '|' || + parser.buffer[parser.buffer_pos] == '>' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '"' || parser.buffer[parser.buffer_pos] == '%' || + parser.buffer[parser.buffer_pos] == '@' || parser.buffer[parser.buffer_pos] == '`') || + (parser.buffer[parser.buffer_pos] == '-' && !is_blank(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level == 0 && + (parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':') && + !is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_plain_scalar(parser) + } + + // If we don't determine the token type so far, it is an error. + return yaml_parser_set_scanner_error(parser, + "while scanning for the next token", parser.mark, + "found character that cannot start any token") +} + +// Check the list of potential simple keys and remove the positions that +// cannot contain simple keys anymore. +func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool { + // Check for a potential simple key for each flow level. + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + + // The specification requires that a simple key + // + // - is limited to a single line, + // - is shorter than 1024 characters. + if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) { + + // Check if the potential simple key to be removed is required. + if simple_key.required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false + } + } + return true +} + +// Check if a simple key may start at the current position and add it if +// needed. +func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { + // A simple key is required at the current position if the scanner is in + // the block context and the current column coincides with the indentation + // level. + + required := parser.flow_level == 0 && parser.indent == parser.mark.column + + // + // If the current position may start a simple key, save it. + // + if parser.simple_key_allowed { + simple_key := yaml_simple_key_t{ + possible: true, + required: required, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + } + simple_key.mark = parser.mark + + if !yaml_parser_remove_simple_key(parser) { + return false + } + parser.simple_keys[len(parser.simple_keys)-1] = simple_key + } + return true +} + +// Remove a potential simple key at the current flow level. +func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { + i := len(parser.simple_keys) - 1 + if parser.simple_keys[i].possible { + // If the key is required, it is an error. + if parser.simple_keys[i].required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", parser.simple_keys[i].mark, + "could not find expected ':'") + } + } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + return true +} + +// Increase the flow level and resize the simple key list if needed. +func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { + // Reset the simple key on the next level. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + // Increase the flow level. + parser.flow_level++ + return true +} + +// Decrease the flow level. +func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { + if parser.flow_level > 0 { + parser.flow_level-- + parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] + } + return true +} + +// Push the current indentation level to the stack and set the new level +// the current column is greater than the indentation level. In this case, +// append or insert the specified token into the token queue. +func yaml_parser_roll_indent(parser *yaml_parser_t, column, number int, typ yaml_token_type_t, mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + if parser.indent < column { + // Push the current indentation level to the stack and set the new + // indentation level. + parser.indents = append(parser.indents, parser.indent) + parser.indent = column + + // Create a token and insert it into the queue. + token := yaml_token_t{ + typ: typ, + start_mark: mark, + end_mark: mark, + } + if number > -1 { + number -= parser.tokens_parsed + } + yaml_insert_token(parser, number, &token) + } + return true +} + +// Pop indentation levels from the indents stack until the current level +// becomes less or equal to the column. For each indentation level, append +// the BLOCK-END token. +func yaml_parser_unroll_indent(parser *yaml_parser_t, column int) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + // Loop through the indentation levels in the stack. + for parser.indent > column { + // Create a token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + + // Pop the indentation level. + parser.indent = parser.indents[len(parser.indents)-1] + parser.indents = parser.indents[:len(parser.indents)-1] + } + return true +} + +// Initialize the scanner and produce the STREAM-START token. +func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { + + // Set the initial indentation. + parser.indent = -1 + + // Initialize the simple key stack. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + // A simple key is allowed at the beginning of the stream. + parser.simple_key_allowed = true + + // We have started. + parser.stream_start_produced = true + + // Create the STREAM-START token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_START_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + encoding: parser.encoding, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the STREAM-END token and shut down the scanner. +func yaml_parser_fetch_stream_end(parser *yaml_parser_t) bool { + + // Force new line. + if parser.mark.column != 0 { + parser.mark.column = 0 + parser.mark.line++ + } + + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the STREAM-END token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token. +func yaml_parser_fetch_directive(parser *yaml_parser_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. + token := yaml_token_t{} + if !yaml_parser_scan_directive(parser, &token) { + return false + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the DOCUMENT-START or DOCUMENT-END token. +func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Consume the token. + start_mark := parser.mark + + skip(parser) + skip(parser) + skip(parser) + + end_mark := parser.mark + + // Create the DOCUMENT-START or DOCUMENT-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token. +func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // The indicators '[' and '{' may start a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // Increase the flow level. + if !yaml_parser_increase_flow_level(parser) { + return false + } + + // A simple key may follow the indicators '[' and '{'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token. +func yaml_parser_fetch_flow_collection_end(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset any potential simple key on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Decrease the flow level. + if !yaml_parser_decrease_flow_level(parser) { + return false + } + + // No simple keys after the indicators ']' and '}'. + parser.simple_key_allowed = false + + // Consume the token. + + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-ENTRY token. +func yaml_parser_fetch_flow_entry(parser *yaml_parser_t) bool { + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after ','. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_FLOW_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the BLOCK-ENTRY token. +func yaml_parser_fetch_block_entry(parser *yaml_parser_t) bool { + // Check if the scanner is in the block context. + if parser.flow_level == 0 { + // Check if we are allowed to start a new entry. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "block sequence entries are not allowed in this context") + } + // Add the BLOCK-SEQUENCE-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_SEQUENCE_START_TOKEN, parser.mark) { + return false + } + } else { + // It is an error for the '-' indicator to occur in the flow context, + // but we let the Parser detect and report about it because the Parser + // is able to point to the context. + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '-'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the BLOCK-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the KEY token. +func yaml_parser_fetch_key(parser *yaml_parser_t) bool { + + // In the block context, additional checks are required. + if parser.flow_level == 0 { + // Check if we are allowed to start a new key (not nessesary simple). + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping keys are not allowed in this context") + } + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '?' in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the KEY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the VALUE token. +func yaml_parser_fetch_value(parser *yaml_parser_t) bool { + + simple_key := &parser.simple_keys[len(parser.simple_keys)-1] + + // Have we found a simple key? + if simple_key.possible { + // Create the KEY token and insert it into the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: simple_key.mark, + end_mark: simple_key.mark, + } + yaml_insert_token(parser, simple_key.token_number-parser.tokens_parsed, &token) + + // In the block context, we may need to add the BLOCK-MAPPING-START token. + if !yaml_parser_roll_indent(parser, simple_key.mark.column, + simple_key.token_number, + yaml_BLOCK_MAPPING_START_TOKEN, simple_key.mark) { + return false + } + + // Remove the simple key. + simple_key.possible = false + + // A simple key cannot follow another simple key. + parser.simple_key_allowed = false + + } else { + // The ':' indicator follows a complex key. + + // In the block context, extra checks are required. + if parser.flow_level == 0 { + + // Check if we are allowed to start a complex value. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping values are not allowed in this context") + } + + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Simple keys after ':' are allowed in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + } + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the VALUE token and append it to the queue. + token := yaml_token_t{ + typ: yaml_VALUE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the ALIAS or ANCHOR token. +func yaml_parser_fetch_anchor(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // An anchor or an alias could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow an anchor or an alias. + parser.simple_key_allowed = false + + // Create the ALIAS or ANCHOR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_anchor(parser, &token, typ) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the TAG token. +func yaml_parser_fetch_tag(parser *yaml_parser_t) bool { + // A tag could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a tag. + parser.simple_key_allowed = false + + // Create the TAG token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_tag(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens. +func yaml_parser_fetch_block_scalar(parser *yaml_parser_t, literal bool) bool { + // Remove any potential simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // A simple key may follow a block scalar. + parser.simple_key_allowed = true + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_block_scalar(parser, &token, literal) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens. +func yaml_parser_fetch_flow_scalar(parser *yaml_parser_t, single bool) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_flow_scalar(parser, &token, single) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,plain) token. +func yaml_parser_fetch_plain_scalar(parser *yaml_parser_t) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_plain_scalar(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Eat whitespaces and comments until the next token is found. +func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { + + // Until the next token is not found. + for { + // Allow the BOM mark to start a line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) { + skip(parser) + } + + // Eat whitespaces. + // Tabs are allowed: + // - in the flow context + // - in the block context, but not at the beginning of the line or + // after '-', '?', or ':' (complex value). + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Eat a comment until a line break. + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // If it is a line break, eat it. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + + // In the block context, a new line may start a simple key. + if parser.flow_level == 0 { + parser.simple_key_allowed = true + } + } else { + break // We have found a token. + } + } + + return true +} + +// Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool { + // Eat '%'. + start_mark := parser.mark + skip(parser) + + // Scan the directive name. + var name []byte + if !yaml_parser_scan_directive_name(parser, start_mark, &name) { + return false + } + + // Is it a YAML directive? + if bytes.Equal(name, []byte("YAML")) { + // Scan the VERSION directive value. + var major, minor int8 + if !yaml_parser_scan_version_directive_value(parser, start_mark, &major, &minor) { + return false + } + end_mark := parser.mark + + // Create a VERSION-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_VERSION_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + major: major, + minor: minor, + } + + // Is it a TAG directive? + } else if bytes.Equal(name, []byte("TAG")) { + // Scan the TAG directive value. + var handle, prefix []byte + if !yaml_parser_scan_tag_directive_value(parser, start_mark, &handle, &prefix) { + return false + } + end_mark := parser.mark + + // Create a TAG-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_TAG_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + prefix: prefix, + } + + // Unknown directive. + } else { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unknown directive name") + return false + } + + // Eat the rest of the line including any comments. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + return true +} + +// Scan the directive name. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^ +// +func yaml_parser_scan_directive_name(parser *yaml_parser_t, start_mark yaml_mark_t, name *[]byte) bool { + // Consume the directive name. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + var s []byte + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the name is empty. + if len(s) == 0 { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "could not find expected directive name") + return false + } + + // Check for an blank character after the name. + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unexpected non-alphabetical character") + return false + } + *name = s + return true +} + +// Scan the value of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^ +func yaml_parser_scan_version_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, major, minor *int8) bool { + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the major version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, major) { + return false + } + + // Eat '.'. + if parser.buffer[parser.buffer_pos] != '.' { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected digit or '.' character") + } + + skip(parser) + + // Consume the minor version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, minor) { + return false + } + return true +} + +const max_number_length = 2 + +// Scan the version number of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^ +// %YAML 1.1 # a comment \n +// ^ +func yaml_parser_scan_version_directive_number(parser *yaml_parser_t, start_mark yaml_mark_t, number *int8) bool { + + // Repeat while the next character is digit. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var value, length int8 + for is_digit(parser.buffer, parser.buffer_pos) { + // Check if the number is too long. + length++ + if length > max_number_length { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "found extremely long version number") + } + value = value*10 + int8(as_digit(parser.buffer, parser.buffer_pos)) + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the number was present. + if length == 0 { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected version number") + } + *number = value + return true +} + +// Scan the value of a TAG-DIRECTIVE token. +// +// Scope: +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_tag_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, handle, prefix *[]byte) bool { + var handle_value, prefix_value []byte + + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a handle. + if !yaml_parser_scan_tag_handle(parser, true, start_mark, &handle_value) { + return false + } + + // Expect a whitespace. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blank(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace") + return false + } + + // Eat whitespaces. + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a prefix. + if !yaml_parser_scan_tag_uri(parser, true, nil, start_mark, &prefix_value) { + return false + } + + // Expect a whitespace or line break. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace or line break") + return false + } + + *handle = handle_value + *prefix = prefix_value + return true +} + +func yaml_parser_scan_anchor(parser *yaml_parser_t, token *yaml_token_t, typ yaml_token_type_t) bool { + var s []byte + + // Eat the indicator character. + start_mark := parser.mark + skip(parser) + + // Consume the value. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + end_mark := parser.mark + + /* + * Check if length of the anchor is greater than 0 and it is followed by + * a whitespace character or one of the indicators: + * + * '?', ':', ',', ']', '}', '%', '@', '`'. + */ + + if len(s) == 0 || + !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '}' || + parser.buffer[parser.buffer_pos] == '%' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '`') { + context := "while scanning an alias" + if typ == yaml_ANCHOR_TOKEN { + context = "while scanning an anchor" + } + yaml_parser_set_scanner_error(parser, context, start_mark, + "did not find expected alphabetic or numeric character") + return false + } + + // Create a token. + *token = yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + value: s, + } + + return true +} + +/* + * Scan a TAG token. + */ + +func yaml_parser_scan_tag(parser *yaml_parser_t, token *yaml_token_t) bool { + var handle, suffix []byte + + start_mark := parser.mark + + // Check if the tag is in the canonical form. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + if parser.buffer[parser.buffer_pos+1] == '<' { + // Keep the handle as '' + + // Eat '!<' + skip(parser) + skip(parser) + + // Consume the tag value. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + + // Check for '>' and eat it. + if parser.buffer[parser.buffer_pos] != '>' { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find the expected '>'") + return false + } + + skip(parser) + } else { + // The tag has either the '!suffix' or the '!handle!suffix' form. + + // First, try to scan a handle. + if !yaml_parser_scan_tag_handle(parser, false, start_mark, &handle) { + return false + } + + // Check if it is, indeed, handle. + if handle[0] == '!' && len(handle) > 1 && handle[len(handle)-1] == '!' { + // Scan the suffix now. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + } else { + // It wasn't a handle after all. Scan the rest of the tag. + if !yaml_parser_scan_tag_uri(parser, false, handle, start_mark, &suffix) { + return false + } + + // Set the handle to '!'. + handle = []byte{'!'} + + // A special case: the '!' tag. Set the handle to '' and the + // suffix to '!'. + if len(suffix) == 0 { + handle, suffix = suffix, handle + } + } + } + + // Check the character which ends the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find expected whitespace or line break") + return false + } + + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_TAG_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + suffix: suffix, + } + return true +} + +// Scan a tag handle. +func yaml_parser_scan_tag_handle(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, handle *[]byte) bool { + // Check the initial '!' character. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] != '!' { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + + var s []byte + + // Copy the '!' character. + s = read(parser, s) + + // Copy all subsequent alphabetical and numerical characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the trailing character is '!' and copy it. + if parser.buffer[parser.buffer_pos] == '!' { + s = read(parser, s) + } else { + // It's either the '!' tag or not really a tag handle. If it's a %TAG + // directive, it's an error. If it's a tag token, it must be a part of URI. + if directive && string(s) != "!" { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + } + + *handle = s + return true +} + +// Scan a tag. +func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte, start_mark yaml_mark_t, uri *[]byte) bool { + //size_t length = head ? strlen((char *)head) : 0 + var s []byte + hasTag := len(head) > 0 + + // Copy the head if needed. + // + // Note that we don't copy the leading '!' character. + if len(head) > 1 { + s = append(s, head[1:]...) + } + + // Scan the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // The set of characters that may appear in URI is as follows: + // + // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&', + // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']', + // '%'. + // [Go] Convert this into more reasonable logic. + for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' || + parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '=' || + parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '$' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '.' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '~' || + parser.buffer[parser.buffer_pos] == '*' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '(' || parser.buffer[parser.buffer_pos] == ')' || + parser.buffer[parser.buffer_pos] == '[' || parser.buffer[parser.buffer_pos] == ']' || + parser.buffer[parser.buffer_pos] == '%' { + // Check if it is a URI-escape sequence. + if parser.buffer[parser.buffer_pos] == '%' { + if !yaml_parser_scan_uri_escapes(parser, directive, start_mark, &s) { + return false + } + } else { + s = read(parser, s) + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + hasTag = true + } + + if !hasTag { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected tag URI") + return false + } + *uri = s + return true +} + +// Decode an URI-escape sequence corresponding to a single UTF-8 character. +func yaml_parser_scan_uri_escapes(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, s *[]byte) bool { + + // Decode the required number of characters. + w := 1024 + for w > 0 { + // Check for a URI-escaped octet. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + + if !(parser.buffer[parser.buffer_pos] == '%' && + is_hex(parser.buffer, parser.buffer_pos+1) && + is_hex(parser.buffer, parser.buffer_pos+2)) { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find URI escaped octet") + } + + // Get the octet. + octet := byte((as_hex(parser.buffer, parser.buffer_pos+1) << 4) + as_hex(parser.buffer, parser.buffer_pos+2)) + + // If it is the leading octet, determine the length of the UTF-8 sequence. + if w == 1024 { + w = width(octet) + if w == 0 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect leading UTF-8 octet") + } + } else { + // Check if the trailing octet is correct. + if octet&0xC0 != 0x80 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect trailing UTF-8 octet") + } + } + + // Copy the octet and move the pointers. + *s = append(*s, octet) + skip(parser) + skip(parser) + skip(parser) + w-- + } + return true +} + +// Scan a block scalar. +func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, literal bool) bool { + // Eat the indicator '|' or '>'. + start_mark := parser.mark + skip(parser) + + // Scan the additional block scalar indicators. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check for a chomping indicator. + var chomping, increment int + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + // Set the chomping method and eat the indicator. + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + + // Check for an indentation indicator. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_digit(parser.buffer, parser.buffer_pos) { + // Check that the indentation is greater than 0. + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + + // Get the indentation level and eat the indicator. + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + } + + } else if is_digit(parser.buffer, parser.buffer_pos) { + // Do the same as above, but in the opposite order. + + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + } + } + + // Eat whitespaces and comments to the end of the line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + end_mark := parser.mark + + // Set the indentation level if it was specified. + var indent int + if increment > 0 { + if parser.indent >= 0 { + indent = parser.indent + increment + } else { + indent = increment + } + } + + // Scan the leading line breaks and determine the indentation level if needed. + var s, leading_break, trailing_breaks []byte + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + + // Scan the block scalar content. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var leading_blank, trailing_blank bool + for parser.mark.column == indent && !is_z(parser.buffer, parser.buffer_pos) { + // We are at the beginning of a non-empty line. + + // Is it a trailing whitespace? + trailing_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Check if we need to fold the leading line break. + if !literal && !leading_blank && !trailing_blank && len(leading_break) > 0 && leading_break[0] == '\n' { + // Do we need to join the lines by space? + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } + } else { + s = append(s, leading_break...) + } + leading_break = leading_break[:0] + + // Append the remaining line breaks. + s = append(s, trailing_breaks...) + trailing_breaks = trailing_breaks[:0] + + // Is it a leading whitespace? + leading_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Consume the current line. + for !is_breakz(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + leading_break = read_line(parser, leading_break) + + // Eat the following indentation spaces and line breaks. + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + } + + // Chomp the tail. + if chomping != -1 { + s = append(s, leading_break...) + } + if chomping == 1 { + s = append(s, trailing_breaks...) + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_LITERAL_SCALAR_STYLE, + } + if !literal { + token.style = yaml_FOLDED_SCALAR_STYLE + } + return true +} + +// Scan indentation spaces and line breaks for a block scalar. Determine the +// indentation level if needed. +func yaml_parser_scan_block_scalar_breaks(parser *yaml_parser_t, indent *int, breaks *[]byte, start_mark yaml_mark_t, end_mark *yaml_mark_t) bool { + *end_mark = parser.mark + + // Eat the indentation spaces and line breaks. + max_indent := 0 + for { + // Eat the indentation spaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for (*indent == 0 || parser.mark.column < *indent) && is_space(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.mark.column > max_indent { + max_indent = parser.mark.column + } + + // Check for a tab character messing the indentation. + if (*indent == 0 || parser.mark.column < *indent) && is_tab(parser.buffer, parser.buffer_pos) { + return yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found a tab character where an indentation space is expected") + } + + // Have we found a non-empty line? + if !is_break(parser.buffer, parser.buffer_pos) { + break + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + // [Go] Should really be returning breaks instead. + *breaks = read_line(parser, *breaks) + *end_mark = parser.mark + } + + // Determine the indentation level if needed. + if *indent == 0 { + *indent = max_indent + if *indent < parser.indent+1 { + *indent = parser.indent + 1 + } + if *indent < 1 { + *indent = 1 + } + } + return true +} + +// Scan a quoted scalar. +func yaml_parser_scan_flow_scalar(parser *yaml_parser_t, token *yaml_token_t, single bool) bool { + // Eat the left quote. + start_mark := parser.mark + skip(parser) + + // Consume the content of the quoted scalar. + var s, leading_break, trailing_breaks, whitespaces []byte + for { + // Check that there are no document indicators at the beginning of the line. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected document indicator") + return false + } + + // Check for EOF. + if is_z(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected end of stream") + return false + } + + // Consume non-blank characters. + leading_blanks := false + for !is_blankz(parser.buffer, parser.buffer_pos) { + if single && parser.buffer[parser.buffer_pos] == '\'' && parser.buffer[parser.buffer_pos+1] == '\'' { + // Is is an escaped single quote. + s = append(s, '\'') + skip(parser) + skip(parser) + + } else if single && parser.buffer[parser.buffer_pos] == '\'' { + // It is a right single quote. + break + } else if !single && parser.buffer[parser.buffer_pos] == '"' { + // It is a right double quote. + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' && is_break(parser.buffer, parser.buffer_pos+1) { + // It is an escaped line break. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + skip(parser) + skip_line(parser) + leading_blanks = true + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' { + // It is an escape sequence. + code_length := 0 + + // Check the escape character. + switch parser.buffer[parser.buffer_pos+1] { + case '0': + s = append(s, 0) + case 'a': + s = append(s, '\x07') + case 'b': + s = append(s, '\x08') + case 't', '\t': + s = append(s, '\x09') + case 'n': + s = append(s, '\x0A') + case 'v': + s = append(s, '\x0B') + case 'f': + s = append(s, '\x0C') + case 'r': + s = append(s, '\x0D') + case 'e': + s = append(s, '\x1B') + case ' ': + s = append(s, '\x20') + case '"': + s = append(s, '"') + case '\'': + s = append(s, '\'') + case '\\': + s = append(s, '\\') + case 'N': // NEL (#x85) + s = append(s, '\xC2') + s = append(s, '\x85') + case '_': // #xA0 + s = append(s, '\xC2') + s = append(s, '\xA0') + case 'L': // LS (#x2028) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA8') + case 'P': // PS (#x2029) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA9') + case 'x': + code_length = 2 + case 'u': + code_length = 4 + case 'U': + code_length = 8 + default: + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found unknown escape character") + return false + } + + skip(parser) + skip(parser) + + // Consume an arbitrary escape code. + if code_length > 0 { + var value int + + // Scan the character value. + if parser.unread < code_length && !yaml_parser_update_buffer(parser, code_length) { + return false + } + for k := 0; k < code_length; k++ { + if !is_hex(parser.buffer, parser.buffer_pos+k) { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "did not find expected hexdecimal number") + return false + } + value = (value << 4) + as_hex(parser.buffer, parser.buffer_pos+k) + } + + // Check the value and write the character. + if (value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found invalid Unicode character escape code") + return false + } + if value <= 0x7F { + s = append(s, byte(value)) + } else if value <= 0x7FF { + s = append(s, byte(0xC0+(value>>6))) + s = append(s, byte(0x80+(value&0x3F))) + } else if value <= 0xFFFF { + s = append(s, byte(0xE0+(value>>12))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } else { + s = append(s, byte(0xF0+(value>>18))) + s = append(s, byte(0x80+((value>>12)&0x3F))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } + + // Advance the pointer. + for k := 0; k < code_length; k++ { + skip(parser) + } + } + } else { + // It is a non-escaped non-blank character. + s = read(parser, s) + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we are at the end of the scalar. + if single { + if parser.buffer[parser.buffer_pos] == '\'' { + break + } + } else { + if parser.buffer[parser.buffer_pos] == '"' { + break + } + } + + // Consume blank characters. + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Join the whitespaces or fold line breaks. + if leading_blanks { + // Do we need to fold line breaks? + if len(leading_break) > 0 && leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Eat the right quote. + skip(parser) + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_SINGLE_QUOTED_SCALAR_STYLE, + } + if !single { + token.style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + return true +} + +// Scan a plain scalar. +func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) bool { + + var s, leading_break, trailing_breaks, whitespaces []byte + var leading_blanks bool + var indent = parser.indent + 1 + + start_mark := parser.mark + end_mark := parser.mark + + // Consume the content of the plain scalar. + for { + // Check for a document indicator. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + break + } + + // Check for a comment. + if parser.buffer[parser.buffer_pos] == '#' { + break + } + + // Consume non-blank characters. + for !is_blankz(parser.buffer, parser.buffer_pos) { + + // Check for indicators that may end a plain scalar. + if (parser.buffer[parser.buffer_pos] == ':' && is_blankz(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level > 0 && + (parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}')) { + break + } + + // Check if we need to join whitespaces and breaks. + if leading_blanks || len(whitespaces) > 0 { + if leading_blanks { + // Do we need to fold line breaks? + if leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + leading_blanks = false + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Copy the character. + s = read(parser, s) + + end_mark = parser.mark + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + // Is it the end? + if !(is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos)) { + break + } + + // Consume blank characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + + // Check for tab characters that abuse indentation. + if leading_blanks && parser.mark.column < indent && is_tab(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a plain scalar", + start_mark, "found a tab character that violates indentation") + return false + } + + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check indentation level. + if parser.flow_level == 0 && parser.mark.column < indent { + break + } + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_PLAIN_SCALAR_STYLE, + } + + // Note that we change the 'simple_key_allowed' flag. + if leading_blanks { + parser.simple_key_allowed = true + } + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go new file mode 100644 index 00000000000..4c45e660a8f --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go @@ -0,0 +1,113 @@ +package yaml + +import ( + "reflect" + "unicode" +) + +type keyList []reflect.Value + +func (l keyList) Len() int { return len(l) } +func (l keyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l keyList) Less(i, j int) bool { + a := l[i] + b := l[j] + ak := a.Kind() + bk := b.Kind() + for (ak == reflect.Interface || ak == reflect.Ptr) && !a.IsNil() { + a = a.Elem() + ak = a.Kind() + } + for (bk == reflect.Interface || bk == reflect.Ptr) && !b.IsNil() { + b = b.Elem() + bk = b.Kind() + } + af, aok := keyFloat(a) + bf, bok := keyFloat(b) + if aok && bok { + if af != bf { + return af < bf + } + if ak != bk { + return ak < bk + } + return numLess(a, b) + } + if ak != reflect.String || bk != reflect.String { + return ak < bk + } + ar, br := []rune(a.String()), []rune(b.String()) + for i := 0; i < len(ar) && i < len(br); i++ { + if ar[i] == br[i] { + continue + } + al := unicode.IsLetter(ar[i]) + bl := unicode.IsLetter(br[i]) + if al && bl { + return ar[i] < br[i] + } + if al || bl { + return bl + } + var ai, bi int + var an, bn int64 + if ar[i] == '0' || br[i] == '0' { + for j := i-1; j >= 0 && unicode.IsDigit(ar[j]); j-- { + if ar[j] != '0' { + an = 1 + bn = 1 + break + } + } + } + for ai = i; ai < len(ar) && unicode.IsDigit(ar[ai]); ai++ { + an = an*10 + int64(ar[ai]-'0') + } + for bi = i; bi < len(br) && unicode.IsDigit(br[bi]); bi++ { + bn = bn*10 + int64(br[bi]-'0') + } + if an != bn { + return an < bn + } + if ai != bi { + return ai < bi + } + return ar[i] < br[i] + } + return len(ar) < len(br) +} + +// keyFloat returns a float value for v if it is a number/bool +// and whether it is a number/bool or not. +func keyFloat(v reflect.Value) (f float64, ok bool) { + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(v.Int()), true + case reflect.Float32, reflect.Float64: + return v.Float(), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return float64(v.Uint()), true + case reflect.Bool: + if v.Bool() { + return 1, true + } + return 0, true + } + return 0, false +} + +// numLess returns whether a < b. +// a and b must necessarily have the same kind. +func numLess(a, b reflect.Value) bool { + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return a.Uint() < b.Uint() + case reflect.Bool: + return !a.Bool() && b.Bool() + } + panic("not a number") +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go new file mode 100644 index 00000000000..a2dde608cb7 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go @@ -0,0 +1,26 @@ +package yaml + +// Set the writer error and return false. +func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_WRITER_ERROR + emitter.problem = problem + return false +} + +// Flush the output buffer. +func yaml_emitter_flush(emitter *yaml_emitter_t) bool { + if emitter.write_handler == nil { + panic("write handler not set") + } + + // Check if the buffer is empty. + if emitter.buffer_pos == 0 { + return true + } + + if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil { + return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error()) + } + emitter.buffer_pos = 0 + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go new file mode 100644 index 00000000000..de85aa4cdb7 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go @@ -0,0 +1,466 @@ +// Package yaml implements YAML support for the Go language. +// +// Source code and other details for the project are available at GitHub: +// +// https://github.com/go-yaml/yaml +// +package yaml + +import ( + "errors" + "fmt" + "io" + "reflect" + "strings" + "sync" +) + +// MapSlice encodes and decodes as a YAML map. +// The order of keys is preserved when encoding and decoding. +type MapSlice []MapItem + +// MapItem is an item in a MapSlice. +type MapItem struct { + Key, Value interface{} +} + +// The Unmarshaler interface may be implemented by types to customize their +// behavior when being unmarshaled from a YAML document. The UnmarshalYAML +// method receives a function that may be called to unmarshal the original +// YAML value into a field or variable. It is safe to call the unmarshal +// function parameter more than once if necessary. +type Unmarshaler interface { + UnmarshalYAML(unmarshal func(interface{}) error) error +} + +// The Marshaler interface may be implemented by types to customize their +// behavior when being marshaled into a YAML document. The returned value +// is marshaled in place of the original value implementing Marshaler. +// +// If an error is returned by MarshalYAML, the marshaling procedure stops +// and returns with the provided error. +type Marshaler interface { + MarshalYAML() (interface{}, error) +} + +// Unmarshal decodes the first document found within the in byte slice +// and assigns decoded values into the out value. +// +// Maps and pointers (to a struct, string, int, etc) are accepted as out +// values. If an internal pointer within a struct is not initialized, +// the yaml package will initialize it if necessary for unmarshalling +// the provided data. The out parameter must not be nil. +// +// The type of the decoded values should be compatible with the respective +// values in out. If one or more values cannot be decoded due to a type +// mismatches, decoding continues partially until the end of the YAML +// content, and a *yaml.TypeError is returned with details for all +// missed values. +// +// Struct fields are only unmarshalled if they are exported (have an +// upper case first letter), and are unmarshalled using the field name +// lowercased as the default key. Custom keys may be defined via the +// "yaml" name in the field tag: the content preceding the first comma +// is used as the key, and the following comma-separated options are +// used to tweak the marshalling process (see Marshal). +// Conflicting names result in a runtime error. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// +// See the documentation of Marshal for the format of tags and a list of +// supported tag options. +// +func Unmarshal(in []byte, out interface{}) (err error) { + return unmarshal(in, out, false) +} + +// UnmarshalStrict is like Unmarshal except that any fields that are found +// in the data that do not have corresponding struct members, or mapping +// keys that are duplicates, will result in +// an error. +func UnmarshalStrict(in []byte, out interface{}) (err error) { + return unmarshal(in, out, true) +} + +// A Decorder reads and decodes YAML values from an input stream. +type Decoder struct { + strict bool + parser *parser +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read +// data from r beyond the YAML values requested. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + parser: newParserFromReader(r), + } +} + +// SetStrict sets whether strict decoding behaviour is enabled when +// decoding items in the data (see UnmarshalStrict). By default, decoding is not strict. +func (dec *Decoder) SetStrict(strict bool) { + dec.strict = strict +} + +// Decode reads the next YAML-encoded value from its input +// and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (dec *Decoder) Decode(v interface{}) (err error) { + d := newDecoder(dec.strict) + defer handleErr(&err) + node := dec.parser.parse() + if node == nil { + return io.EOF + } + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(node, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +func unmarshal(in []byte, out interface{}, strict bool) (err error) { + defer handleErr(&err) + d := newDecoder(strict) + p := newParser(in) + defer p.destroy() + node := p.parse() + if node != nil { + v := reflect.ValueOf(out) + if v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + d.unmarshal(node, v) + } + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Marshal serializes the value provided into a YAML document. The structure +// of the generated document will reflect the structure of the value itself. +// Maps and pointers (to struct, string, int, etc) are accepted as the in value. +// +// Struct fields are only marshalled if they are exported (have an upper case +// first letter), and are marshalled using the field name lowercased as the +// default key. Custom keys may be defined via the "yaml" name in the field +// tag: the content preceding the first comma is used as the key, and the +// following comma-separated options are used to tweak the marshalling process. +// Conflicting names result in a runtime error. +// +// The field tag format accepted is: +// +// `(...) yaml:"[][,[,]]" (...)` +// +// The following flags are currently supported: +// +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be included if that method returns true. +// +// flow Marshal using a flow style (useful for structs, +// sequences and maps). +// +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. +// +// In addition, if the key is "-", the field is ignored. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" +// +func Marshal(in interface{}) (out []byte, err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(in)) + e.finish() + out = e.out + return +} + +// An Encoder writes YAML values to an output stream. +type Encoder struct { + encoder *encoder +} + +// NewEncoder returns a new encoder that writes to w. +// The Encoder should be closed after use to flush all data +// to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + encoder: newEncoderWithWriter(w), + } +} + +// Encode writes the YAML encoding of v to the stream. +// If multiple items are encoded to the stream, the +// second and subsequent document will be preceded +// with a "---" document separator, but the first will not. +// +// See the documentation for Marshal for details about the conversion of Go +// values to YAML. +func (e *Encoder) Encode(v interface{}) (err error) { + defer handleErr(&err) + e.encoder.marshalDoc("", reflect.ValueOf(v)) + return nil +} + +// Close closes the encoder by writing any remaining data. +// It does not write a stream terminating string "...". +func (e *Encoder) Close() (err error) { + defer handleErr(&err) + e.encoder.finish() + return nil +} + +func handleErr(err *error) { + if v := recover(); v != nil { + if e, ok := v.(yamlError); ok { + *err = e.err + } else { + panic(v) + } + } +} + +type yamlError struct { + err error +} + +func fail(err error) { + panic(yamlError{err}) +} + +func failf(format string, args ...interface{}) { + panic(yamlError{fmt.Errorf("yaml: "+format, args...)}) +} + +// A TypeError is returned by Unmarshal when one or more fields in +// the YAML document cannot be properly decoded into the requested +// types. When this error is returned, the value is still +// unmarshaled partially. +type TypeError struct { + Errors []string +} + +func (e *TypeError) Error() string { + return fmt.Sprintf("yaml: unmarshal errors:\n %s", strings.Join(e.Errors, "\n ")) +} + +// -------------------------------------------------------------------------- +// Maintain a mapping of keys to structure field indexes + +// The code in this section was copied from mgo/bson. + +// structInfo holds details for the serialization of fields of +// a given struct. +type structInfo struct { + FieldsMap map[string]fieldInfo + FieldsList []fieldInfo + + // InlineMap is the number of the field in the struct that + // contains an ,inline map, or -1 if there's none. + InlineMap int +} + +type fieldInfo struct { + Key string + Num int + OmitEmpty bool + Flow bool + // Id holds the unique field identifier, so we can cheaply + // check for field duplicates without maintaining an extra map. + Id int + + // Inline holds the field index if the field is part of an inlined struct. + Inline []int +} + +var structMap = make(map[reflect.Type]*structInfo) +var fieldMapMutex sync.RWMutex + +func getStructInfo(st reflect.Type) (*structInfo, error) { + fieldMapMutex.RLock() + sinfo, found := structMap[st] + fieldMapMutex.RUnlock() + if found { + return sinfo, nil + } + + n := st.NumField() + fieldsMap := make(map[string]fieldInfo) + fieldsList := make([]fieldInfo, 0, n) + inlineMap := -1 + for i := 0; i != n; i++ { + field := st.Field(i) + if field.PkgPath != "" && !field.Anonymous { + continue // Private field + } + + info := fieldInfo{Num: i} + + tag := field.Tag.Get("yaml") + if tag == "" && strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + if tag == "-" { + continue + } + + inline := false + fields := strings.Split(tag, ",") + if len(fields) > 1 { + for _, flag := range fields[1:] { + switch flag { + case "omitempty": + info.OmitEmpty = true + case "flow": + info.Flow = true + case "inline": + inline = true + default: + return nil, errors.New(fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st)) + } + } + tag = fields[0] + } + + if inline { + switch field.Type.Kind() { + case reflect.Map: + if inlineMap >= 0 { + return nil, errors.New("Multiple ,inline maps in struct " + st.String()) + } + if field.Type.Key() != reflect.TypeOf("") { + return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String()) + } + inlineMap = info.Num + case reflect.Struct: + sinfo, err := getStructInfo(field.Type) + if err != nil { + return nil, err + } + for _, finfo := range sinfo.FieldsList { + if _, found := fieldsMap[finfo.Key]; found { + msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + if finfo.Inline == nil { + finfo.Inline = []int{i, finfo.Num} + } else { + finfo.Inline = append([]int{i}, finfo.Inline...) + } + finfo.Id = len(fieldsList) + fieldsMap[finfo.Key] = finfo + fieldsList = append(fieldsList, finfo) + } + default: + //return nil, errors.New("Option ,inline needs a struct value or map field") + return nil, errors.New("Option ,inline needs a struct value field") + } + continue + } + + if tag != "" { + info.Key = tag + } else { + info.Key = strings.ToLower(field.Name) + } + + if _, found = fieldsMap[info.Key]; found { + msg := "Duplicated key '" + info.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + + info.Id = len(fieldsList) + fieldsList = append(fieldsList, info) + fieldsMap[info.Key] = info + } + + sinfo = &structInfo{ + FieldsMap: fieldsMap, + FieldsList: fieldsList, + InlineMap: inlineMap, + } + + fieldMapMutex.Lock() + structMap[st] = sinfo + fieldMapMutex.Unlock() + return sinfo, nil +} + +// IsZeroer is used to check whether an object is zero to +// determine whether it should be omitted when marshaling +// with the omitempty flag. One notable implementation +// is time.Time. +type IsZeroer interface { + IsZero() bool +} + +func isZero(v reflect.Value) bool { + kind := v.Kind() + if z, ok := v.Interface().(IsZeroer); ok { + if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() { + return true + } + return z.IsZero() + } + switch kind { + case reflect.String: + return len(v.String()) == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + case reflect.Slice: + return v.Len() == 0 + case reflect.Map: + return v.Len() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Struct: + vt := v.Type() + for i := v.NumField() - 1; i >= 0; i-- { + if vt.Field(i).PkgPath != "" { + continue // Private field + } + if !isZero(v.Field(i)) { + return false + } + } + return true + } + return false +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go new file mode 100644 index 00000000000..e25cee563be --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go @@ -0,0 +1,738 @@ +package yaml + +import ( + "fmt" + "io" +) + +// The version directive data. +type yaml_version_directive_t struct { + major int8 // The major version number. + minor int8 // The minor version number. +} + +// The tag directive data. +type yaml_tag_directive_t struct { + handle []byte // The tag handle. + prefix []byte // The tag prefix. +} + +type yaml_encoding_t int + +// The stream encoding. +const ( + // Let the parser choose the encoding. + yaml_ANY_ENCODING yaml_encoding_t = iota + + yaml_UTF8_ENCODING // The default UTF-8 encoding. + yaml_UTF16LE_ENCODING // The UTF-16-LE encoding with BOM. + yaml_UTF16BE_ENCODING // The UTF-16-BE encoding with BOM. +) + +type yaml_break_t int + +// Line break types. +const ( + // Let the parser choose the break type. + yaml_ANY_BREAK yaml_break_t = iota + + yaml_CR_BREAK // Use CR for line breaks (Mac style). + yaml_LN_BREAK // Use LN for line breaks (Unix style). + yaml_CRLN_BREAK // Use CR LN for line breaks (DOS style). +) + +type yaml_error_type_t int + +// Many bad things could happen with the parser and emitter. +const ( + // No error is produced. + yaml_NO_ERROR yaml_error_type_t = iota + + yaml_MEMORY_ERROR // Cannot allocate or reallocate a block of memory. + yaml_READER_ERROR // Cannot read or decode the input stream. + yaml_SCANNER_ERROR // Cannot scan the input stream. + yaml_PARSER_ERROR // Cannot parse the input stream. + yaml_COMPOSER_ERROR // Cannot compose a YAML document. + yaml_WRITER_ERROR // Cannot write to the output stream. + yaml_EMITTER_ERROR // Cannot emit a YAML stream. +) + +// The pointer position. +type yaml_mark_t struct { + index int // The position index. + line int // The position line. + column int // The position column. +} + +// Node Styles + +type yaml_style_t int8 + +type yaml_scalar_style_t yaml_style_t + +// Scalar styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = iota + + yaml_PLAIN_SCALAR_STYLE // The plain scalar style. + yaml_SINGLE_QUOTED_SCALAR_STYLE // The single-quoted scalar style. + yaml_DOUBLE_QUOTED_SCALAR_STYLE // The double-quoted scalar style. + yaml_LITERAL_SCALAR_STYLE // The literal scalar style. + yaml_FOLDED_SCALAR_STYLE // The folded scalar style. +) + +type yaml_sequence_style_t yaml_style_t + +// Sequence styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SEQUENCE_STYLE yaml_sequence_style_t = iota + + yaml_BLOCK_SEQUENCE_STYLE // The block sequence style. + yaml_FLOW_SEQUENCE_STYLE // The flow sequence style. +) + +type yaml_mapping_style_t yaml_style_t + +// Mapping styles. +const ( + // Let the emitter choose the style. + yaml_ANY_MAPPING_STYLE yaml_mapping_style_t = iota + + yaml_BLOCK_MAPPING_STYLE // The block mapping style. + yaml_FLOW_MAPPING_STYLE // The flow mapping style. +) + +// Tokens + +type yaml_token_type_t int + +// Token types. +const ( + // An empty token. + yaml_NO_TOKEN yaml_token_type_t = iota + + yaml_STREAM_START_TOKEN // A STREAM-START token. + yaml_STREAM_END_TOKEN // A STREAM-END token. + + yaml_VERSION_DIRECTIVE_TOKEN // A VERSION-DIRECTIVE token. + yaml_TAG_DIRECTIVE_TOKEN // A TAG-DIRECTIVE token. + yaml_DOCUMENT_START_TOKEN // A DOCUMENT-START token. + yaml_DOCUMENT_END_TOKEN // A DOCUMENT-END token. + + yaml_BLOCK_SEQUENCE_START_TOKEN // A BLOCK-SEQUENCE-START token. + yaml_BLOCK_MAPPING_START_TOKEN // A BLOCK-SEQUENCE-END token. + yaml_BLOCK_END_TOKEN // A BLOCK-END token. + + yaml_FLOW_SEQUENCE_START_TOKEN // A FLOW-SEQUENCE-START token. + yaml_FLOW_SEQUENCE_END_TOKEN // A FLOW-SEQUENCE-END token. + yaml_FLOW_MAPPING_START_TOKEN // A FLOW-MAPPING-START token. + yaml_FLOW_MAPPING_END_TOKEN // A FLOW-MAPPING-END token. + + yaml_BLOCK_ENTRY_TOKEN // A BLOCK-ENTRY token. + yaml_FLOW_ENTRY_TOKEN // A FLOW-ENTRY token. + yaml_KEY_TOKEN // A KEY token. + yaml_VALUE_TOKEN // A VALUE token. + + yaml_ALIAS_TOKEN // An ALIAS token. + yaml_ANCHOR_TOKEN // An ANCHOR token. + yaml_TAG_TOKEN // A TAG token. + yaml_SCALAR_TOKEN // A SCALAR token. +) + +func (tt yaml_token_type_t) String() string { + switch tt { + case yaml_NO_TOKEN: + return "yaml_NO_TOKEN" + case yaml_STREAM_START_TOKEN: + return "yaml_STREAM_START_TOKEN" + case yaml_STREAM_END_TOKEN: + return "yaml_STREAM_END_TOKEN" + case yaml_VERSION_DIRECTIVE_TOKEN: + return "yaml_VERSION_DIRECTIVE_TOKEN" + case yaml_TAG_DIRECTIVE_TOKEN: + return "yaml_TAG_DIRECTIVE_TOKEN" + case yaml_DOCUMENT_START_TOKEN: + return "yaml_DOCUMENT_START_TOKEN" + case yaml_DOCUMENT_END_TOKEN: + return "yaml_DOCUMENT_END_TOKEN" + case yaml_BLOCK_SEQUENCE_START_TOKEN: + return "yaml_BLOCK_SEQUENCE_START_TOKEN" + case yaml_BLOCK_MAPPING_START_TOKEN: + return "yaml_BLOCK_MAPPING_START_TOKEN" + case yaml_BLOCK_END_TOKEN: + return "yaml_BLOCK_END_TOKEN" + case yaml_FLOW_SEQUENCE_START_TOKEN: + return "yaml_FLOW_SEQUENCE_START_TOKEN" + case yaml_FLOW_SEQUENCE_END_TOKEN: + return "yaml_FLOW_SEQUENCE_END_TOKEN" + case yaml_FLOW_MAPPING_START_TOKEN: + return "yaml_FLOW_MAPPING_START_TOKEN" + case yaml_FLOW_MAPPING_END_TOKEN: + return "yaml_FLOW_MAPPING_END_TOKEN" + case yaml_BLOCK_ENTRY_TOKEN: + return "yaml_BLOCK_ENTRY_TOKEN" + case yaml_FLOW_ENTRY_TOKEN: + return "yaml_FLOW_ENTRY_TOKEN" + case yaml_KEY_TOKEN: + return "yaml_KEY_TOKEN" + case yaml_VALUE_TOKEN: + return "yaml_VALUE_TOKEN" + case yaml_ALIAS_TOKEN: + return "yaml_ALIAS_TOKEN" + case yaml_ANCHOR_TOKEN: + return "yaml_ANCHOR_TOKEN" + case yaml_TAG_TOKEN: + return "yaml_TAG_TOKEN" + case yaml_SCALAR_TOKEN: + return "yaml_SCALAR_TOKEN" + } + return "" +} + +// The token structure. +type yaml_token_t struct { + // The token type. + typ yaml_token_type_t + + // The start/end of the token. + start_mark, end_mark yaml_mark_t + + // The stream encoding (for yaml_STREAM_START_TOKEN). + encoding yaml_encoding_t + + // The alias/anchor/scalar value or tag/tag directive handle + // (for yaml_ALIAS_TOKEN, yaml_ANCHOR_TOKEN, yaml_SCALAR_TOKEN, yaml_TAG_TOKEN, yaml_TAG_DIRECTIVE_TOKEN). + value []byte + + // The tag suffix (for yaml_TAG_TOKEN). + suffix []byte + + // The tag directive prefix (for yaml_TAG_DIRECTIVE_TOKEN). + prefix []byte + + // The scalar style (for yaml_SCALAR_TOKEN). + style yaml_scalar_style_t + + // The version directive major/minor (for yaml_VERSION_DIRECTIVE_TOKEN). + major, minor int8 +} + +// Events + +type yaml_event_type_t int8 + +// Event types. +const ( + // An empty event. + yaml_NO_EVENT yaml_event_type_t = iota + + yaml_STREAM_START_EVENT // A STREAM-START event. + yaml_STREAM_END_EVENT // A STREAM-END event. + yaml_DOCUMENT_START_EVENT // A DOCUMENT-START event. + yaml_DOCUMENT_END_EVENT // A DOCUMENT-END event. + yaml_ALIAS_EVENT // An ALIAS event. + yaml_SCALAR_EVENT // A SCALAR event. + yaml_SEQUENCE_START_EVENT // A SEQUENCE-START event. + yaml_SEQUENCE_END_EVENT // A SEQUENCE-END event. + yaml_MAPPING_START_EVENT // A MAPPING-START event. + yaml_MAPPING_END_EVENT // A MAPPING-END event. +) + +var eventStrings = []string{ + yaml_NO_EVENT: "none", + yaml_STREAM_START_EVENT: "stream start", + yaml_STREAM_END_EVENT: "stream end", + yaml_DOCUMENT_START_EVENT: "document start", + yaml_DOCUMENT_END_EVENT: "document end", + yaml_ALIAS_EVENT: "alias", + yaml_SCALAR_EVENT: "scalar", + yaml_SEQUENCE_START_EVENT: "sequence start", + yaml_SEQUENCE_END_EVENT: "sequence end", + yaml_MAPPING_START_EVENT: "mapping start", + yaml_MAPPING_END_EVENT: "mapping end", +} + +func (e yaml_event_type_t) String() string { + if e < 0 || int(e) >= len(eventStrings) { + return fmt.Sprintf("unknown event %d", e) + } + return eventStrings[e] +} + +// The event structure. +type yaml_event_t struct { + + // The event type. + typ yaml_event_type_t + + // The start and end of the event. + start_mark, end_mark yaml_mark_t + + // The document encoding (for yaml_STREAM_START_EVENT). + encoding yaml_encoding_t + + // The version directive (for yaml_DOCUMENT_START_EVENT). + version_directive *yaml_version_directive_t + + // The list of tag directives (for yaml_DOCUMENT_START_EVENT). + tag_directives []yaml_tag_directive_t + + // The anchor (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_ALIAS_EVENT). + anchor []byte + + // The tag (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + tag []byte + + // The scalar value (for yaml_SCALAR_EVENT). + value []byte + + // Is the document start/end indicator implicit, or the tag optional? + // (for yaml_DOCUMENT_START_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_SCALAR_EVENT). + implicit bool + + // Is the tag optional for any non-plain style? (for yaml_SCALAR_EVENT). + quoted_implicit bool + + // The style (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + style yaml_style_t +} + +func (e *yaml_event_t) scalar_style() yaml_scalar_style_t { return yaml_scalar_style_t(e.style) } +func (e *yaml_event_t) sequence_style() yaml_sequence_style_t { return yaml_sequence_style_t(e.style) } +func (e *yaml_event_t) mapping_style() yaml_mapping_style_t { return yaml_mapping_style_t(e.style) } + +// Nodes + +const ( + yaml_NULL_TAG = "tag:yaml.org,2002:null" // The tag !!null with the only possible value: null. + yaml_BOOL_TAG = "tag:yaml.org,2002:bool" // The tag !!bool with the values: true and false. + yaml_STR_TAG = "tag:yaml.org,2002:str" // The tag !!str for string values. + yaml_INT_TAG = "tag:yaml.org,2002:int" // The tag !!int for integer values. + yaml_FLOAT_TAG = "tag:yaml.org,2002:float" // The tag !!float for float values. + yaml_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" // The tag !!timestamp for date and time values. + + yaml_SEQ_TAG = "tag:yaml.org,2002:seq" // The tag !!seq is used to denote sequences. + yaml_MAP_TAG = "tag:yaml.org,2002:map" // The tag !!map is used to denote mapping. + + // Not in original libyaml. + yaml_BINARY_TAG = "tag:yaml.org,2002:binary" + yaml_MERGE_TAG = "tag:yaml.org,2002:merge" + + yaml_DEFAULT_SCALAR_TAG = yaml_STR_TAG // The default scalar tag is !!str. + yaml_DEFAULT_SEQUENCE_TAG = yaml_SEQ_TAG // The default sequence tag is !!seq. + yaml_DEFAULT_MAPPING_TAG = yaml_MAP_TAG // The default mapping tag is !!map. +) + +type yaml_node_type_t int + +// Node types. +const ( + // An empty node. + yaml_NO_NODE yaml_node_type_t = iota + + yaml_SCALAR_NODE // A scalar node. + yaml_SEQUENCE_NODE // A sequence node. + yaml_MAPPING_NODE // A mapping node. +) + +// An element of a sequence node. +type yaml_node_item_t int + +// An element of a mapping node. +type yaml_node_pair_t struct { + key int // The key of the element. + value int // The value of the element. +} + +// The node structure. +type yaml_node_t struct { + typ yaml_node_type_t // The node type. + tag []byte // The node tag. + + // The node data. + + // The scalar parameters (for yaml_SCALAR_NODE). + scalar struct { + value []byte // The scalar value. + length int // The length of the scalar value. + style yaml_scalar_style_t // The scalar style. + } + + // The sequence parameters (for YAML_SEQUENCE_NODE). + sequence struct { + items_data []yaml_node_item_t // The stack of sequence items. + style yaml_sequence_style_t // The sequence style. + } + + // The mapping parameters (for yaml_MAPPING_NODE). + mapping struct { + pairs_data []yaml_node_pair_t // The stack of mapping pairs (key, value). + pairs_start *yaml_node_pair_t // The beginning of the stack. + pairs_end *yaml_node_pair_t // The end of the stack. + pairs_top *yaml_node_pair_t // The top of the stack. + style yaml_mapping_style_t // The mapping style. + } + + start_mark yaml_mark_t // The beginning of the node. + end_mark yaml_mark_t // The end of the node. + +} + +// The document structure. +type yaml_document_t struct { + + // The document nodes. + nodes []yaml_node_t + + // The version directive. + version_directive *yaml_version_directive_t + + // The list of tag directives. + tag_directives_data []yaml_tag_directive_t + tag_directives_start int // The beginning of the tag directives list. + tag_directives_end int // The end of the tag directives list. + + start_implicit int // Is the document start indicator implicit? + end_implicit int // Is the document end indicator implicit? + + // The start/end of the document. + start_mark, end_mark yaml_mark_t +} + +// The prototype of a read handler. +// +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. +// +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. +// +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. +type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) + +// This structure holds information about a potential simple key. +type yaml_simple_key_t struct { + possible bool // Is a simple key possible? + required bool // Is a simple key required? + token_number int // The number of the token. + mark yaml_mark_t // The position mark. +} + +// The states of the parser. +type yaml_parser_state_t int + +const ( + yaml_PARSE_STREAM_START_STATE yaml_parser_state_t = iota + + yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE // Expect the beginning of an implicit document. + yaml_PARSE_DOCUMENT_START_STATE // Expect DOCUMENT-START. + yaml_PARSE_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_PARSE_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_PARSE_BLOCK_NODE_STATE // Expect a block node. + yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE // Expect a block node or indentless sequence. + yaml_PARSE_FLOW_NODE_STATE // Expect a flow node. + yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a block sequence. + yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE // Expect an entry of a block sequence. + yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE // Expect an entry of an indentless sequence. + yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_PARSE_BLOCK_MAPPING_KEY_STATE // Expect a block mapping key. + yaml_PARSE_BLOCK_MAPPING_VALUE_STATE // Expect a block mapping value. + yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE // Expect an entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE // Expect a key of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE // Expect a value of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE // Expect the and of an ordered mapping entry. + yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE // Expect an empty value of a flow mapping. + yaml_PARSE_END_STATE // Expect nothing. +) + +func (ps yaml_parser_state_t) String() string { + switch ps { + case yaml_PARSE_STREAM_START_STATE: + return "yaml_PARSE_STREAM_START_STATE" + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return "yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_START_STATE: + return "yaml_PARSE_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return "yaml_PARSE_DOCUMENT_CONTENT_STATE" + case yaml_PARSE_DOCUMENT_END_STATE: + return "yaml_PARSE_DOCUMENT_END_STATE" + case yaml_PARSE_BLOCK_NODE_STATE: + return "yaml_PARSE_BLOCK_NODE_STATE" + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return "yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE" + case yaml_PARSE_FLOW_NODE_STATE: + return "yaml_PARSE_FLOW_NODE_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return "yaml_PARSE_BLOCK_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE" + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE" + case yaml_PARSE_END_STATE: + return "yaml_PARSE_END_STATE" + } + return "" +} + +// This structure holds aliases data. +type yaml_alias_data_t struct { + anchor []byte // The anchor. + index int // The node id. + mark yaml_mark_t // The anchor mark. +} + +// The parser structure. +// +// All members are internal. Manage the structure using the +// yaml_parser_ family of functions. +type yaml_parser_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + + problem string // Error description. + + // The byte about which the problem occurred. + problem_offset int + problem_value int + problem_mark yaml_mark_t + + // The error context. + context string + context_mark yaml_mark_t + + // Reader stuff + + read_handler yaml_read_handler_t // Read handler. + + input_reader io.Reader // File input data. + input []byte // String input data. + input_pos int + + eof bool // EOF flag + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + unread int // The number of unread characters in the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The input encoding. + + offset int // The offset of the current position (in bytes). + mark yaml_mark_t // The mark of the current position. + + // Scanner stuff + + stream_start_produced bool // Have we started to scan the input stream? + stream_end_produced bool // Have we reached the end of the input stream? + + flow_level int // The number of unclosed '[' and '{' indicators. + + tokens []yaml_token_t // The tokens queue. + tokens_head int // The head of the tokens queue. + tokens_parsed int // The number of tokens fetched from the queue. + token_available bool // Does the tokens queue contain a token ready for dequeueing. + + indent int // The current indentation level. + indents []int // The indentation levels stack. + + simple_key_allowed bool // May a simple key occur at the current position? + simple_keys []yaml_simple_key_t // The stack of simple keys. + + // Parser stuff + + state yaml_parser_state_t // The current parser state. + states []yaml_parser_state_t // The parser states stack. + marks []yaml_mark_t // The stack of marks. + tag_directives []yaml_tag_directive_t // The list of TAG directives. + + // Dumper stuff + + aliases []yaml_alias_data_t // The alias data. + + document *yaml_document_t // The currently parsed document. +} + +// Emitter Definitions + +// The prototype of a write handler. +// +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. +// +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. +// +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. +// +type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error + +type yaml_emitter_state_t int + +// The emitter states. +const ( + // Expect STREAM-START. + yaml_EMIT_STREAM_START_STATE yaml_emitter_state_t = iota + + yaml_EMIT_FIRST_DOCUMENT_START_STATE // Expect the first DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_START_STATE // Expect DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_EMIT_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a flow sequence. + yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE // Expect an item of a flow sequence. + yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a block sequence. + yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE // Expect an item of a block sequence. + yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_KEY_STATE // Expect the key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_VALUE_STATE // Expect a value of a block mapping. + yaml_EMIT_END_STATE // Expect nothing. +) + +// The emitter structure. +// +// All members are internal. Manage the structure using the @c yaml_emitter_ +// family of functions. +type yaml_emitter_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + problem string // Error description. + + // Writer stuff + + write_handler yaml_write_handler_t // Write handler. + + output_buffer *[]byte // String output data. + output_writer io.Writer // File output data. + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The stream encoding. + + // Emitter stuff + + canonical bool // If the output is in the canonical style? + best_indent int // The number of indentation spaces. + best_width int // The preferred width of the output lines. + unicode bool // Allow unescaped non-ASCII characters? + line_break yaml_break_t // The preferred line break. + + state yaml_emitter_state_t // The current emitter state. + states []yaml_emitter_state_t // The stack of states. + + events []yaml_event_t // The event queue. + events_head int // The head of the event queue. + + indents []int // The stack of indentation levels. + + tag_directives []yaml_tag_directive_t // The list of tag directives. + + indent int // The current indentation level. + + flow_level int // The current flow level. + + root_context bool // Is it the document root context? + sequence_context bool // Is it a sequence context? + mapping_context bool // Is it a mapping context? + simple_key_context bool // Is it a simple mapping key context? + + line int // The current line. + column int // The current column. + whitespace bool // If the last character was a whitespace? + indention bool // If the last character was an indentation character (' ', '-', '?', ':')? + open_ended bool // If an explicit document end is required? + + // Anchor analysis. + anchor_data struct { + anchor []byte // The anchor value. + alias bool // Is it an alias? + } + + // Tag analysis. + tag_data struct { + handle []byte // The tag handle. + suffix []byte // The tag suffix. + } + + // Scalar analysis. + scalar_data struct { + value []byte // The scalar value. + multiline bool // Does the scalar contain line breaks? + flow_plain_allowed bool // Can the scalar be expessed in the flow plain style? + block_plain_allowed bool // Can the scalar be expressed in the block plain style? + single_quoted_allowed bool // Can the scalar be expressed in the single quoted style? + block_allowed bool // Can the scalar be expressed in the literal or folded styles? + style yaml_scalar_style_t // The output style. + } + + // Dumper stuff + + opened bool // If the stream was already opened? + closed bool // If the stream was already closed? + + // The information associated with the document nodes. + anchors *struct { + references int // The number of references. + anchor int // The anchor id. + serialized bool // If the node has been emitted? + } + + last_anchor_id int // The last assigned anchor id. + + document *yaml_document_t // The currently emitted document. +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go new file mode 100644 index 00000000000..8110ce3c37a --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go @@ -0,0 +1,173 @@ +package yaml + +const ( + // The size of the input raw buffer. + input_raw_buffer_size = 512 + + // The size of the input buffer. + // It should be possible to decode the whole raw buffer. + input_buffer_size = input_raw_buffer_size * 3 + + // The size of the output buffer. + output_buffer_size = 128 + + // The size of the output raw buffer. + // It should be possible to encode the whole output buffer. + output_raw_buffer_size = (output_buffer_size*2 + 2) + + // The size of other stacks and queues. + initial_stack_size = 16 + initial_queue_size = 16 + initial_string_size = 16 +) + +// Check if the character at the specified position is an alphabetical +// character, a digit, '_', or '-'. +func is_alpha(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'Z' || b[i] >= 'a' && b[i] <= 'z' || b[i] == '_' || b[i] == '-' +} + +// Check if the character at the specified position is a digit. +func is_digit(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' +} + +// Get the value of a digit. +func as_digit(b []byte, i int) int { + return int(b[i]) - '0' +} + +// Check if the character at the specified position is a hex-digit. +func is_hex(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'F' || b[i] >= 'a' && b[i] <= 'f' +} + +// Get the value of a hex-digit. +func as_hex(b []byte, i int) int { + bi := b[i] + if bi >= 'A' && bi <= 'F' { + return int(bi) - 'A' + 10 + } + if bi >= 'a' && bi <= 'f' { + return int(bi) - 'a' + 10 + } + return int(bi) - '0' +} + +// Check if the character is ASCII. +func is_ascii(b []byte, i int) bool { + return b[i] <= 0x7F +} + +// Check if the character at the start of the buffer can be printed unescaped. +func is_printable(b []byte, i int) bool { + return ((b[i] == 0x0A) || // . == #x0A + (b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E + (b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF + (b[i] > 0xC2 && b[i] < 0xED) || + (b[i] == 0xED && b[i+1] < 0xA0) || + (b[i] == 0xEE) || + (b[i] == 0xEF && // #xE000 <= . <= #xFFFD + !(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF + !(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF)))) +} + +// Check if the character at the specified position is NUL. +func is_z(b []byte, i int) bool { + return b[i] == 0x00 +} + +// Check if the beginning of the buffer is a BOM. +func is_bom(b []byte, i int) bool { + return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF +} + +// Check if the character at the specified position is space. +func is_space(b []byte, i int) bool { + return b[i] == ' ' +} + +// Check if the character at the specified position is tab. +func is_tab(b []byte, i int) bool { + return b[i] == '\t' +} + +// Check if the character at the specified position is blank (space or tab). +func is_blank(b []byte, i int) bool { + //return is_space(b, i) || is_tab(b, i) + return b[i] == ' ' || b[i] == '\t' +} + +// Check if the character at the specified position is a line break. +func is_break(b []byte, i int) bool { + return (b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029) +} + +func is_crlf(b []byte, i int) bool { + return b[i] == '\r' && b[i+1] == '\n' +} + +// Check if the character is a line break or NUL. +func is_breakz(b []byte, i int) bool { + //return is_break(b, i) || is_z(b, i) + return ( // is_break: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + // is_z: + b[i] == 0) +} + +// Check if the character is a line break, space, or NUL. +func is_spacez(b []byte, i int) bool { + //return is_space(b, i) || is_breakz(b, i) + return ( // is_space: + b[i] == ' ' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Check if the character is a line break, space, tab, or NUL. +func is_blankz(b []byte, i int) bool { + //return is_blank(b, i) || is_breakz(b, i) + return ( // is_blank: + b[i] == ' ' || b[i] == '\t' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Determine the width of the character. +func width(b byte) int { + // Don't replace these by a switch without first + // confirming that it is being inlined. + if b&0x80 == 0x00 { + return 1 + } + if b&0xE0 == 0xC0 { + return 2 + } + if b&0xF0 == 0xE0 { + return 3 + } + if b&0xF8 == 0xF0 { + return 4 + } + return 0 + +} diff --git a/tools/voluspa/version.go b/tools/voluspa/version.go new file mode 100644 index 00000000000..97ea06e872e --- /dev/null +++ b/tools/voluspa/version.go @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +var ( + // Version is the build version + Version = "dev" + + // GitVersion is the Git revision hash + GitVersion = "dev" +) diff --git a/tools/voluspa/version/version.go b/tools/voluspa/version/version.go new file mode 100644 index 00000000000..cc71dc4f65d --- /dev/null +++ b/tools/voluspa/version/version.go @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package version + +import "expvar" + +var ( + // Version is the build version + Version = "dev" + + // GitVersion is the Git revision hash + GitVersion = "dev" +) + +func init() { + if len(GitVersion) > 0 { + Version = Version + "/" + GitVersion + } + + v := expvar.NewString("version") + v.Set(Version) +} diff --git a/tools/voluspa/version/version_test.go b/tools/voluspa/version/version_test.go new file mode 100644 index 00000000000..701d5109d2d --- /dev/null +++ b/tools/voluspa/version/version_test.go @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package version diff --git a/tools/voluspa/voluspa.go b/tools/voluspa/voluspa.go new file mode 100644 index 00000000000..2de67d2118f --- /dev/null +++ b/tools/voluspa/voluspa.go @@ -0,0 +1,330 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "sort" + "strconv" + "strings" + + "github.com/xeipuuv/gojsonschema" + yaml "gopkg.in/yaml.v2" +) + +var ( + ErrMinimumConfigsNotMet = errors.New("Need at least 1 parsed config file") +) + +type Voluspa struct { + parsedConfigs []*CustomerConfig + + installLocation string + options *Options + defaultSchemaVersion int +} + +func NewVoluspa() (*Voluspa, error) { + return NewVoluspaWithOptions(&Options{}, DefaultTrafficserverConfigurationDir) +} + +func NewVoluspaWithOptions(options *Options, installLocation string) (*Voluspa, error) { + return &Voluspa{ + installLocation: installLocation, + options: options, + defaultSchemaVersion: 1, + }, nil +} + +func (v *Voluspa) SchemaDefaultVersion() int { + return v.defaultSchemaVersion +} + +func (v *Voluspa) SchemaDefinition(version int) ([]byte, error) { + if version == 0 { + version = v.SchemaDefaultVersion() + } + + var filename string + if len(v.options.SchemaLocation) > 0 { + filename = fmt.Sprintf("%s/schema_v%d.json", v.options.SchemaLocation, version) + } else { + filename = fmt.Sprintf("schema_v%d.json", version) + } + + return ioutil.ReadFile(filename) +} + +func (v *Voluspa) validateSchema(contents []byte) error { + var ycfg interface{} + if err := yaml.Unmarshal(contents, &ycfg); err != nil { + return err + } + + ycfg = convert(ycfg) + + jcfg, err := json.Marshal(ycfg) + if err != nil { + return err + } + + schemaVersion := v.SchemaDefaultVersion() // TODO extract schema_version from ycfg + schemaContent, err := v.SchemaDefinition(schemaVersion) + if err != nil { + return err + } + + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaContent)) + if err != nil { + return fmt.Errorf("error loading the version %d schema definition: %v", schemaVersion, err) + } + + result, err := schema.Validate(gojsonschema.NewBytesLoader(jcfg)) + if err != nil { + return err + } + + if result.Valid() { + return nil + } + + var buf bytes.Buffer + for _, desc := range result.Errors() { + buf.WriteString(fmt.Sprintf(" - %s\n", desc)) + } + + return fmt.Errorf("\n%s", buf.String()) +} + +func convert(i interface{}) interface{} { + switch x := i.(type) { + case map[interface{}]interface{}: + m2 := map[string]interface{}{} + for k, v := range x { + var k2 string + switch k := k.(type) { + case int: + k2 = strconv.Itoa(k) + default: + k2 = k.(string) + } + m2[k2] = convert(v) + } + return m2 + case []interface{}: + for i, v := range x { + x[i] = convert(v) + } + } + return i +} + +// AddConfigByBytes takes a []byte buffer containing a Voluspa YAML config file and a filename and processes it +func (v *Voluspa) AddConfigByBytes(buffer []byte, filename string) error { + if !v.options.SkipSchemaValidation { + if err := v.validateSchema(buffer); err != nil { + return wrapError(filename, err) + } + } + + cfg, err := NewPropertyConfigWithDefaults(buffer, v.options.DefaultsLocation) + if err != nil { + return wrapError(filename, err) + } + + cc, err := NewCustomerConfig(cfg, filename, *v.options) + if err != nil { + return wrapError(filename, err) + } + + v.parsedConfigs = append(v.parsedConfigs, cc) + + return nil +} + +func (v *Voluspa) AddConfig(filename string) error { + buffer, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + return v.AddConfigByBytes(buffer, filename) +} + +func (v *Voluspa) filterConfigsByCDNAndRole(cdns, roles []string) []*CustomerConfig { + if len(cdns) == 0 { + return v.parsedConfigs + } + + cdnMap := make(map[string]interface{}) + for k := range cdns { + cdnMap[cdns[k]] = nil + } + + roleMap := make(map[string]interface{}) + for k := range roles { + roleMap[roles[k]] = nil + } + + var grouped []*CustomerConfig + for _, config := range v.parsedConfigs { + if config.lifecycle == Retired { + continue + } + + if !v.options.PromoteRolesToCDN && isPromotableRole(config.role) { + continue + } + + includeCDN := false + for cdn := range config.cdn { + if _, exists := cdnMap[cdn]; exists { + includeCDN = true + break + } + } + + var includeRole bool + if len(config.role) > 0 { + _, includeRole = roleMap[config.role] + } else { + includeRole = true + } + + if includeCDN && includeRole { + grouped = append(grouped, config) + } + } + + sort.Stable(sort.Reverse(byQPSAndName(grouped))) + + return grouped +} + +// Validate will run all validation methods and return an error if any issues were found +// strict will enable extra checks that are known to fail with reasonable sets of configuration files +func (v *Voluspa) Validate(strict bool) error { + var errs Errors + errs = append(errs, v.ensurePropertyNameUniqueness()...) + errs = append(errs, v.ensureAliasUniqueness()...) + errs = append(errs, v.validateRegexOptions()...) + errs = append(errs, v.ensureValidSchemes()...) + + if strict { + errs = append(errs, v.ensureHostsResolve()...) + } + + if len(errs) > 0 { + return errs + } + + return nil +} + +// WriteAllFiles will write out all files, filter by passed in cdns and roles +func (v *Voluspa) WriteAllFiles(cdns, roles []string) error { + allFiles, err := v.GetManagedFilesByCDNAndRole(cdns, roles) + if err != nil { + return err + } + + writer, err := NewFilesystemWriter(v.options) + if err != nil { + return err + } + + return writer.WriteFiles(allFiles) +} + +// PropertyCount returns the number of properties +func (v *Voluspa) PropertyCount() int { + return len(v.parsedConfigs) +} + +type configGenerator interface { + // Do processes parsedConfigs, optionally merging, returning a slice of ManagedFiles + Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) +} + +func getGenerators(options *Options) []configGenerator { + var cg []configGenerator + cg = append(cg, newPropertyRemapGenerator(options)) + cg = append(cg, newTopLevelRemapGenerator(options)) + cg = append(cg, newParentConfigurator(options)) + cg = append(cg, newSSLMulticertConfigurator(options)) + cg = append(cg, &HAProxyConfigGenerator{}) + cg = append(cg, newHostingConfigurator(options)) + return cg +} + +func (v *Voluspa) GetManagedFiles(cdns []string) ([]ManagedFile, error) { + return v.GetManagedFilesByCDNAndRole(cdns, nil) +} + +func (v *Voluspa) GetManagedFilesByCDNAndRole(cdns, roles []string) ([]ManagedFile, error) { + cc := v.filterConfigsByCDNAndRole(cdns, roles) + merged := false + if len(cdns) > 0 { + merged = true + } + + var allFiles []ManagedFile + for _, g := range getGenerators(v.options) { + managedFiles, err := g.Do(cc, merged) + if err != nil { + return nil, err + } + allFiles = append(allFiles, managedFiles...) + } + return allFiles, nil +} + +func wrapError(filename string, err error) error { + _, ok := err.(Errors) + if !ok { + return fmt.Errorf("problem loading %q: %s", filename, err) + } + return fmt.Errorf("problem loading %q:\n%s", filename, err) +} + +// byQPSAndName implements sort.Interface, sorting by a CustomerConfig by qps/name. +type byQPSAndName []*CustomerConfig + +func (s byQPSAndName) Len() int { return len(s) } +func (s byQPSAndName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byQPSAndName) Less(i, j int) bool { + if s[i].qps < s[j].qps { + return true + } + if s[i].qps > s[j].qps { + return false + } + + // normalize the property names for sorting + return strings.ToLower(s[i].property) > strings.ToLower(s[j].property) +} + +// isPromotableRole returns true of role is promotable +func isPromotableRole(role string) bool { + return strings.HasPrefix(role, "roles_") +}