From 43ae653c113dedcfef9a0c47caf7dfcfe4c3b735 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 19 Jul 2019 13:28:16 -0600 Subject: [PATCH 1/7] Removes the balancer plugin. See #5395 --- doc/admin-guide/plugins/balancer.en.rst | 87 ------- doc/admin-guide/plugins/index.en.rst | 4 - .../admin-guide/introduction.en.po | 9 - .../admin-guide/plugins/balancer.en.po | 134 ---------- .../admin-guide/plugins/index.en.po | 4 - plugins/Makefile.am | 1 - plugins/experimental/balancer/Makefile.inc | 23 -- plugins/experimental/balancer/balancer.cc | 196 --------------- plugins/experimental/balancer/balancer.h | 50 ---- plugins/experimental/balancer/hash.cc | 230 ------------------ plugins/experimental/balancer/roundrobin.cc | 72 ------ .../pluginTest/url_sig/url_sig.test.py | 26 +- 12 files changed, 13 insertions(+), 823 deletions(-) delete mode 100644 doc/admin-guide/plugins/balancer.en.rst delete mode 100644 doc/locale/ja/LC_MESSAGES/admin-guide/plugins/balancer.en.po delete mode 100644 plugins/experimental/balancer/Makefile.inc delete mode 100644 plugins/experimental/balancer/balancer.cc delete mode 100644 plugins/experimental/balancer/balancer.h delete mode 100644 plugins/experimental/balancer/hash.cc delete mode 100644 plugins/experimental/balancer/roundrobin.cc diff --git a/doc/admin-guide/plugins/balancer.en.rst b/doc/admin-guide/plugins/balancer.en.rst deleted file mode 100644 index 8283c3818a6..00000000000 --- a/doc/admin-guide/plugins/balancer.en.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. _admin-plugins-balancer: - -Balancer Plugin -*************** - -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. note:: - - All of the the features in this plugin (and more) are found in - :file:`parent.config`. As a result, this plugin is likely to be deprecated. - -The ``balancer`` balances requests across multiple origin servers. -To use this plugin, configure it in a :file:`remap.config` rule, specifying -a balancing policy and a set of origin servers. For example:: - - map http://foo.com http://foo.com \ - @plugin=balancer.so @pparam=--policy=hash,url @pparam=one.bar.com @pparam=two.bar.com - -The ``replacement`` URL in the mapping rule is not used. The argument -to the ``--policy`` option is a comma-separated list of keywords. -The first keyword is the name of a balancing policy. The subsequent -keywords are used to refine the requested policy. - -The remaining plugin arguments are balancer targets. Typically, -these will be the host names of origin servers that requests should -be balanced across. The target name may contain a colon-separated -port number. - -Hash Balancing Policy ---------------------- - -The ``hash`` balancing policy performs a consistent hash across the -set of origins. This minimizes the number of hash entries that must -be moved when the set of origin servers changes. An optional list -of hash fields follows the ``hash`` keyword. Each specified hash -field is hashed to select an outbound origin server. - -The following fields can be supplied to the hash: - -key - The request cache key. Note that the cache key will only be - set if you have already chained a plugin that sets a custom - cache key. - -url - The request URL. This is the default hash field that is used if - no other fields are specified. - -srcaddr - The source IP address of the request. - -dstaddr - The destination IP address of the request. - -Round Robin Balancing Policy ----------------------------- - -The ``roundrobin`` balancing policy simply allocates requests to -origin servers in order. Over time, the number of requests received -by each origin should be approximately the same. - -Health Checking ---------------- - -The ``balancer`` plugin does not check the health of the origin -servers, however the plugin is fully reloadable so health checking -is usually simple to implement. Most production environments already -have mechanisms to check service health. It is recommended that you -write a simple script to monitor this information and rewrite -:file:`remap.config` when appropriate. Running :option:`traffic_ctl config reload` -will reload the ``balancer`` plugin with the new set of origin servers. diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index 3d422dc66fd..ff078b46c62 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -144,7 +144,6 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :hidden: Access Control - Balancer Buffer Upload Certifier Cert Reporting Tool @@ -173,9 +172,6 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`Access Control ` Access control plugin that handles various access control use-cases. -:doc:`Balancer ` - Balances requests across multiple origin servers. - :doc:`Buffer Upload ` Buffers POST data before connecting to the Origin server. diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/introduction.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/introduction.en.po index 78e29285367..6a0a120ffac 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/introduction.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/introduction.en.po @@ -200,15 +200,6 @@ msgstr "" "場合 Traffic Server の内部的な API やアーキテクチャ的な制限が実装を難しくし" "ていることもあります。" -#: ../../../admin-guide/introduction.en.rst:106 -#, fuzzy -msgid "" -"Load Balancing - note that there is an experimental plugin for this, :ref:" -"`admin-plugins-balancer`." -msgstr "" -"ロードバランシング - このための実験的なプラクインがあることに注意してくださ" -"い。 :ref:`balancer-plugin`" - #: ../../../admin-guide/introduction.en.rst:112 msgid "Traffic Server Components" msgstr "Traffic Server Components" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/balancer.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/balancer.en.po deleted file mode 100644 index d20a8f62e5a..00000000000 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/balancer.en.po +++ /dev/null @@ -1,134 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: Apache Traffic Server 6.2\n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2016-06-30 14:07+0900\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=utf-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.2.0\n" - -#: ../../../admin-guide/plugins/balancer.en.rst:4 -msgid "Balancer Plugin" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:24 -msgid "" -"The ``balancer`` balances requests across multiple origin servers. To use " -"this plugin, configure it in a :file:`remap.config` rule, specifying a " -"balancing policy and a set of origin servers. For example::" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:31 -msgid "" -"The ``replacement`` URL in the mapping rule is not used. The argument to " -"the ``--policy`` option is a comma-separated list of keywords. The first " -"keyword is the name of a balancing policy. The subsequent keywords are used " -"to refine the requested policy." -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:36 -msgid "" -"The remaining plugin arguments are balancer targets. Typically, these will " -"be the host names of origin servers that requests should be balanced " -"across. The target name may contain a colon-separated port number." -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:42 -msgid "Hash Balancing Policy" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:44 -msgid "" -"The ``hash`` balancing policy performs a consistent hash across the set of " -"origins. This minimizes the number of hash entries that must be moved when " -"the set of origin servers changes. An optional list of hash fields follows " -"the ``hash`` keyword. Each specified hash field is hashed to select an " -"outbound origin server." -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:50 -msgid "The following fields can be supplied to the hash:" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:55 -msgid "key" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:53 -msgid "" -"The request cache key. Note that the cache key will only be set if you have " -"already chained a plugin that sets a custom cache key." -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:59 -msgid "url" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:58 -msgid "" -"The request URL. This is the default hash field that is used if no other " -"fields are specified." -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:62 -msgid "srcaddr" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:62 -msgid "The source IP address of the request." -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:65 -msgid "dstaddr" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:65 -msgid "The destination IP address of the request." -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:68 -msgid "Round Robin Balancing Policy" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:70 -msgid "" -"The ``roundrobin`` balancing policy simply allocates requests to origin " -"servers in order. Over time, the number of requests received by each origin " -"should be approximately the same." -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:75 -msgid "Health Checking" -msgstr "" - -#: ../../../admin-guide/plugins/balancer.en.rst:77 -msgid "" -"The ``balancer`` plugin does not check the health of the origin servers, " -"however the plugin is fully reloadable so health checking is usually simple " -"to implement. Most production environments already have mechanisms to check " -"service health. It is recommended that you write a simple script to monitor " -"this information and rewrite :file:`remap.config` when appropriate. " -"Running :option:`traffic_ctl config reload` will reload the ``balancer`` " -"plugin with the new set of origin servers." -msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po index ac06ffd790a..2eb3788f0ed 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po @@ -178,10 +178,6 @@ msgid "" "into cache." msgstr "" -#: ../../../admin-guide/plugins/index.en.rst:135 -msgid ":doc:`Balancer `" -msgstr "" - #: ../../../admin-guide/plugins/index.en.rst:135 msgid "Balances requests across multiple origin servers." msgstr "" diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 18a08ac1c0d..d868692e3c7 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -58,7 +58,6 @@ if BUILD_EXPERIMENTAL_PLUGINS include experimental/access_control/Makefile.inc include experimental/acme/Makefile.inc -include experimental/balancer/Makefile.inc include experimental/buffer_upload/Makefile.inc include experimental/cert_reporting_tool/Makefile.inc include experimental/collapsed_forwarding/Makefile.inc diff --git a/plugins/experimental/balancer/Makefile.inc b/plugins/experimental/balancer/Makefile.inc deleted file mode 100644 index 6c93be50723..00000000000 --- a/plugins/experimental/balancer/Makefile.inc +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pkglib_LTLIBRARIES += experimental/balancer/balancer.la - -experimental_balancer_balancer_la_SOURCES = \ - experimental/balancer/balancer.cc\ - experimental/balancer/roundrobin.cc \ - experimental/balancer/hash.cc \ - experimental/balancer/balancer.h diff --git a/plugins/experimental/balancer/balancer.cc b/plugins/experimental/balancer/balancer.cc deleted file mode 100644 index 4bce592a94e..00000000000 --- a/plugins/experimental/balancer/balancer.cc +++ /dev/null @@ -1,196 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "balancer.h" -#include -#include -#include -#include -#include -#include - -// Using ink_inet API is cheating, but I was too lazy to write new IPv6 address parsing routines ;) -#include "tscore/ink_inet.h" - -// The policy type is the first comma-separated token. -static BalancerInstance * -MakeBalancerInstance(const char *opt) -{ - const char *end = strchr(opt, ','); - size_t len = end ? std::distance(opt, end) : strlen(opt); - - if (len == lengthof("hash") && strncmp(opt, "hash", len) == 0) { - return MakeHashBalancer(end ? end + 1 : nullptr); - } else if (len == lengthof("roundrobin") && strncmp(opt, "roundrobin", len) == 0) { - return MakeRoundRobinBalancer(end ? end + 1 : nullptr); - } else { - TSError("[balancer] Invalid balancing policy '%.*s'", (int)len, opt); - return nullptr; - } -} - -static BalancerTarget -MakeBalancerTarget(const char *strval) -{ - BalancerTarget target = BalancerTarget(); - - union { - struct sockaddr_storage storage; - struct sockaddr sa; - } address; - - memset(&address, 0, sizeof(address)); - - // First, check whether we have an address literal. - if (ats_ip_pton(strval, &address.sa) == 0) { - char namebuf[INET6_ADDRSTRLEN]; - - target.port = ats_ip_port_host_order(&address.sa); - target.name = ats_ip_ntop(&address.sa, namebuf, sizeof(namebuf)); - } else { - const char *colon = strrchr(strval, ':'); - - if (colon) { - size_t len = std::distance(strval, colon); - - target.port = strtol(colon + 1, nullptr, 10); - target.name = std::string(strval, len); - } else { - target.port = 0; - target.name = strval; - } - } - - if (target.port > UINT16_MAX) { - TSError("[balancer] Ignoring invalid port number for target '%s'", strval); - target.port = 0; - } - - return target; -} - -TSReturnCode -TSRemapInit(TSRemapInterface * /* api */, char * /* errbuf */, int /* bufsz */) -{ - return TS_SUCCESS; -} - -/////////////////////////////////////////////////////////////////////////////// -// One instance per remap.config invocation. -// -TSReturnCode -TSRemapNewInstance(int argc, char *argv[], void **instance, char *errbuf, int errbuf_size) -{ - static const struct option longopt[] = { - {const_cast("policy"), required_argument, nullptr, 'p'}, - {nullptr, 0, nullptr, 0}, - }; - - BalancerInstance *balancer = nullptr; - - // The first two arguments are the "from" and "to" URL string. We need to - // skip them, but we also require that there be an option to masquerade as - // argv[0], so we increment the argument indexes by 1 rather than by 2. - argc--; - argv++; - - for (;;) { - int opt; - - opt = getopt_long(argc, (char *const *)argv, "", longopt, nullptr); - switch (opt) { - case 'p': - if (!balancer) { - balancer = MakeBalancerInstance(optarg); - } else { - TSError("[balancer] Duplicate --policy options, ignored %s", optarg); - } - break; - case -1: - break; - default: - snprintf(errbuf, errbuf_size, "invalid balancer option '%d'", opt); - delete balancer; - return TS_ERROR; - } - - if (opt == -1) { - break; - } - } - - if (!balancer) { - strncpy(errbuf, "missing balancer policy", errbuf_size); - return TS_ERROR; - } - - // Pick up the remaining options as balance targets. - for (int i = optind; i < argc; ++i) { - BalancerTarget target = MakeBalancerTarget(argv[i]); - - balancer->push_target(target); - if (target.port) { - TSDebug("balancer", "added target -> %s:%u", target.name.c_str(), target.port); - } else { - TSDebug("balancer", "added target -> %s", target.name.c_str()); - } - } - - *instance = balancer; - return TS_SUCCESS; -} - -void -TSRemapDeleteInstance(void *instance) -{ - delete (BalancerInstance *)instance; -} - -TSRemapStatus -TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo *rri) -{ - BalancerInstance *balancer = (BalancerInstance *)instance; - const BalancerTarget &target = balancer->balance(txn, rri); - - if (TSIsDebugTagSet("balancer")) { - char *url; - int len; - - url = TSHttpTxnEffectiveUrlStringGet(txn, &len); - if (target.port) { - TSDebug("balancer", "%s:%u <- %.*s", target.name.c_str(), target.port, len, url); - } else { - TSDebug("balancer", "%s <- %.*s", target.name.c_str(), len, url); - } - - TSfree(url); - } - - TSUrlHostSet(rri->requestBufp, rri->requestUrl, target.name.data(), target.name.size()); - - if (target.port) { - TSUrlPortSet(rri->requestBufp, rri->requestUrl, target.port); - } - - return TSREMAP_DID_REMAP; -} diff --git a/plugins/experimental/balancer/balancer.h b/plugins/experimental/balancer/balancer.h deleted file mode 100644 index 54bedd142d0..00000000000 --- a/plugins/experimental/balancer/balancer.h +++ /dev/null @@ -1,50 +0,0 @@ -/** @file - * - * A brief file description - * - * @section license License - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -// Return the length of a string literal. -template -unsigned -lengthof(const char (&)[N]) -{ - return N - 1; -} - -struct BalancerTarget { - std::string name; - unsigned port; -}; - -struct BalancerInstance { - virtual ~BalancerInstance() {} - virtual void push_target(const BalancerTarget &) = 0; - virtual const BalancerTarget &balance(TSHttpTxn, TSRemapRequestInfo *) = 0; -}; - -BalancerInstance *MakeHashBalancer(const char *); -BalancerInstance *MakeRoundRobinBalancer(const char *); diff --git a/plugins/experimental/balancer/hash.cc b/plugins/experimental/balancer/hash.cc deleted file mode 100644 index c6ff0a60331..00000000000 --- a/plugins/experimental/balancer/hash.cc +++ /dev/null @@ -1,230 +0,0 @@ -/** @file - * - * A brief file description - * - * @section license License - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "balancer.h" -#include -#include -#include -#include -#include -#include -#include - -namespace -{ -size_t -sockaddrlen(const struct sockaddr *sa) -{ - switch (sa->sa_family) { - case AF_INET: - return sizeof(struct sockaddr_in); - case AF_INET6: - return sizeof(struct sockaddr_in6); - default: - TSReleaseAssert(0 && "unsupported socket type"); - } - - return 0; -} - -struct md5_key { - md5_key() {} - md5_key(const BalancerTarget &target, unsigned i) - { - MD5_CTX ctx; - - MD5_Init(&ctx); - MD5_Update(&ctx, target.name.data(), target.name.size()); - MD5_Update(&ctx, &target.port, sizeof(target.port)); - MD5_Update(&ctx, &i, sizeof(i)); - MD5_Final(this->key, &ctx); - } - - bool - operator<(const md5_key &rhs) const - { - return memcmp(this->key, rhs.key, sizeof(this->key)) < 0; - } - - unsigned char key[MD5_DIGEST_LENGTH]; -}; - -using HashComponent = void (*)(TSHttpTxn, TSRemapRequestInfo *, MD5_CTX *); - -// Hash on the source (client) IP address. -void -HashTxnSrcaddr(TSHttpTxn txn, TSRemapRequestInfo *, MD5_CTX *ctx) -{ - struct sockaddr const *sa; - - sa = TSHttpTxnClientAddrGet(txn); - if (sa) { - MD5_Update(ctx, sa, sockaddrlen(sa)); - TSDebug("balancer", "%s(addr[%zu]]", __func__, sockaddrlen(sa)); - } -} - -// Hash on the destination (server) IP address; -void -HashTxnDstaddr(TSHttpTxn txn, TSRemapRequestInfo *, MD5_CTX *ctx) -{ - struct sockaddr const *sa; - - sa = TSHttpTxnIncomingAddrGet(txn); - if (sa) { - MD5_Update(ctx, sa, sockaddrlen(sa)); - TSDebug("balancer", "%s(addr[%zu]]", __func__, sockaddrlen(sa)); - } -} - -// Hash on the request URL. -void -HashTxnUrl(TSHttpTxn txn, TSRemapRequestInfo *, MD5_CTX *ctx) -{ - char *url; - int len; - - url = TSHttpTxnEffectiveUrlStringGet(txn, &len); - if (url && len) { - MD5_Update(ctx, url, len); - TSDebug("balancer", "%s(%.*s)", __func__, len, url); - } - - TSfree(url); -} - -// Hash on the cache key. This is not typically set at remap time, unless by another plugin. -void -HashTxnKey(TSHttpTxn txn, TSRemapRequestInfo *rri, MD5_CTX *ctx) -{ - TSMLoc url = TS_NULL_MLOC; - char *str = nullptr; - int len; - - if (TSUrlCreate(rri->requestBufp, &url) != TS_SUCCESS) { - goto done; - } - - if (TSHttpTxnCacheLookupUrlGet(txn, rri->requestBufp, url) != TS_SUCCESS) { - TSDebug("balancer", "no cache key"); - goto done; - } - - str = TSUrlStringGet(rri->requestBufp, url, &len); - if (str && len) { - TSDebug("balancer", "%s(%.*s)", __func__, len, str); - MD5_Update(ctx, str, len); - } - -done: - if (url != TS_NULL_MLOC) { - TSHandleMLocRelease(rri->requestBufp, TS_NULL_MLOC, url); - } - - TSfree(str); -} - -struct HashBalancer : public BalancerInstance { - typedef std::map hash_ring_type; - using hash_part_type = std::vector; - - enum { - iterations = 10, - }; - - HashBalancer() { this->hash_parts.push_back(HashTxnUrl); } - void - push_target(const BalancerTarget &target) override - { - for (unsigned i = 0; i < iterations; ++i) { - this->hash_ring.insert(std::make_pair(md5_key(target, i), target)); - } - } - - const BalancerTarget & - balance(TSHttpTxn txn, TSRemapRequestInfo *rri) override - { - md5_key key; - MD5_CTX ctx; - hash_ring_type::const_iterator loc; - - // We'd better have some hash functions set by now ... - TSReleaseAssert(!hash_parts.empty()); - - MD5_Init(&ctx); - - for (hash_part_type::const_iterator i = this->hash_parts.begin(); i != this->hash_parts.end(); ++i) { - (*i)(txn, rri, &ctx); - } - - MD5_Final(key.key, &ctx); - - // OK, now look up this hash in the hash ring. lower_bound() finds the first element that is not less than the - // target, so the element we find is the first key that is greater than our target. To visualize this in the - // hash ring, that means that each node owns the preceding keyspace (ie. the node is at the end of each keyspace - // range). This means that when we wrap, the first node owns the wrapping portion of the keyspace. - loc = this->hash_ring.lower_bound(key); - if (loc == this->hash_ring.end()) { - loc = this->hash_ring.begin(); - } - - return loc->second; - } - - hash_ring_type hash_ring; - hash_part_type hash_parts; -}; - -} // namespace - -BalancerInstance * -MakeHashBalancer(const char *options) -{ - HashBalancer *hash = new HashBalancer(); - char *opt; - char *tmp; - - TSDebug("balancer", "making hash balancer with options '%s'", options); - - if (options) { - hash->hash_parts.clear(); // clear the default hash type if we have options - options = tmp = strdup(options); - while ((opt = strsep(&tmp, ",")) != nullptr) { - if (strcmp(opt, "key") == 0) { - hash->hash_parts.push_back(HashTxnKey); - } else if (strcmp(opt, "url") == 0) { - hash->hash_parts.push_back(HashTxnUrl); - } else if (strcmp(opt, "srcaddr") == 0) { - hash->hash_parts.push_back(HashTxnSrcaddr); - } else if (strcmp(opt, "dstaddr") == 0) { - hash->hash_parts.push_back(HashTxnDstaddr); - } else { - TSError("[balancer] Ignoring invalid hash field '%s'", opt); - } - } - - free((void *)options); - } - - return hash; -} diff --git a/plugins/experimental/balancer/roundrobin.cc b/plugins/experimental/balancer/roundrobin.cc deleted file mode 100644 index 6090ab97795..00000000000 --- a/plugins/experimental/balancer/roundrobin.cc +++ /dev/null @@ -1,72 +0,0 @@ -/** @file - * - * A brief file description - * - * @section license License - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "balancer.h" -#include -#include -#include -#include -#include - -namespace -{ -struct RoundRobinBalancer : public BalancerInstance { - RoundRobinBalancer() : targets() {} - void - push_target(const BalancerTarget &target) override - { - this->targets.push_back(target); - } - - const BalancerTarget & - balance(TSHttpTxn, TSRemapRequestInfo *) override - { - return this->targets[++next % this->targets.size()]; - } - - std::vector targets; - unsigned next = 0; -}; - -} // namespace - -BalancerInstance * -MakeRoundRobinBalancer(const char *options) -{ - RoundRobinBalancer *hash = new RoundRobinBalancer(); - char *opt; - char *tmp; - - TSDebug("balancer", "making round robin balancer with options '%s'", options); - - if (options) { - options = tmp = strdup(options); - while ((opt = strsep(&tmp, ",")) != nullptr) { - TSError("[balancer] Ignoring invalid round robin field '%s'", opt); - } - - free((void *)options); - } - - return hash; -} diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py index 69d8c622717..3c96e898035 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py @@ -28,7 +28,6 @@ # Skip if plugins not present. Test.SkipUnless(Condition.PluginExists('url_sig.so')) -Test.SkipUnless(Condition.PluginExists('balancer.so')) # Set up to check the output after the tests have run. # @@ -89,8 +88,7 @@ # Use pristine URL, incoming URL changed. # ts.Disk.remap_config.AddLine( - 'map http://seven.eight.nine/ http://dummy' + - ' @plugin=balancer.so @pparam=--policy=hash,url @pparam=127.0.0.1:{}'.format(server.Variables.Port) + + 'map http://seven.eight.nine/ http://127.0.0.1:{}'.format(server.Variables.Port) + ' @plugin=url_sig.so @pparam={}/url_sig.config @pparam=PristineUrl'.format(Test.TestDirectory) ) @@ -232,17 +230,19 @@ LogTee ) + def sign(payload, key): - secret=bytes(key,'utf-8') - data=bytes(payload, 'utf-8') - md=bytes(hmac.new(secret, data, digestmod=hashlib.sha1).digest().hex(), 'utf-8') - return md.decode("utf-8") + secret = bytes(key, 'utf-8') + data = bytes(payload, 'utf-8') + md = bytes(hmac.new(secret, data, digestmod=hashlib.sha1).digest().hex(), 'utf-8') + return md.decode("utf-8") + # No client / SHA1 / P=1 / URL not pristine / URL not altered. # -path="foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=" -to_sign="127.0.0.1:{}/".format(server.Variables.Port) + path -url="http://one.two.three/" + path + sign(to_sign, "dqsgopTSM_doT6iAysasQVUKaPykyb6e") +path = "foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=" +to_sign = "127.0.0.1:{}/".format(server.Variables.Port) + path +url = "http://one.two.three/" + path + sign(to_sign, "dqsgopTSM_doT6iAysasQVUKaPykyb6e") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -252,9 +252,9 @@ def sign(payload, key): # No client / SHA1 / P=1 / URL not pristine / URL not altered -- HTTPS. # -path="foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=" -to_sign="127.0.0.1:{}/".format(server.Variables.Port) + path -url="https://127.0.0.1:{}/".format(ts.Variables.ssl_port) + path + sign(to_sign, "dqsgopTSM_doT6iAysasQVUKaPykyb6e") +path = "foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=" +to_sign = "127.0.0.1:{}/".format(server.Variables.Port) + path +url = "https://127.0.0.1:{}/".format(ts.Variables.ssl_port) + path + sign(to_sign, "dqsgopTSM_doT6iAysasQVUKaPykyb6e") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 From 3fbc8e661c8280ecdfa35b7f5bfe9b21b2087898 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 19 Jul 2019 13:38:26 -0600 Subject: [PATCH 2/7] Removes the buffer_upload plugin. See #5395 --- doc/admin-guide/plugins/buffer_upload.en.rst | 82 -- doc/admin-guide/plugins/index.en.rst | 4 - .../admin-guide/plugins/buffer_upload.en.po | 135 -- .../admin-guide/plugins/index.en.po | 4 - plugins/Makefile.am | 1 - .../experimental/buffer_upload/Makefile.inc | 20 - plugins/experimental/buffer_upload/README | 77 - .../buffer_upload/buffer_upload.cc | 1251 ----------------- 8 files changed, 1574 deletions(-) delete mode 100644 doc/admin-guide/plugins/buffer_upload.en.rst delete mode 100644 doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po delete mode 100644 plugins/experimental/buffer_upload/Makefile.inc delete mode 100644 plugins/experimental/buffer_upload/README delete mode 100644 plugins/experimental/buffer_upload/buffer_upload.cc diff --git a/doc/admin-guide/plugins/buffer_upload.en.rst b/doc/admin-guide/plugins/buffer_upload.en.rst deleted file mode 100644 index 6f22d801420..00000000000 --- a/doc/admin-guide/plugins/buffer_upload.en.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. _admin-plugins-buffer-upload: - -Buffer Upload Plugin -******************** - -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -The Buffer Upload plugin offers the following features - -Installation -============ - -Configuration can be explicitly specified as a parameter in ``plugin.config`` :: - - buffer_upload.so /FOOBAR/upload.conf - -Memory buffering (buffer the entire POST data in IOBuffer before connecting to OS) -================================================================================== - -Memory buffer size is configured with "mem_buffer_size" in config file. Default and minimum value is 32K. You can -increase it in the config file. If the size of a request is larger than the "mem_buffer_size" value specified in the -config file, then the upload proxy feature will be disabled for this particular request - -Disk buffering (buffer the entire POST data on disk before connecting to OS) -============================================================================ - -1. Disk async IO is used. AIO api call only involves certain amount of threads. The number of threads is configurable in -plugin's config file (default is 4) - -2. Directories and files are generated on disk . Base directory is /FOOBAR/var/buffer_upload_tmp/ (configurable in -config file). Number of subdirectories is 64 (configurable in config file). Filename are randomly generated. Files will -be removed when the entire data have been sent out to OS . At startup time, dangling files are removed (left on disk due -to transaction interruption or traffic server crash) - -3. Default chunk size when reading from disk is 16K, configurable in config file - -Trigger POST buffering on certain URLs -====================================== - -1. Certain URLs will be provided in a plain text file (one URL each line) -2. Specify filename in config file by "url_list_file" -3. max length of each URL is 4096 (configurable in config file) -4. use exact match, don't support regex for now - -Other Features -============== - -1. Default buffering mode is disk aio buffering mode. To turn off disk buffering, add a "use_disk_buffer 0" line in -config file - -2. All request headers including cookies plus the entire POST data will be buffered (either in memory or on disk) - -Configuration File -================== - -sample config file :: - - use_disk_buffer 1 - convert_url 1 - chunk_size 1024 - url_list_file /tmp/url_list.conf - max_url_length 10000 - base_dir /tmp/test1 - subdir_num 100 - thread_num 10 - mem_buffer_size 40000 - diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index ff078b46c62..fecae9615f8 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -144,7 +144,6 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :hidden: Access Control - Buffer Upload Certifier Cert Reporting Tool Collapsed-Forwarding @@ -172,9 +171,6 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`Access Control ` Access control plugin that handles various access control use-cases. -:doc:`Buffer Upload ` - Buffers POST data before connecting to the Origin server. - :doc:`Certifier ` Manages and/or generates certificates for incoming HTTPS requests. diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po deleted file mode 100644 index b8074d755e8..00000000000 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po +++ /dev/null @@ -1,135 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: Apache Traffic Server 6.2\n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2016-01-02 21:32+0000\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"Language: ja_JP\n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=utf-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.1.1\n" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:42 -msgid "" -"1. Disk async IO is used. AIO api call only involves certain amount of " -"threads. The number of threads is configurable in plugin's config file " -"(default is 4)" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:4 -msgid "Buffer Upload Plugin" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:28 -msgid "" -"Configuration can be explicitly specified as a parameter in ``plugin." -"config`` ::" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:26 -msgid "Installation" -msgstr "インストール" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:23 -msgid "The Buffer Upload plugin offers the following features" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:63 -msgid "" -"1. Default buffering mode is disk aio buffering mode. To turn off disk " -"buffering, add a \"use_disk_buffer 0\" line in config file" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:45 -msgid "" -"2. Directories and files are generated on disk . Base directory is /FOOBAR/" -"var/buffer_upload_tmp/ (configurable in config file). Number of " -"subdirectories is 64 (configurable in config file). Filename are randomly " -"generated. Files will be removed when the entire data have been sent out to " -"OS . At startup time, dangling files are removed (left on disk due to " -"transaction interruption or traffic server crash)" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:55 -msgid "Certain URLs will be provided in a plain text file (one URL each line)" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:50 -msgid "" -"Default chunk size when reading from disk is 16K, configurable in config " -"file" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:35 -msgid "" -"Memory buffer size is configured with \"mem_buffer_size\" in config file. " -"Default and minimum value is 32K. You can increase it in the config file. " -"If the size of a request is larger than the \"mem_buffer_size\" value " -"specifiied in the config file, then the upload proxy feature will be " -"disabled for this particular request" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:33 -msgid "" -"Memory buffering (buffer the entire POST data in IOBuffer before connecting " -"to OS)" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:66 -msgid "" -"All request headers inlcuding cookies plus the entire POST data will be " -"buffered (either in memory or on disk)" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:40 -msgid "" -"Disk buffering (buffer the entire POST data on disk before connecting to OS)" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:69 -msgid "Configuration File" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:61 -msgid "Other Features" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:56 -msgid "Specify filename in config file by \"url_list_file\"" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:53 -msgid "Trigger POST buffering on certain URLs" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:57 -msgid "max length of each URL is 4096 (configurable in config file)" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:71 -msgid "sample config file ::" -msgstr "" - -#: ../../admin-guide/plugins/buffer_upload.en.rst:58 -msgid "use exact match, don't support regex for now" -msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po index 2eb3788f0ed..7f2da5180f1 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po @@ -182,10 +182,6 @@ msgstr "" msgid "Balances requests across multiple origin servers." msgstr "" -#: ../../../admin-guide/plugins/index.en.rst:138 -msgid ":doc:`Buffer Upload `" -msgstr "" - #: ../../../admin-guide/plugins/index.en.rst:138 msgid "Buffers POST data before connecting to the Origin server." msgstr "" diff --git a/plugins/Makefile.am b/plugins/Makefile.am index d868692e3c7..89cbdb2270f 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -58,7 +58,6 @@ if BUILD_EXPERIMENTAL_PLUGINS include experimental/access_control/Makefile.inc include experimental/acme/Makefile.inc -include experimental/buffer_upload/Makefile.inc include experimental/cert_reporting_tool/Makefile.inc include experimental/collapsed_forwarding/Makefile.inc include experimental/cookie_remap/Makefile.inc diff --git a/plugins/experimental/buffer_upload/Makefile.inc b/plugins/experimental/buffer_upload/Makefile.inc deleted file mode 100644 index ee820e22253..00000000000 --- a/plugins/experimental/buffer_upload/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pkglib_LTLIBRARIES += experimental/buffer_upload/buffer_upload.la - -experimental_buffer_upload_buffer_upload_la_SOURCES = \ - experimental/buffer_upload/buffer_upload.cc diff --git a/plugins/experimental/buffer_upload/README b/plugins/experimental/buffer_upload/README deleted file mode 100644 index 24d4b41e7d7..00000000000 --- a/plugins/experimental/buffer_upload/README +++ /dev/null @@ -1,77 +0,0 @@ -Version 0.3 (11/02/11) - - fixed some pointer comparisons in an attempt to prevent a crash where INKHandleStringRelease was called on a bad char* - -Version 0.2 (10/31/11) - - fixed lots of memory leaks - -Version 0.1 (04/20/10) - - initial version - -Upload proxy specs for phase I: - -1. Memory buffering (buffer the entire POST data in IOBuffer before connecting to OS) - 1.1. Memory buffer size is configured with "mem_buffer_size" in config file. Default and minimum value is 32K - You can increase it in the config file. If a request's size is larger than "mem_buffer_size" specified - in config file, then the upload proxy feature will be disabled for this particular request - -2. Disk buffering (buffer the entire POST data on disk before connecting to OS) - 2.1. Use disk async IO. This involved some changes in ATS core - . new APIs wrapping around ink_aio_read() and ink_aio_write() - . change to distinguish between api call's AIO and cache's AIO - . guarantee api call's AIO only involves certain amount of threads - . the number of threads is configurable in plugin's config file (default is 4) - - 2.2. Directories and files generated on disk - . base directory: FOOBAR/var/buffer_upload_tmp/ (configurable in config file) - . number of subdirectories: 64 (configurable in config file) - . filename are randomly generated - . files will be removed when the entire data have been sent out to OS - . remove dangling files (left on disk due to transaction interruption or traffic server crash) at startup time - - 2.3. Default chunk size when reading from disk: 16K, configurable in config file - -3. Default buffering mode: disk aio buffering mode - 3.1. to turn off disk buffering, add a "use_disk_buffer 0" line in config file - -4. Trigger POST buffering on certain URLs - 4.1. certain URLs will be provided in a plain text file (one URL each line) - 4.2. specify filename in config file by "url_list_file" - 4.3. max length of each URL: 4096 (configurable in config file) - 4.4. use exact match, don't support regex for now - -5. URL conversion for Mail's specific URL format - 5.1. for now check if the "host" part in the URL is same as the proxy server name, then will do this conversion - 5.2. To turn on URL conversion feature, set "convert_url 1" in config file - -6. All request headers including cookies plus the entire POST data will be buffered (either in memory or on disk) - -7. Config file can be explicitly specified as a parameter in command line (in plugin.config file) - - -a sample config file: - -use_disk_buffer 1 -convert_url 1 -chunk_size 1024 -url_list_file /tmp/url_list.conf -max_url_length 10000 -base_dir /tmp/test1 -subdir_num 100 -thread_num 10 -mem_buffer_size 40000 - - -default config file: FOOBAR/etc/upload.conf - -default config values: -use_disk_buffer 1 -convert_url 0 -chunk_size 16384 -url_list_file none -max_url_length 4096 -base_dir FOOBAR/var/buffer_upload_tmp -subdir_num 64 -thread_num 4 -mem_buffer_size 32768 - - diff --git a/plugins/experimental/buffer_upload/buffer_upload.cc b/plugins/experimental/buffer_upload/buffer_upload.cc deleted file mode 100644 index 92e64fd1fcd..00000000000 --- a/plugins/experimental/buffer_upload/buffer_upload.cc +++ /dev/null @@ -1,1251 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/* buffer_upload.c - plugin for buffering POST data on proxy server - * before connecting to origin server. It supports two types of buffering: - * memory-only buffering and disk buffering - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* #define DEBUG 1 */ -#define DEBUG_TAG "buffer_upload-dbg" - -/************************************************** - Log macros for error code return verification -**************************************************/ -#define PLUGIN_NAME "buffer_upload" -//#define LOG_SET_FUNCTION_NAME(NAME) const char * FUNCTION_NAME = NAME -#define LOG_ERROR(API_NAME) \ - { \ - TSError("[%s] %s %s %s File %s, line number %d", PLUGIN_NAME, API_NAME, "APIFAIL", __FUNCTION__, __FILE__, __LINE__); \ - } -#define LOG_ERROR_AND_RETURN(API_NAME) \ - { \ - LOG_ERROR(API_NAME); \ - return TS_ERROR; \ - } - -#define VALID_PTR(X) (NULL != X) -#define NOT_VALID_PTR(X) (NULL == X) - -struct upload_config_t { - bool use_disk_buffer; - bool convert_url; - int64_t mem_buffer_size; - int64_t chunk_size; - char *url_list_file; - int64_t max_url_length; - int url_num; - char **urls; - char *base_dir; - int subdir_num; - int thread_num; -}; - -using upload_config = struct upload_config_t; - -enum config_type { - TYPE_INT, - TYPE_UINT, - TYPE_LONG, - TYPE_ULONG, - TYPE_STRING, - TYPE_BOOL, -}; - -struct config_val_ul { - const char *str; - enum config_type type; - void *val; -}; - -static int upload_vc_count; - -static upload_config *uconfig = nullptr; - -struct pvc_state_t { - TSVConn p_vc; - TSVIO p_read_vio; - TSVIO p_write_vio; - - TSVConn net_vc; - TSVIO n_read_vio; - TSVIO n_write_vio; - - TSIOBuffer req_buffer; - TSIOBufferReader req_reader; - - TSIOBuffer resp_buffer; - TSIOBufferReader resp_reader; - - TSIOBufferReader req_hdr_reader; - TSIOBuffer req_hdr_buffer; - - TSMutex disk_io_mutex; - - int fd; - - int64_t req_finished; - int64_t resp_finished; - int64_t nbytes_to_consume; - int64_t req_size; - int64_t size_written; - int64_t size_read; - - int64_t write_offset; - int64_t read_offset; - - char *chunk_buffer; // buffer to store the data read from disk - int is_reading_from_disk; - - TSHttpTxn http_txnp; -}; - -using pvc_state = struct pvc_state_t; - -// print IOBuffer for test purpose -/* -static void -print_buffer(TSIOBufferReader reader) -{ - TSIOBufferBlock block; - int64_t size; - const char *ptr; - - block = TSIOBufferReaderStart(reader); - while (block != NULL) { - ptr = TSIOBufferBlockReadStart(block, reader, &size); - TSDebug(DEBUG_TAG, "buffer size: %d", size); - TSDebug(DEBUG_TAG, "buffer: %.*s", size, ptr); - block = TSIOBufferBlockNext(block); - } -} -*/ - -static int -write_buffer_to_disk(TSIOBufferReader reader, pvc_state *my_state, TSCont contp) -{ - TSIOBufferBlock block; - int64_t size; - const char *ptr; - char *pBuf; - - // LOG_SET_FUNCTION_NAME("write_buffer_to_disk"); - block = TSIOBufferReaderStart(reader); - while (block != nullptr) { - ptr = TSIOBufferBlockReadStart(block, reader, &size); - pBuf = (char *)TSmalloc(sizeof(char) * size); - if (pBuf == nullptr) { - LOG_ERROR_AND_RETURN("TSAIOWrite"); - } - memcpy(pBuf, ptr, size); - if (TSAIOWrite(my_state->fd, my_state->write_offset, pBuf, size, contp) == TS_ERROR) { - LOG_ERROR_AND_RETURN("TSAIOWrite"); - } - my_state->write_offset += size; - block = TSIOBufferBlockNext(block); - } - return TS_SUCCESS; -} - -static int -call_httpconnect(TSCont contp, pvc_state *my_state) -{ - // LOG_SET_FUNCTION_NAME("call_httpconnect"); - - // unsigned int client_ip = TSHttpTxnClientIPGet(my_state->http_txnp); - sockaddr const *client_ip = TSHttpTxnClientAddrGet(my_state->http_txnp); - - TSDebug(DEBUG_TAG, "call TSHttpConnect()"); - if ((my_state->net_vc = TSHttpConnect(client_ip)) == nullptr) { - LOG_ERROR_AND_RETURN("TSHttpConnect"); - } - my_state->p_write_vio = TSVConnWrite(my_state->p_vc, contp, my_state->resp_reader, INT_MAX); - if (my_state->p_write_vio == nullptr) { - LOG_ERROR_AND_RETURN("TSVConnWrite"); - } - my_state->n_read_vio = TSVConnRead(my_state->net_vc, contp, my_state->resp_buffer, INT_MAX); - if (my_state->n_read_vio == nullptr) { - LOG_ERROR_AND_RETURN("TSVConnRead"); - } - my_state->n_write_vio = TSVConnWrite(my_state->net_vc, contp, my_state->req_reader, INT_MAX); - if (my_state->n_write_vio == nullptr) { - LOG_ERROR_AND_RETURN("TSVConnWrite"); - } - return TS_SUCCESS; -} - -static void -pvc_cleanup(TSCont contp, pvc_state *my_state) -{ - if (my_state->req_buffer) { - TSIOBufferReaderFree(my_state->req_reader); - my_state->req_reader = nullptr; - TSIOBufferDestroy(my_state->req_buffer); - my_state->req_buffer = nullptr; - } - - if (my_state->resp_buffer) { - TSIOBufferReaderFree(my_state->resp_reader); - my_state->resp_reader = nullptr; - TSIOBufferDestroy(my_state->resp_buffer); - my_state->resp_buffer = nullptr; - } - - if (my_state->req_hdr_buffer) { - TSIOBufferReaderFree(my_state->req_hdr_reader); - my_state->req_hdr_reader = nullptr; - TSIOBufferDestroy(my_state->req_hdr_buffer); - my_state->req_hdr_buffer = nullptr; - } - - if (uconfig->use_disk_buffer && my_state->fd != -1) { - close(my_state->fd); - my_state->fd = -1; - } - - if (my_state->chunk_buffer) { - TSfree(my_state->chunk_buffer); - my_state->chunk_buffer = nullptr; - } - - TSfree(my_state); - TSContDestroy(contp); - - /* Decrement upload_vc_count */ - TSStatIntDecrement(upload_vc_count, 1); -} - -static void -pvc_check_done(TSCont contp, pvc_state *my_state) -{ - if (my_state->req_finished && my_state->resp_finished) { - TSVConnClose(my_state->p_vc); - TSVConnClose(my_state->net_vc); - pvc_cleanup(contp, my_state); - } -} - -static void -pvc_process_accept(TSCont contp, int event, void *edata, pvc_state *my_state) -{ - TSDebug(DEBUG_TAG, "plugin called: pvc_process_accept with event %d", event); - - if (event == TS_EVENT_NET_ACCEPT) { - my_state->p_vc = (TSVConn)edata; - - my_state->req_buffer = TSIOBufferCreate(); - my_state->req_reader = TSIOBufferReaderAlloc(my_state->req_buffer); - // set the maximum memory buffer size for request (both request header and post data), default is 32K - // only apply to memory buffer mode - if (uconfig->use_disk_buffer == 0) { - TSIOBufferWaterMarkSet(my_state->req_buffer, uconfig->mem_buffer_size); - } - my_state->resp_buffer = TSIOBufferCreate(); - my_state->resp_reader = TSIOBufferReaderAlloc(my_state->resp_buffer); - - if ((my_state->req_reader == nullptr) || (my_state->resp_reader == nullptr)) { - LOG_ERROR("TSIOBufferReaderAlloc"); - TSVConnClose(my_state->p_vc); - pvc_cleanup(contp, my_state); - } else { - my_state->p_read_vio = TSVConnRead(my_state->p_vc, contp, my_state->req_buffer, INT_MAX); - if (my_state->p_read_vio == nullptr) { - LOG_ERROR("TSVConnRead"); - } - } - } else if (event == TS_EVENT_NET_ACCEPT_FAILED) { - pvc_cleanup(contp, my_state); - } else { - TSReleaseAssert(!"Unexpected Event"); - } -} - -static void -pvc_process_p_read(TSCont contp, TSEvent event, pvc_state *my_state) -{ - int size, consume_size; - - // TSDebug(DEBUG_TAG, "plugin called: pvc_process_p_read with event %d", event); - - switch (event) { - case TS_EVENT_VCONN_READ_READY: - // Here we need to replace the server request header with client request header - // print_buffer(my_state->req_reader); - if (my_state->nbytes_to_consume == -1) { // -1 is the initial value - my_state->nbytes_to_consume = TSHttpTxnServerReqHdrBytesGet(my_state->http_txnp); - } - size = TSIOBufferReaderAvail(my_state->req_reader); - if (my_state->nbytes_to_consume > 0) { - consume_size = (my_state->nbytes_to_consume < size) ? my_state->nbytes_to_consume : size; - TSIOBufferReaderConsume(my_state->req_reader, consume_size); - my_state->nbytes_to_consume -= consume_size; - size -= consume_size; - } - if (my_state->nbytes_to_consume == 0) { // the entire server request header has been consumed - if (uconfig->use_disk_buffer) { - TSMutexLock(my_state->disk_io_mutex); - if (write_buffer_to_disk(my_state->req_hdr_reader, my_state, contp) == TS_ERROR) { - LOG_ERROR("write_buffer_to_disk"); - uconfig->use_disk_buffer = false; - close(my_state->fd); - my_state->fd = -1; - } - TSMutexUnlock(my_state->disk_io_mutex); - } - if (size > 0) { - if (uconfig->use_disk_buffer) { - TSMutexLock(my_state->disk_io_mutex); - if (write_buffer_to_disk(my_state->req_reader, my_state, contp) == TS_ERROR) { - TSDebug(DEBUG_TAG, "Error in writing to disk"); - } - TSMutexUnlock(my_state->disk_io_mutex); - } else { - // never get chance to test this line, didn't get a test case to fall into this situation - TSIOBufferCopy(my_state->req_hdr_buffer, my_state->req_reader, size, 0); - } - TSIOBufferReaderConsume(my_state->req_reader, size); - } - if (!uconfig->use_disk_buffer) { - size = TSIOBufferReaderAvail(my_state->req_hdr_reader); - TSIOBufferCopy(my_state->req_buffer, my_state->req_hdr_reader, size, 0); - } - my_state->nbytes_to_consume = -2; // -2 indicates the header replacement is done - } - if (my_state->nbytes_to_consume == -2) { - size = TSIOBufferReaderAvail(my_state->req_reader); - if (uconfig->use_disk_buffer) { - if (size > 0) { - TSMutexLock(my_state->disk_io_mutex); - if (write_buffer_to_disk(my_state->req_reader, my_state, contp) == TS_ERROR) { - TSDebug(DEBUG_TAG, "Error in writing to disk"); - } - TSIOBufferReaderConsume(my_state->req_reader, size); - TSMutexUnlock(my_state->disk_io_mutex); - } - } else { - // if the entire post data had been read in memory, then connect to origin server. - if (size >= my_state->req_size) { - if (call_httpconnect(contp, my_state) == TS_ERROR) { - LOG_ERROR("call_httpconnect"); - } - } - } - } - - break; - case TS_EVENT_VCONN_READ_COMPLETE: - case TS_EVENT_VCONN_EOS: - case TS_EVENT_ERROR: { - /* We're finished reading from the plugin vc */ - int ndone; - - ndone = TSVIONDoneGet(my_state->p_read_vio); - if (ndone == TS_ERROR) { - LOG_ERROR("TSVIODoneGet"); - } - - my_state->p_read_vio = nullptr; - - TSVConnShutdown(my_state->p_vc, 1, 0); - // if client aborted the uploading in middle, need to cleanup the file from disk - if (event == TS_EVENT_VCONN_EOS && uconfig->use_disk_buffer && my_state->fd != -1) { - close(my_state->fd); - my_state->fd = -1; - } - - break; - } - default: - TSReleaseAssert(!"Unexpected Event"); - break; - } -} - -static void -pvc_process_n_write(TSCont contp, TSEvent event, pvc_state *my_state) -{ - int size; - - // TSDebug(DEBUG_TAG, "plugin called: pvc_process_n_write with event %d", event); - - switch (event) { - case TS_EVENT_VCONN_WRITE_READY: - // print_buffer(my_state->req_reader); - if (uconfig->use_disk_buffer) { - TSMutexLock(my_state->disk_io_mutex); - size = (my_state->req_size - my_state->read_offset) > uconfig->chunk_size ? uconfig->chunk_size : - (my_state->req_size - my_state->read_offset); - if (size > 0 && !my_state->is_reading_from_disk) { - my_state->is_reading_from_disk = 1; - TSAIORead(my_state->fd, my_state->read_offset, my_state->chunk_buffer, size, contp); - my_state->read_offset += size; - } - TSMutexUnlock(my_state->disk_io_mutex); - } - break; - case TS_EVENT_ERROR: - if (my_state->p_read_vio) { - TSVConnShutdown(my_state->p_vc, 1, 0); - my_state->p_read_vio = nullptr; - } - /* FALL THROUGH */ - case TS_EVENT_VCONN_WRITE_COMPLETE: - /* We should have already shutdown read pvc side */ - TSAssert(my_state->p_read_vio == nullptr); - TSVConnShutdown(my_state->net_vc, 0, 1); - my_state->req_finished = 1; - - if (uconfig->use_disk_buffer && my_state->fd != -1) { - close(my_state->fd); - my_state->fd = -1; - } - pvc_check_done(contp, my_state); - break; - - default: - TSReleaseAssert(!"Unexpected Event"); - break; - } -} - -static void -pvc_process_n_read(TSCont contp, TSEvent event, pvc_state *my_state) -{ - // TSDebug(DEBUG_TAG, "plugin called: pvc_process_n_read with event %d", event); - - switch (event) { - case TS_EVENT_VCONN_READ_READY: - // print_buffer(my_state->resp_reader); - TSVIOReenable(my_state->p_write_vio); - break; - case TS_EVENT_VCONN_READ_COMPLETE: - case TS_EVENT_VCONN_EOS: - case TS_EVENT_ERROR: { - /* We're finished reading from the plugin vc */ - int ndone; - int todo; - - ndone = TSVIONDoneGet(my_state->n_read_vio); - if (ndone == TS_ERROR) { - LOG_ERROR("TSVIODoneGet"); - } - - my_state->n_read_vio = nullptr; - TSVIONBytesSet(my_state->p_write_vio, ndone); - TSVConnShutdown(my_state->net_vc, 1, 0); - - todo = TSVIONTodoGet(my_state->p_write_vio); - if (todo == TS_ERROR) { - LOG_ERROR("TSVIOTodoGet"); - /* Error so set it to 0 to cleanup */ - todo = 0; - } - - if (todo == 0) { - my_state->resp_finished = 1; - TSVConnShutdown(my_state->p_vc, 0, 1); - pvc_check_done(contp, my_state); - } else { - TSVIOReenable(my_state->p_write_vio); - } - - break; - } - default: - TSReleaseAssert(!"Unexpected Event"); - break; - } -} - -static void -pvc_process_p_write(TSCont contp, TSEvent event, pvc_state *my_state) -{ - // TSDebug(DEBUG_TAG, "plugin called: pvc_process_p_write with event %d", event); - - switch (event) { - case TS_EVENT_VCONN_WRITE_READY: - if (my_state->n_read_vio) { - TSVIOReenable(my_state->n_read_vio); - } - break; - case TS_EVENT_ERROR: - if (my_state->n_read_vio) { - TSVConnShutdown(my_state->net_vc, 1, 0); - my_state->n_read_vio = nullptr; - } - /* FALL THROUGH */ - case TS_EVENT_VCONN_WRITE_COMPLETE: - /* We should have already shutdown read net side */ - TSAssert(my_state->n_read_vio == nullptr); - TSVConnShutdown(my_state->p_vc, 0, 1); - my_state->resp_finished = 1; - pvc_check_done(contp, my_state); - break; - default: - TSReleaseAssert(!"Unexpected Event"); - break; - } -} - -static int -pvc_plugin(TSCont contp, TSEvent event, void *edata) -{ - pvc_state *my_state = static_cast(TSContDataGet(contp)); - TSAIOCallback callback = static_cast(edata); - - if (my_state == nullptr) { - TSReleaseAssert(!"Unexpected: my_state is NULL"); - return 0; - } - - if (event == TS_EVENT_NET_ACCEPT || event == TS_EVENT_NET_ACCEPT_FAILED) { - pvc_process_accept(contp, event, edata, my_state); - } else if (edata == my_state->p_read_vio) { - pvc_process_p_read(contp, event, my_state); - } else if (edata == my_state->p_write_vio) { - pvc_process_p_write(contp, event, my_state); - } else if (edata == my_state->n_read_vio) { - pvc_process_n_read(contp, event, my_state); - } else if (edata == my_state->n_write_vio) { - pvc_process_n_write(contp, event, my_state); - } else if (event == TS_EVENT_AIO_DONE && uconfig->use_disk_buffer) { - TSMutexLock(my_state->disk_io_mutex); - int size = TSAIONBytesGet(callback); - char *buf = TSAIOBufGet(callback); - if (buf != my_state->chunk_buffer) { - // this TS_EVENT_AIO_DONE event is from TSAIOWrite() - TSDebug(DEBUG_TAG, "aio write size: %d", size); - my_state->size_written += size; - if (buf != nullptr) { - TSfree(buf); - } - if (my_state->size_written >= my_state->req_size) { - // the entire post data had been written to disk already, make the connection now - if (call_httpconnect(contp, my_state) == TS_ERROR) { - TSDebug(DEBUG_TAG, "call_httpconnect"); - } - } - } else { - // this TS_EVENT_AIO_DONE event is from TSAIORead() - TSDebug(DEBUG_TAG, "aio read size: %d", size); - TSIOBufferWrite(my_state->req_buffer, my_state->chunk_buffer, size); - my_state->size_read += size; - if (my_state->size_read >= my_state->req_size && my_state->fd != -1) { - close(my_state->fd); - my_state->fd = -1; - } - my_state->is_reading_from_disk = 0; - TSVIOReenable(my_state->n_write_vio); - } - TSMutexUnlock(my_state->disk_io_mutex); - - } else { - TSDebug(DEBUG_TAG, "event: %d", event); - TSReleaseAssert(!"Unexpected Event"); - } - - return 0; -} - -/* - * Convert specific URL format - */ -static void -convert_url_func(TSMBuffer req_bufp, TSMLoc req_loc) -{ - TSMLoc url_loc; - TSMLoc field_loc; - const char *str; - int len, port; - - if (TSHttpHdrUrlGet(req_bufp, req_loc, &url_loc) == TS_ERROR) { - return; - } - - const char *hostname = getenv("HOSTNAME"); - if (hostname == nullptr) { - return; - } - - // in reverse proxy mode, TSUrlHostGet returns NULL here - str = TSUrlHostGet(req_bufp, url_loc, &len); - - port = TSUrlPortGet(req_bufp, url_loc); - - // for now we assume the in the format is the hostname - // but this needs to be verified later - if ((NOT_VALID_PTR(str) || !strncmp(str, hostname, len)) && strlen(hostname) == (size_t)len) { - const char *slash; - const char *colon; - // if (VALID_PTR(str)) - // TSHandleStringRelease(req_bufp, url_loc, str); - str = TSUrlPathGet(req_bufp, url_loc, &len); - slash = strstr(str, "/"); - if (slash == nullptr) { - // if (VALID_PTR(str)) - // TSHandleStringRelease(req_bufp, url_loc, str); - TSHandleMLocRelease(req_bufp, req_loc, url_loc); - return; - } - char pathTmp[len + 1]; - memcpy(pathTmp, str, len); - pathTmp[len] = '\0'; - TSDebug(DEBUG_TAG, "convert_url_func working on path: %s", pathTmp); - colon = strstr(str, ":"); - if (colon != nullptr && colon < slash) { - char *port_str = (char *)TSmalloc(sizeof(char) * (slash - colon)); - strncpy(port_str, colon + 1, slash - colon - 1); - port_str[slash - colon - 1] = '\0'; - TSUrlPortSet(req_bufp, url_loc, atoi(port_str)); - TSfree(port_str); - } else { - int length = 0; - const char *scheme = TSUrlSchemeGet(req_bufp, url_loc, &length); - - if ((length == TS_URL_LEN_HTTP && strncmp(TS_URL_SCHEME_HTTP, scheme, length) == 0 && port != 80) || - (length == TS_URL_LEN_HTTPS && strncmp(TS_URL_SCHEME_HTTPS, scheme, length) == 0 && port != 443)) { - TSUrlPortSet(req_bufp, url_loc, port); - } - colon = slash; - } - - TSUrlHostSet(req_bufp, url_loc, str, colon - str); - TSUrlPathSet(req_bufp, url_loc, slash + 1, len - (slash - str) - 1); - if ((field_loc = TSMimeHdrFieldFind(req_bufp, req_loc, TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST)) != TS_NULL_MLOC && - field_loc != nullptr) { - TSMimeHdrFieldValueStringSet(req_bufp, req_loc, field_loc, 0, str, slash - str); - TSHandleMLocRelease(req_bufp, req_loc, field_loc); - } - } else { - // if (VALID_PTR(str)) - // TSHandleStringRelease(req_bufp, url_loc, str); - } - - TSHandleMLocRelease(req_bufp, req_loc, url_loc); -} - -static int -attach_pvc_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata) -{ - TSHttpTxn txnp = (TSHttpTxn)edata; - TSMutex mutex; - TSCont new_cont; - pvc_state *my_state; - TSMBuffer req_bufp; - TSMLoc req_loc; - TSMLoc field_loc; - TSMLoc url_loc; - char *url; - int url_len; - int content_length = 0; - const char *method; - int method_len; - const char *host_str; - int host_str_len; - const char *host_hdr_str_val; - int host_hdr_str_val_len; - - TSDebug(DEBUG_TAG, "inside attach_pvc_plugin"); - switch (event) { - case TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP: - - // if the request is issued by the TSHttpConnect() in this plugin, don't get in the endless cycle. - if (TSHttpTxnIsInternal(txnp)) { - TSDebug(DEBUG_TAG, "internal request"); - break; - } - - if (TSHttpTxnClientReqGet(txnp, &req_bufp, &req_loc) == TS_ERROR) { - LOG_ERROR("Error while retrieving client request header"); - break; - } - - method = TSHttpHdrMethodGet(req_bufp, req_loc, &method_len); - TSDebug(DEBUG_TAG, "inside handler"); - - if (NOT_VALID_PTR(method) || method_len == 0) { - TSDebug(DEBUG_TAG, "invalid method"); - - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - break; - } - // only deal with POST method - TSDebug(DEBUG_TAG, "method: %s", method); - - if (static_cast(method_len) != strlen(TS_HTTP_METHOD_POST) || - strncasecmp(method, TS_HTTP_METHOD_POST, method_len) != 0) { - TSDebug(DEBUG_TAG, "Not POST method"); - - // TSHandleStringRelease(req_bufp, req_loc, method); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - break; - } - - // TSHandleStringRelease(req_bufp, req_loc, method); - - TSDebug(DEBUG_TAG, "Got POST req"); - if (uconfig->url_list_file != nullptr) { - TSDebug(DEBUG_TAG, "url_list_file != NULL"); - // check against URL list - if (TSHttpHdrUrlGet(req_bufp, req_loc, &url_loc) == TS_ERROR) { - LOG_ERROR("Couldn't get the url"); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - break; - } - host_str = TSUrlHostGet(req_bufp, url_loc, &host_str_len); - if (NOT_VALID_PTR(host_str) || host_str_len <= 0) { - // reverse proxy mode - field_loc = TSMimeHdrFieldFind(req_bufp, req_loc, TS_MIME_FIELD_HOST, -1); - if (NOT_VALID_PTR(field_loc)) { - // if (VALID_PTR(str)) - // TSHandleStringRelease(req_bufp, url_loc, str); - LOG_ERROR("Host field not found"); - TSHandleMLocRelease(req_bufp, req_loc, url_loc); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - break; - } - host_hdr_str_val = TSMimeHdrFieldValueStringGet(req_bufp, req_loc, field_loc, -1, &host_hdr_str_val_len); - if (NOT_VALID_PTR(host_hdr_str_val) || host_hdr_str_val_len <= 0) { - // if (VALID_PTR(str)) - // TSHandleStringRelease(req_bufp, field_loc, str); - TSHandleMLocRelease(req_bufp, req_loc, field_loc); - TSHandleMLocRelease(req_bufp, req_loc, url_loc); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - break; - } - - char replacement_host_str[host_hdr_str_val_len + 1]; - memcpy(replacement_host_str, host_hdr_str_val, host_hdr_str_val_len); - replacement_host_str[host_hdr_str_val_len] = '\0'; - TSDebug(DEBUG_TAG, "Adding host to request url: %s", replacement_host_str); - - const char *colon = strchr(replacement_host_str, ':'); - if (colon != nullptr && colon[1] != '\0') { - int length = 0; - const char *scheme = TSUrlSchemeGet(req_bufp, url_loc, &length); - int port_str_val = atoi(colon + 1); - - if ((length == TS_URL_LEN_HTTP && strncmp(TS_URL_SCHEME_HTTP, scheme, length) == 0 && port_str_val != 80) || - (length == TS_URL_LEN_HTTPS && strncmp(TS_URL_SCHEME_HTTPS, scheme, length) == 0 && port_str_val != 443)) { - TSUrlPortSet(req_bufp, url_loc, port_str_val); - } - host_hdr_str_val_len = colon - replacement_host_str; - } - TSUrlHostSet(req_bufp, url_loc, host_hdr_str_val, host_hdr_str_val_len); - - // TSHandleStringRelease(req_bufp, field_loc, str); - TSHandleMLocRelease(req_bufp, req_loc, field_loc); - } else { - // TSHandleStringRelease(req_bufp, url_loc, str); - } - - int i = uconfig->url_num; - url = TSUrlStringGet(req_bufp, url_loc, &url_len); - if (VALID_PTR(url)) { - char urlStr[url_len + 1]; - memcpy(urlStr, url, url_len); - urlStr[url_len] = '\0'; - TSDebug(DEBUG_TAG, "Request url: %s", urlStr); - - for (i = 0; i < uconfig->url_num; i++) { - TSDebug(DEBUG_TAG, "uconfig url: %s", uconfig->urls[i]); - if (strncmp(url, uconfig->urls[i], url_len) == 0) { - break; - } - } - - TSfree(url); - } - TSHandleMLocRelease(req_bufp, req_loc, url_loc); - - if (uconfig->url_num > 0 && i == uconfig->url_num) { - TSDebug(DEBUG_TAG, "breaking: url_num > 0 and i== url_num, URL match not found"); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - break; - } - } - - if (uconfig->convert_url) { - TSDebug(DEBUG_TAG, "doing convert url"); - convert_url_func(req_bufp, req_loc); - } - - field_loc = TSMimeHdrFieldFind(req_bufp, req_loc, TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH); - if (field_loc == nullptr) { - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - LOG_ERROR("TSMimeHdrFieldRetrieve"); - break; - } - - content_length = TSMimeHdrFieldValueIntGet(req_bufp, req_loc, field_loc, 0); - /*{ - TSHandleMLocRelease(req_bufp, req_loc, field_loc); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - LOG_ERROR("TSMimeFieldValueGet"); -} else - */ - // content_length = value; - - mutex = TSMutexCreate(); - if (NOT_VALID_PTR(mutex)) { - TSHandleMLocRelease(req_bufp, req_loc, field_loc); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - LOG_ERROR("TSMutexCreate"); - break; - } - - new_cont = TSContCreate(pvc_plugin, mutex); - if (NOT_VALID_PTR(new_cont)) { - TSHandleMLocRelease(req_bufp, req_loc, field_loc); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - LOG_ERROR("TSContCreate"); - break; - } - - my_state = (pvc_state *)TSmalloc(sizeof(pvc_state)); - my_state->req_size = content_length; - my_state->p_vc = nullptr; - my_state->p_read_vio = nullptr; - my_state->p_write_vio = nullptr; - - my_state->net_vc = nullptr; - my_state->n_read_vio = nullptr; - my_state->n_write_vio = nullptr; - - my_state->req_buffer = nullptr; - my_state->req_reader = nullptr; - my_state->resp_buffer = nullptr; - my_state->resp_reader = nullptr; - my_state->fd = -1; - my_state->disk_io_mutex = nullptr; - - my_state->http_txnp = txnp; // not in use now, may need in the future - - my_state->req_finished = 0; - my_state->resp_finished = 0; - my_state->nbytes_to_consume = - -1; // the length of server request header to remove from incoming stream (will replace with client request header) - - my_state->size_written = 0; - my_state->size_read = 0; - my_state->write_offset = 0; - my_state->read_offset = 0; - my_state->is_reading_from_disk = 0; - - my_state->chunk_buffer = (char *)TSmalloc(sizeof(char) * uconfig->chunk_size); - - my_state->disk_io_mutex = TSMutexCreate(); - if (NOT_VALID_PTR(my_state->disk_io_mutex)) { - LOG_ERROR("TSMutexCreate"); - } - - my_state->req_hdr_buffer = TSIOBufferCreate(); - my_state->req_hdr_reader = TSIOBufferReaderAlloc(my_state->req_hdr_buffer); - TSHttpHdrPrint(req_bufp, req_loc, my_state->req_hdr_buffer); - // print_buffer(my_state->req_hdr_reader); - - my_state->req_size += TSIOBufferReaderAvail(my_state->req_hdr_reader); - - /* Increment upload_vc_count */ - TSStatIntIncrement(upload_vc_count, 1); - - if (!uconfig->use_disk_buffer && my_state->req_size > uconfig->mem_buffer_size) { - TSDebug(DEBUG_TAG, - "The request size %" PRId64 " is larger than memory buffer size %" PRId64 - ", bypass upload proxy feature for this request", - my_state->req_size, uconfig->mem_buffer_size); - - pvc_cleanup(new_cont, my_state); - TSHandleMLocRelease(req_bufp, req_loc, field_loc); - TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); - break; - } - - TSContDataSet(new_cont, my_state); - - if (uconfig->use_disk_buffer) { - char path[500]; - // coverity[dont_call] - int index = (int)(random() % uconfig->subdir_num); - - sprintf(path, "%s/%02X/tmp-XXXXXX", uconfig->base_dir, index); - - my_state->fd = mkstemp(path); - unlink(path); - if (my_state->fd < 0) { - LOG_ERROR("open"); - uconfig->use_disk_buffer = false; - my_state->fd = -1; - } else { - TSDebug(DEBUG_TAG, "temp filename: %s", path); - } - } - - TSDebug(DEBUG_TAG, "calling TSHttpTxnIntercept()"); - TSHttpTxnIntercept(new_cont, txnp); - - break; - default: - TSReleaseAssert(!"Unexpected Event"); - break; - } - - TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); - return 0; -} - -static int -create_directory() -{ - char str[10]; - char cwd[4096]; - int i; - DIR *dir; - struct dirent *d; - - if (getcwd(cwd, 4096) == nullptr) { - TSError("[%s] getcwd fails", PLUGIN_NAME); - return 0; - } - - if (chdir(uconfig->base_dir) < 0) { - if (mkdir(uconfig->base_dir, S_IRWXU | S_IRWXG | S_IRWXO) < 0) { - TSError("[%s] Unable to enter or create %s", PLUGIN_NAME, uconfig->base_dir); - goto error_out; - } - if (chdir(uconfig->base_dir) < 0) { - TSError("[%s] Unable enter %s", PLUGIN_NAME, uconfig->base_dir); - goto error_out; - } - } - for (i = 0; i < uconfig->subdir_num; i++) { - snprintf(str, 10, "%02X", i); - if (chdir(str) < 0) { - if (mkdir(str, S_IRWXU | S_IRWXG | S_IRWXO) < 0) { - TSError("[%s] Unable to enter or create %s/%s", PLUGIN_NAME, uconfig->base_dir, str); - goto error_out; - } - if (chdir(str) < 0) { - TSError("[%s] Unable to enter %s/%s", PLUGIN_NAME, uconfig->base_dir, str); - goto error_out; - } - } - dir = opendir("."); - if (dir == nullptr) { - goto error_out; - } - while ((d = readdir(dir))) { - if (remove(d->d_name) < 0) { - TSError("[%s] Unable to remove '%s': %s", PLUGIN_NAME, d->d_name, strerror(errno)); - closedir(dir); - goto error_out; - } - } - closedir(dir); - if (chdir("..") == -1) { - return 0; - } - } - - if (chdir(cwd) == -1) { - return 0; - } - - return 1; - -error_out: - /* Debian's compiler chain complains about not using the return - value of chdir() and cannot be silenced - The reason is the combination of -D_FORTIFY_SOURCE=2 -O - */ - if (chdir(cwd) == -1) { - return 0; - } - - return 0; -} - -static void -load_urls(char *filename) -{ - TSFile file; - char *url_buf; - char *eol; - int i; - - url_buf = (char *)TSmalloc(sizeof(char) * (uconfig->max_url_length + 1)); - url_buf[uconfig->max_url_length] = '\0'; - - for (i = 0; i < 2; i++) { - if ((file = TSfopen(filename, "r")) == nullptr) { - TSfree(url_buf); - TSError("[%s] Fail to open %s", PLUGIN_NAME, filename); - return; - } - if (i == 0) { // first round - uconfig->url_num = 0; - while (TSfgets(file, url_buf, uconfig->max_url_length) != nullptr) { - uconfig->url_num++; - } - uconfig->urls = (char **)TSmalloc(sizeof(char *) * uconfig->url_num); - } else { // second round - int idx = 0; - while (TSfgets(file, url_buf, uconfig->max_url_length) != nullptr && idx < uconfig->url_num) { - if ((eol = strstr(url_buf, "\r\n")) != nullptr) { - /* To handle newlines on Windows */ - *eol = '\0'; - } else if ((eol = strchr(url_buf, '\n')) != nullptr) { - *eol = '\0'; - } else { - /* Not a valid line, skip it */ - continue; - } - uconfig->urls[idx] = TSstrdup(url_buf); - idx++; - } - uconfig->url_num = idx; - } - TSfclose(file); - } - TSfree(url_buf); -} - -void -parse_config_line(char *line, const struct config_val_ul *cv) -{ - const char *delim = "\t\r\n "; - char *save = nullptr; - char *tok = strtok_r(line, delim, &save); - - while (tok && cv->str) { - if (!strcmp(tok, cv->str)) { - tok = strtok_r(nullptr, delim, &save); - if (tok) { - switch (cv->type) { - case TYPE_INT: { - char *end = tok; - int iv = strtol(tok, &end, 10); - if (end && *end == '\0') { - *((int *)cv->val) = iv; - TSError("[%s] Parsed int config value %s : %d", PLUGIN_NAME, cv->str, iv); - TSDebug(DEBUG_TAG, "Parsed int config value %s : %d", cv->str, iv); - } - break; - } - case TYPE_UINT: { - char *end = tok; - unsigned int uiv = strtoul(tok, &end, 10); - if (end && *end == '\0') { - *((unsigned int *)cv->val) = uiv; - TSError("[%s] Parsed uint config value %s : %u", PLUGIN_NAME, cv->str, uiv); - TSDebug(DEBUG_TAG, "Parsed uint config value %s : %u", cv->str, uiv); - } - break; - } - case TYPE_LONG: { - char *end = tok; - long lv = strtol(tok, &end, 10); - if (end && *end == '\0') { - *((long *)cv->val) = lv; - TSError("[%s] Parsed long config value %s : %ld", PLUGIN_NAME, cv->str, lv); - TSDebug(DEBUG_TAG, "Parsed long config value %s : %ld", cv->str, lv); - } - break; - } - case TYPE_ULONG: { - char *end = tok; - unsigned long ulv = strtoul(tok, &end, 10); - if (end && *end == '\0') { - *((unsigned long *)cv->val) = ulv; - TSError("[%s] Parsed ulong config value %s : %lu", PLUGIN_NAME, cv->str, ulv); - TSDebug(DEBUG_TAG, "Parsed ulong config value %s : %lu", cv->str, ulv); - } - break; - } - case TYPE_STRING: { - size_t len = strlen(tok); - if (len > 0) { - *((char **)cv->val) = (char *)TSmalloc(len + 1); - strcpy(*((char **)cv->val), tok); - TSError("[%s] Parsed string config value %s : %s", PLUGIN_NAME, cv->str, tok); - TSDebug(DEBUG_TAG, "Parsed string config value %s : %s", cv->str, tok); - } - break; - } - case TYPE_BOOL: { - size_t len = strlen(tok); - if (len > 0) { - if (*tok == '1' || *tok == 't') { - *((bool *)cv->val) = true; - } else { - *((bool *)cv->val) = false; - } - TSError("[%s] Parsed bool config value %s : %d", PLUGIN_NAME, cv->str, *((bool *)cv->val)); - TSDebug(DEBUG_TAG, "Parsed bool config value %s : %d", cv->str, *((bool *)cv->val)); - } - break; - } - default: - break; - } - } - } - cv++; - } -} - -bool -read_upload_config(const char *file_name) -{ - TSDebug(DEBUG_TAG, "read_upload_config: %s", file_name); - uconfig = (upload_config *)TSmalloc(sizeof(upload_config)); - uconfig->use_disk_buffer = true; - uconfig->convert_url = false; - uconfig->chunk_size = 16 * 1024; - uconfig->mem_buffer_size = 32 * 1024; - uconfig->url_list_file = nullptr; - uconfig->max_url_length = 4096; - uconfig->url_num = 0; - uconfig->urls = nullptr; - uconfig->base_dir = nullptr; - uconfig->subdir_num = 64; - uconfig->thread_num = 4; - - struct config_val_ul config_vals[] = {{"use_disk_buffer", TYPE_BOOL, &(uconfig->use_disk_buffer)}, - {"convert_url", TYPE_BOOL, &(uconfig->convert_url)}, - {"chunk_size", TYPE_ULONG, &(uconfig->chunk_size)}, - {"mem_buffer_size", TYPE_ULONG, &(uconfig->mem_buffer_size)}, - {"url_list_file", TYPE_STRING, &(uconfig->url_list_file)}, - {"max_url_length", TYPE_ULONG, &(uconfig->max_url_length)}, - {"base_dir", TYPE_STRING, &(uconfig->base_dir)}, - {"subdir_num", TYPE_UINT, &(uconfig->subdir_num)}, - {"thread_num", TYPE_UINT, &(uconfig->thread_num)}, - {nullptr, TYPE_LONG, nullptr}}; - TSFile conf_file; - conf_file = TSfopen(file_name, "r"); - - if (conf_file != nullptr) { - TSDebug(DEBUG_TAG, "opened config: %s", file_name); - char buf[1024]; - while (TSfgets(conf_file, buf, sizeof(buf) - 1) != nullptr) { - if (buf[0] != '#') { - parse_config_line(buf, config_vals); - } - } - TSfclose(conf_file); - } else { - TSError("[%s] Failed to open upload config file %s", PLUGIN_NAME, file_name); - // if fail to open config file, use the default config - } - - if (uconfig->base_dir == nullptr) { - uconfig->base_dir = TSstrdup("/FOOBAR/var/buffer_upload_tmp"); - } else { - // remove the "/" at the end. - if (uconfig->base_dir[strlen(uconfig->base_dir) - 1] == '/') { - uconfig->base_dir[strlen(uconfig->base_dir) - 1] = '\0'; - } - } - - if (uconfig->subdir_num <= 0) { - // default value - uconfig->subdir_num = 64; - } - - if (uconfig->thread_num <= 0) { - // default value - uconfig->thread_num = 4; - } - - return true; -} - -void -TSPluginInit(int argc, const char *argv[]) -{ - TSPluginRegistrationInfo info; - TSCont contp; - char default_filename[1024]; - const char *conf_filename; - - if (argc > 1) { - conf_filename = argv[1]; - } else { - sprintf(default_filename, "%s/upload.conf", TSPluginDirGet()); - conf_filename = default_filename; - } - - if (!read_upload_config(conf_filename) || !uconfig) { - if (argc > 1) { - TSError("[%s] Failed to read upload config %s", PLUGIN_NAME, argv[1]); - } else { - TSError("[%s] No config file specified. Specify conf file in plugin.conf: " - "'buffer_upload.so /path/to/upload.conf'", - PLUGIN_NAME); - } - } - // set the num of threads for disk AIO - if (TSAIOThreadNumSet(uconfig->thread_num) == TS_ERROR) { - TSError("[%s] Failed to set thread number", PLUGIN_NAME); - } - - TSDebug(DEBUG_TAG, "uconfig->url_list_file: %s", uconfig->url_list_file); - if (uconfig->url_list_file) { - load_urls(uconfig->url_list_file); - TSDebug(DEBUG_TAG, "loaded uconfig->url_list_file, num urls: %d", uconfig->url_num); - } - - info.plugin_name = const_cast("buffer_upload"); - info.vendor_name = const_cast("Apache Software Foundation"); - info.support_email = const_cast("dev@trafficserver.apache.org"); - - if (uconfig->use_disk_buffer && !create_directory()) { - TSError("[%s] Directory creation failed", PLUGIN_NAME); - uconfig->use_disk_buffer = false; - } - - if (TSPluginRegister(&info) != TS_SUCCESS) { - TSError("[%s] Plugin registration failed", PLUGIN_NAME); - } - - /* create the statistic variables */ - upload_vc_count = TSStatCreate("upload_vc.count", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); - - contp = TSContCreate(attach_pvc_plugin, nullptr); - TSHttpHookAdd(TS_HTTP_PRE_REMAP_HOOK, contp); -} From 174e17324de5cd874943c8f7e93f9cdbcdbe21ed Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 19 Jul 2019 13:39:07 -0600 Subject: [PATCH 3/7] Removes the header_normalize plugin. See #5395 --- plugins/Makefile.am | 1 - .../header_normalize/Makefile.inc | 20 -- .../header_normalize/header_normalize.cc | 265 ------------------ 3 files changed, 286 deletions(-) delete mode 100644 plugins/experimental/header_normalize/Makefile.inc delete mode 100644 plugins/experimental/header_normalize/header_normalize.cc diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 89cbdb2270f..1cc14672a5a 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -65,7 +65,6 @@ include experimental/custom_redirect/Makefile.inc include experimental/fq_pacing/Makefile.inc include experimental/geoip_acl/Makefile.inc include experimental/header_freq/Makefile.inc -include experimental/header_normalize/Makefile.inc include experimental/hipes/Makefile.inc include experimental/hook-trace/Makefile.inc include experimental/inliner/Makefile.inc diff --git a/plugins/experimental/header_normalize/Makefile.inc b/plugins/experimental/header_normalize/Makefile.inc deleted file mode 100644 index 2224fa0071e..00000000000 --- a/plugins/experimental/header_normalize/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pkglib_LTLIBRARIES += experimental/header_normalize/header_normalize.la - -experimental_header_normalize_header_normalize_la_SOURCES = \ - experimental/header_normalize/header_normalize.cc diff --git a/plugins/experimental/header_normalize/header_normalize.cc b/plugins/experimental/header_normalize/header_normalize.cc deleted file mode 100644 index d408ed5fdfe..00000000000 --- a/plugins/experimental/header_normalize/header_normalize.cc +++ /dev/null @@ -1,265 +0,0 @@ -/** @file - - Plugin to perform background fetches of certain content that would - otherwise not be cached. For example, Range: requests / responses. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -//////////////////////////////////////////////////////////////////////////////// -// header_normalize:: -// -// ATS plugin to convert headers into camel-case. It may be useful to solve -// interworking issues with legacy origins not supporting lower case headers -// required by protocols such as http2 etc. -// -// Note that the plugin currently uses READ_REQUEST_HDR_HOOK to camel-case -// the headers. As an optimization, it can be changed to SEND_REQUEST_HDR_HOOK -// so that it only converts, if/when the request is being sent to the origin. -// -// This plugin supports both global mode as well as per-remap mode activation -////////////////////////////////////////////////////////////////////////////////// - -#define UNUSED __attribute__((unused)) -static char UNUSED rcsId__header_normalize_cc[] = - "@(#) $Id: header_normalize.cc 218 2014-11-11 01:29:16Z sudheerv $ built on " __DATE__ " " __TIME__; - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; -/////////////////////////////////////////////////////////////////////////////// -// Some constants. -// -const char *PLUGIN_NAME = "header_normalize"; - -std::map> hdrMap; - -static void -buildHdrMap() -{ - hdrMap["accept"] = "Accept"; - hdrMap["accept-charset"] = "Accept-Charset"; - hdrMap["accept-encoding"] = "Accept-Encoding"; - hdrMap["accept-language"] = "Accept-Language"; - hdrMap["accept-ranges"] = "Accept-Ranges"; - hdrMap["age"] = "Age"; - hdrMap["allow"] = "Allow"; - hdrMap["approved"] = "Approved"; - hdrMap["bytes"] = "Bytes"; - hdrMap["cache-control"] = "Cache-Control"; - hdrMap["client-ip"] = "Client-Ip"; - hdrMap["connection"] = "Connection"; - hdrMap["content-base"] = "Content-Base"; - hdrMap["content-encoding"] = "Content-Encoding"; - hdrMap["content-language"] = "Content-Language"; - hdrMap["content-length"] = "Content-Length"; - hdrMap["content-location"] = "Content-Location"; - hdrMap["content-md5"] = "Content-MD5"; - hdrMap["content-range"] = "Content-Range"; - hdrMap["content-type"] = "Content-Type"; - hdrMap["control"] = "Control"; - hdrMap["cookie"] = "Cookie"; - hdrMap["date"] = "Date"; - hdrMap["distribution"] = "Distribution"; - hdrMap["etag"] = "Etag"; - hdrMap["expect"] = "Expect"; - hdrMap["expires"] = "Expires"; - hdrMap["followup-to"] = "Followup-To"; - hdrMap["from"] = "From"; - hdrMap["host"] = "Host"; - hdrMap["if-match"] = "If-Match"; - hdrMap["if-modified-since"] = "If-Modified-Since"; - hdrMap["if-none-match"] = "If-None-Match"; - hdrMap["if-range"] = "If-Range"; - hdrMap["if-unmodified-since"] = "If-Unmodified-Since"; - hdrMap["keep-alive"] = "Keep-Alive"; - hdrMap["keywords"] = "Keywords"; - hdrMap["last-modified"] = "Last-Modified"; - hdrMap["lines"] = "Lines"; - hdrMap["location"] = "Location"; - hdrMap["max-forwards"] = "Max-Forwards"; - hdrMap["message-id"] = "Message-Id"; - hdrMap["newsgroups"] = "Newsgroups"; - hdrMap["organization"] = "Organization"; - hdrMap["path"] = "Path"; - hdrMap["pragma"] = "Pragma"; - hdrMap["proxy-authenticate"] = "Proxy-Authenticate"; - hdrMap["proxy-authorization"] = "Proxy-Authorization"; - hdrMap["proxy-connection"] = "Proxy-Connection"; - hdrMap["public"] = "Public"; - hdrMap["range"] = "Range"; - hdrMap["references"] = "References"; - hdrMap["referer"] = "Referer"; - hdrMap["reply-to"] = "Reply-To"; - hdrMap["retry-after"] = "Retry-After"; - hdrMap["sender"] = "Sender"; - hdrMap["server"] = "Server"; - hdrMap["set-cookie"] = "Set-Cookie"; - hdrMap["strict-transport-security"] = "Strict-Transport-Security"; - hdrMap["subject"] = "Subject"; - hdrMap["summary"] = "Summary"; - hdrMap["te"] = "Te"; - hdrMap["transfer-encoding"] = "Transfer-Encoding"; - hdrMap["upgrade"] = "Upgrade"; - hdrMap["user-agent"] = "User-Agent"; - hdrMap["vary"] = "Vary"; - hdrMap["via"] = "Via"; - hdrMap["warning"] = "Warning"; - hdrMap["www-authenticate"] = "Www-Authenticate"; - hdrMap["xref"] = "Xref"; - hdrMap["x-id"] = "X-ID"; - hdrMap["x-forwarded-for"] = "X-Forwarded-For"; - hdrMap["forwarded"] = "Forwarded"; - hdrMap["sec-websocket-key"] = "Sec-WebSocket-Key"; - hdrMap["sec-websocket-version"] = "Sec-WebSocket-Version"; -} - -/////////////////////////////////////////////////////////////////////////////// -// Initialize the plugin. -// -// -// -TSReturnCode -TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) -{ - if (!api_info) { - strncpy(errbuf, "[tsremap_init] - Invalid TSREMAP_INTERFACE argument", errbuf_size - 1); - return TS_ERROR; - } - if (api_info->tsremap_version < TSREMAP_VERSION) { - snprintf(errbuf, errbuf_size, "[tsremap_init] - Incorrect API version %ld.%ld", api_info->tsremap_version >> 16, - (api_info->tsremap_version & 0xffff)); - return TS_ERROR; - } - buildHdrMap(); - TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); - return TS_SUCCESS; -} - -/////////////////////////////////////////////////////////////////////////////// -// One instance per remap.config invocation. -// -TSReturnCode -TSRemapNewInstance(int /* argc */, char * /* argv[] */, void ** /* ih */, char * /* errbuf */, int /* errbuf_size */) -{ - return TS_SUCCESS; -} - -void -TSRemapDeleteInstance(void * /* ih */) -{ -} - -static int -read_request_hook(TSCont /* contp */, TSEvent /* event */, void *edata) -{ - TSHttpTxn rh = (TSHttpTxn)edata; - - TSMLoc hdr, next_hdr; - TSMBuffer hdr_bufp; - TSMLoc req_hdrs; - - if (TSHttpTxnClientReqGet(rh, &hdr_bufp, &req_hdrs) == TS_SUCCESS) { - hdr = TSMimeHdrFieldGet(hdr_bufp, req_hdrs, 0); - int n_mime_headers = TSMimeHdrFieldsCount(hdr_bufp, req_hdrs); - - TSDebug(PLUGIN_NAME, "*** Camel Casing %u hdrs in the request", n_mime_headers); - - for (int i = 0; i < n_mime_headers; ++i) { - if (hdr == nullptr) { - break; - } - next_hdr = TSMimeHdrFieldNext(hdr_bufp, req_hdrs, hdr); - int old_hdr_len; - const char *old_hdr_name = TSMimeHdrFieldNameGet(hdr_bufp, req_hdrs, hdr, &old_hdr_len); - - // TSMimeHdrFieldNameGet returns the MIME_FIELD_NAME - // for all MIME hdrs, which is always in Camel Case - if (islower(old_hdr_name[0])) { - TSDebug(PLUGIN_NAME, "*** non MIME Hdr %s, leaving it for now", old_hdr_name); - - TSHandleMLocRelease(hdr_bufp, req_hdrs, hdr); - hdr = next_hdr; - continue; - } - - int hdr_value_len = 0; - const char *hdr_value = TSMimeHdrFieldValueStringGet(hdr_bufp, req_hdrs, hdr, 0, &hdr_value_len); - - // hdr returned by TSMimeHdrFieldNameGet is already - // in camel case, just destroy the lowercase h2 header - // and replace it with TSMimeHdrFieldNameGet - char *new_hdr_name = (char *)old_hdr_name; - if (new_hdr_name) { - TSMLoc new_hdr_loc; - TSReturnCode rval = TSMimeHdrFieldCreateNamed(hdr_bufp, req_hdrs, new_hdr_name, old_hdr_len, &new_hdr_loc); - - if (rval == TS_SUCCESS) { - TSDebug(PLUGIN_NAME, "*** hdr convert %s to %s", old_hdr_name, new_hdr_name); - TSMimeHdrFieldValueStringSet(hdr_bufp, req_hdrs, new_hdr_loc, -1, hdr_value, hdr_value_len); - TSMimeHdrFieldAppend(hdr_bufp, req_hdrs, new_hdr_loc); - TSHandleMLocRelease(hdr_bufp, req_hdrs, new_hdr_loc); - } - - TSMimeHdrFieldDestroy(hdr_bufp, req_hdrs, hdr); - } else { - TSDebug(PLUGIN_NAME, "*** can't find hdr %s in hdrMap", old_hdr_name); - } - - TSHandleMLocRelease(hdr_bufp, req_hdrs, hdr); - hdr = next_hdr; - } - - TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, req_hdrs); - } - - TSHttpTxnReenable(rh, TS_EVENT_HTTP_CONTINUE); - - return 0; -} - -void -TSPluginInit(int /* argc */, const char * /* argv[] */) -{ - TSDebug(PLUGIN_NAME, "initializing plugin"); - TSCont contp; - buildHdrMap(); - contp = TSContCreate(read_request_hook, nullptr); - TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, contp); -} - -/////////////////////////////////////////////////////////////////////////////// -// This is the main "entry" point for the plugin, called for every request. -// -TSRemapStatus -TSRemapDoRemap(void * /* ih */, TSHttpTxn rh, TSRemapRequestInfo * /* rri */) -{ - read_request_hook(nullptr, TS_EVENT_HTTP_READ_REQUEST_HDR, rh); - return TSREMAP_DID_REMAP; -} From 2c51618ebe79fb41d22a296dc8fcf8f915af9b7c Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 19 Jul 2019 13:39:58 -0600 Subject: [PATCH 4/7] Removes the hipes plugin. See #5395 --- doc/admin-guide/plugins/hipes.en.rst | 126 ---- doc/admin-guide/plugins/index.en.rst | 4 - .../admin-guide/plugins/hipes.en.po | 215 ------- .../admin-guide/plugins/index.en.po | 4 - plugins/Makefile.am | 1 - plugins/experimental/hipes/Makefile.inc | 20 - plugins/experimental/hipes/README | 25 - plugins/experimental/hipes/gen_escape.c | 46 -- plugins/experimental/hipes/hipes.cc | 552 ------------------ 9 files changed, 993 deletions(-) delete mode 100644 doc/admin-guide/plugins/hipes.en.rst delete mode 100644 doc/locale/ja/LC_MESSAGES/admin-guide/plugins/hipes.en.po delete mode 100644 plugins/experimental/hipes/Makefile.inc delete mode 100644 plugins/experimental/hipes/README delete mode 100644 plugins/experimental/hipes/gen_escape.c delete mode 100644 plugins/experimental/hipes/hipes.cc diff --git a/doc/admin-guide/plugins/hipes.en.rst b/doc/admin-guide/plugins/hipes.en.rst deleted file mode 100644 index 75d637dced7..00000000000 --- a/doc/admin-guide/plugins/hipes.en.rst +++ /dev/null @@ -1,126 +0,0 @@ -.. _admin-plugins-hipes: - -HIPES Plugin -************ - -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - -This is a remap plugin used in the HIPES system. - -Configuration -============= - -``urlp:`` - Default: ``url`` - Name of the query parameter for the service URL - -``path:`` - Default: ``/`` - Path to use for the service URL - -``ssl`` - Default: ``no`` - Use SSL when connecting to the service - -``service`` - Service server, ``host[:port]`` - -``server`` - Default: ``hipes.yimg.com`` - Name of HIPES server, ``host[:port]`` - -``active_timeout`` - The active connection timeout in ms - -``no_activity_timeout`` - The no activity timeout in ms - -``connect_timeout`` - The connect timeout in ms - -``dns_timeout`` - The DNS timeout - -The timeout options override the server defaults (from -```records.config``), and -only apply to the connection to the specific "service" host. - -Notes on HIPES -============== - -HTTP Pipes (aka HIPES and pronounced "Hippies") allows data services to be pipelined together, as illustrated by the -example below. - -Example -======= -1. ATS is run on port 80 and apache HTTP web server is run on port 8080 on localhost (127.0.0.1) - -2. The HIPES plugin is used in ``remap.config`` :: - - map http://127.0.0.1/svc_case http://nosuchhost @plugin=hipes.so @pparam=service:127.0.0.1:8080 @pparam=path:svc_case.php @pparam=server:127.0.0.1 - map http://127.0.0.1/svc_reverse http://nosuchhost @plugin=hipes.so @pparam=service:127.0.0.1:8080 @pparam=path:svc_reverse.php @pparam=server:127.0.0.1 - map http://127.0.0.1/test.txt http://127.0.0.1:8080/test.txt - -3. The plugin remaps the incoming URL such as :: - - http://127.0.0.1/svc_reverse/svc_case;case=u/test.txt - -to the following :: - - http://127.0.0.1:8080/svc_reverse?url=http%3A%2F%2F127.0.0.1%2Fsvc_case%3Bcase%3Du%2Ftest.txt - -4. The service ``svc_reverse.php`` fetches the ``url`` from the ATS again and the plugin remaps the URL :: - - http://127.0.0.1/svc_case;case=u/test.txt - -to this URL :: - - http://127.0.0.1:8080/svc_case.php?case=u&url=http%3A%2F%2F127.0.0.1%2Ftest.txt - -5. In this example, the service ``svc_case.php`` fetches and transforms the response of ``http://127.0.0.1/test.txt`` -(which ATS proxies the request to a local file) to upper case. And subsequently the service ``svc_reverse.php`` receives -the response and reverse the order before the response is sent back to the client by ATS. - -Notes on reducing traffic -========================= - -There can be significant overhead using HIPES because the data can traverse through ATS many times. Caching can be -important to reduce traffic to services/through ATS and can be achieved via a proper ``Cache-Control`` header returned -by the services. Another way to reduce traffic through ATS is to have ATS to return 302 redirects to url for the -requests made by service, instead of proxying the requests to that url. However, the service must then be able to follow -the redirect. The down side is that we cannot use ATS to cache intermediate results. Below is an example of using -redirect. - -Modification to above example to reduce traffic using redirect -============================================================== - -1. The service ``svc_reverse.php`` is modified to add a header of ``X-HIPES-Redirect: 2`` to the request made against -``url``. - -2. HIPES plugin will instruct ATS to return a redirect response to this url :: - - http://127.0.0.1:8080/svc_case.php?case=u&url=http%3A%2F%2F127.0.0.1%2Ftest.txt - -for the following request :: - - http://127.0.0.1/svc_case;case=u/test.txt - -3. The service ``svc_reverse.php`` is also modified to follow the redirect. Thus the response of the service of -``svc_case.php`` will not pass through ATS and will pass to ``svc_reverse.php`` service instead. - diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index fecae9615f8..e2ff09a4842 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -150,7 +150,6 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi GeoIP ACL FQ Pacing Header Frequency - HIPES Hook Trace JA3 Fingerprint Memcache @@ -190,9 +189,6 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`Header Frequency ` Count the frequency of headers. -:doc:`HIPES ` - Adds support for HTTP Pipes. - :doc:`JA3 Fingerprint ` Calculates JA3 Fingerprints for incoming SSL traffic. diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/hipes.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/hipes.en.po deleted file mode 100644 index a0e6c6277d7..00000000000 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/hipes.en.po +++ /dev/null @@ -1,215 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: Apache Traffic Server 6.2\n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2016-06-30 14:07+0900\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=utf-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.2.0\n" - -#: ../../../admin-guide/plugins/hipes.en.rst:4 -msgid "HIPES Plugin" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:24 -msgid "This is a remap plugin used in the HIPES system." -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:27 -msgid "Configuration" -msgstr "設定" - -#: ../../../admin-guide/plugins/hipes.en.rst:31 -msgid "``urlp:``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:30 -msgid "Default: ``url`` Name of the query parameter for the service URL" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:35 -msgid "``path:``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:34 -msgid "Default: ``/`` Path to use for the service URL" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:39 -#, fuzzy -msgid "``ssl``" -msgstr "ssl" - -#: ../../../admin-guide/plugins/hipes.en.rst:38 -msgid "Default: ``no`` Use SSL when connecting to the service" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:42 -msgid "``service``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:42 -msgid "Service server, ``host[:port]``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:46 -msgid "``server``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:45 -msgid "Default: ``hipes.yimg.com`` Name of HIPES server, ``host[:port]``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:49 -msgid "``active_timeout``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:49 -msgid "The active connection timeout in ms" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:52 -msgid "``no_activity_timeout``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:52 -msgid "The no activity timeout in ms" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:55 -msgid "``connect_timeout``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:55 -msgid "The connect timeout in ms" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:58 -msgid "``dns_timeout``" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:58 -msgid "The DNS timeout" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:60 -msgid "" -"The timeout options override the server defaults (from ```records." -"config``), and only apply to the connection to the specific \"service\" " -"host." -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:65 -msgid "Notes on HIPES" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:67 -msgid "" -"HTTP Pipes (aka HIPES and pronounced \"Hippies\") allows data services to " -"be pipelined together, as illustrated by the example below." -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:71 -msgid "Example" -msgstr "例" - -#: ../../../admin-guide/plugins/hipes.en.rst:72 -msgid "" -"ATS is run on port 80 and apache HTTP web server is run on port 8080 on " -"localhost (127.0.0.1)" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:74 -msgid "The HIPES plugin is used in ``remap.config`` ::" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:80 -msgid "The plugin remaps the incoming URL such as ::" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:84 -msgid "to the following ::" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:88 -msgid "" -"The service ``svc_reverse.php`` fetches the ``url`` from the ATS again and " -"the plugin remaps the URL ::" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:92 -msgid "to this URL ::" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:96 -msgid "" -"5. In this example, the service ``svc_case.php`` fetches and transforms the " -"response of ``http://127.0.0.1/test.txt`` (which ATS proxies the request to " -"a local file) to upper case. And subsequently the service ``svc_reverse." -"php`` receives the response and reverse the order before the response is " -"sent back to the client by ATS." -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:101 -msgid "Notes on reducing traffic" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:103 -msgid "" -"There can be significant overhead using HIPES because the data can traverse " -"through ATS many times. Caching can be important to reduce traffic to " -"services/through ATS and can be achieved via a proper ``Cache-Control`` " -"header returned by the services. Another way to reduce traffic through ATS " -"is to have ATS to return 302 redirects to url for the requests made by " -"service, instead of proxying the requests to that url. However, the service " -"must then be able to follow the redirect. The down side is that we cannot " -"use ATS to cache intermediate results. Below is an example of using " -"redirect." -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:111 -msgid "Modification to above example to reduce traffic using redirect" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:113 -msgid "" -"1. The service ``svc_reverse.php`` is modified to add a header of ``X-HIPES-" -"Redirect: 2`` to the request made against ``url``." -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:116 -msgid "" -"HIPES plugin will instruct ATS to return a redirect response to this url ::" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:120 -msgid "for the following request ::" -msgstr "" - -#: ../../../admin-guide/plugins/hipes.en.rst:124 -msgid "" -"3. The service ``svc_reverse.php`` is also modified to follow the " -"redirect. Thus the response of the service of ``svc_case.php`` will not " -"pass through ATS and will pass to ``svc_reverse.php`` service instead." -msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po index 7f2da5180f1..76d9172f62f 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po @@ -250,10 +250,6 @@ msgstr "" msgid "Deny or allow requests based on the source IP geo-location." msgstr "" -#: ../../../admin-guide/plugins/index.en.rst:165 -msgid ":doc:`HIPES `" -msgstr "" - #: ../../../admin-guide/plugins/index.en.rst:165 msgid "Adds support for HTTP Pipes." msgstr "" diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 1cc14672a5a..d11e100a2f8 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -65,7 +65,6 @@ include experimental/custom_redirect/Makefile.inc include experimental/fq_pacing/Makefile.inc include experimental/geoip_acl/Makefile.inc include experimental/header_freq/Makefile.inc -include experimental/hipes/Makefile.inc include experimental/hook-trace/Makefile.inc include experimental/inliner/Makefile.inc include experimental/memcache/Makefile.inc diff --git a/plugins/experimental/hipes/Makefile.inc b/plugins/experimental/hipes/Makefile.inc deleted file mode 100644 index 412cd024fd1..00000000000 --- a/plugins/experimental/hipes/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pkglib_LTLIBRARIES += experimental/hipes/hipes.la - -experimental_hipes_hipes_la_SOURCES = \ - experimental/hipes/hipes.cc diff --git a/plugins/experimental/hipes/README b/plugins/experimental/hipes/README deleted file mode 100644 index b12f7227db1..00000000000 --- a/plugins/experimental/hipes/README +++ /dev/null @@ -1,25 +0,0 @@ -This plugin implements the HIPES system. - -Plugin configuration --------------------- - - server: Name of the server to send this request to - urlp: Name of the query parameter for the service URL [url] - path: Path to use for the service URL [/] - ssl Use SSL when connecting to the service [no] - service Service server, host[:port] [no default] - server Name of HIPES server, host[:port] [hipes.yimg.com] - active_timeout The active connection timeout in ms [no default] - no_activity_timeout The no activity timeout in ms [no default] - connect_timeout The connect timeout in ms [no default] - dns_timeout The DNS timeout [no default] - - -The timeout options override the server defaults (from records.config), -and only apply to the connection to the specific "service" host. - -RELEASES --------- - -Version 1.0 (??/??/10) - - First release. diff --git a/plugins/experimental/hipes/gen_escape.c b/plugins/experimental/hipes/gen_escape.c deleted file mode 100644 index 3f1b346a817..00000000000 --- a/plugins/experimental/hipes/gen_escape.c +++ /dev/null @@ -1,46 +0,0 @@ -/** @file - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -#include -#include - -int -main() -{ - unsigned char codes[32]; - unsigned char hex; - int c; - - memset(codes, 0, sizeof(codes)); - - for (c = 0; c <= 255; ++c) { - if (((c >= '0') && (c <= '9')) || ((c >= 'A') && (c <= 'Z')) || ((c >= 'a') && (c <= 'z')) || ((c == '_')) || ((c == '-')) || - ((c == '.'))) { - } else { - codes[c / 8] |= (1 << (7 - c % 8)); - } - } - - for (hex = 0; hex < 32; ++hex) { - printf("0x%02lX, ", codes[hex]); - if (!((hex + 1) % 4)) - printf("\n"); - } -} diff --git a/plugins/experimental/hipes/hipes.cc b/plugins/experimental/hipes/hipes.cc deleted file mode 100644 index c90148a33f9..00000000000 --- a/plugins/experimental/hipes/hipes.cc +++ /dev/null @@ -1,552 +0,0 @@ -/** @file - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -#include -#include -#include -#include -#include - -#include -#include - -static const char *PLUGIN_NAME = "hipes"; -static const char *HIPES_SERVER_NAME = "hipes.example.com"; - -static const int MAX_PATH_SIZE = 2048; -static const int MAX_REDIRECT_URL = 2048; - -/////////////////////////////////////////////////////////////////////////////// -// Escape a URL string. -// -int -escapify_url(const char *src, int src_len, char *dst, int dst_len) -{ - // This bitmap is generated using the gen_escape.c prog. - static unsigned char codes_to_escape[32] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xF9, 0x00, 0x3F, 0x80, 0x00, 0x00, - 0x1E, 0x80, 0x00, 0x00, 0x1F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; - - static char hex_digit[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; - - const char *from = src; - char *to = dst; - int len = 0; - - // Sanity check - if (!src) { - return -1; - } - - while (from < (src + src_len)) { - unsigned char c = *from; - - if (len >= dst_len) { - return -1; // Does not fit.... abort! - } - - if (codes_to_escape[c / 8] & (1 << (7 - c % 8))) { - *to++ = '%'; - *to++ = hex_digit[c / 16]; - *to++ = hex_digit[c % 16]; - len += 3; - } else { - *to++ = *from; - ++len; - } - ++from; - } - *to = '\0'; - - return len; -} - -/////////////////////////////////////////////////////////////////////////////// -// Unescape a string. Have to make sure the destination buffer is at least as -// long as the source buffer. -// -char * -unescapify(const char *src, char *dst, int len) -{ - const char *cur = src; - char *next; - char subStr[3]; - int size; - - subStr[2] = '\0'; - while ((next = (char *)memchr(cur, '%', len))) { - size = next - cur; - if (size > 0) { - memcpy(dst, cur, size); - dst += size; - cur += size; - len -= size; - } - - if (len > 2 && (*cur + 1) != '\0' && *(cur + 2) != '\0') { - subStr[0] = *(++cur); - subStr[1] = *(++cur); - len -= 2; - *dst = (char)strtol(subStr, (char **)nullptr, 16); - } else { - *dst = *cur; - } - ++dst; - ++cur; - --len; - } - - if (len > 0) { - memcpy(dst, cur, len); - dst += len; - } - - return dst; -} - -/////////////////////////////////////////////////////////////////////////////// -// Class encapsulating one service configuration -// -struct HIPESService { - HIPESService() - : url_param("url"), - path(""), - svc_server(""), - - hipes_server(HIPES_SERVER_NAME), - - x_hipes_header("X-HIPES-Redirect"){}; - - std::string url_param; - std::string path; - std::string svc_server; - int svc_port = 80; - bool ssl = false; - std::string hipes_server; - int hipes_port = 80; - unsigned int default_redirect_flag = 1; - std::string x_hipes_header; - int active_timeout = -1; - int no_activity_timeout = -1; - int connect_timeout = -1; - int dns_timeout = -1; -}; - -/////////////////////////////////////////////////////////////////////////////// -// Initialize the plugin. -// -TSReturnCode -TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) -{ - if (!api_info) { - strncpy(errbuf, "[tsremap_init] - Invalid TSRemapInterface argument", errbuf_size - 1); - return TS_ERROR; - } - - if (api_info->tsremap_version < TSREMAP_VERSION) { - snprintf(errbuf, errbuf_size, "[tsremap_init] - Incorrect API version %ld.%ld", api_info->tsremap_version >> 16, - (api_info->tsremap_version & 0xffff)); - return TS_ERROR; - } - - TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); - return TS_SUCCESS; -} - -/////////////////////////////////////////////////////////////////////////////// -// One instance per remap.config invocation. -// -TSReturnCode -TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSED */, int /* errbuf_size ATS_UNUSED */) -{ - HIPESService *ri = new HIPESService(); - - *ih = (void *)ri; - - if (ri == nullptr) { - TSError("[hipes] Unable to create remap instance"); - return TS_ERROR; - } - - for (int ix = 2; ix < argc; ++ix) { - std::string arg = argv[ix]; - std::string::size_type sep = arg.find_first_of(':'); - - if (sep == std::string::npos) { - TSError("[hipes] Malformed options in url_remap: %s", argv[ix]); - } else { - std::string arg_val = arg.substr(sep + 1, std::string::npos); - - if (arg.compare(0, 4, "urlp") == 0) { - ri->url_param = arg_val; - } else if (arg.compare(0, 4, "path") == 0) { - ri->path = arg_val; - if (arg_val[0] == '/') { - ri->path = arg_val.substr(1); - } else { - ri->path = arg_val; - } - } else if (arg.compare(0, 3, "ssl") == 0) { - ri->ssl = true; - } else if (arg.compare(0, 7, "service") == 0) { - std::string::size_type port = arg_val.find_first_of(':'); - - if (port == std::string::npos) { - ri->svc_server = arg_val; - } else { - ri->svc_server = arg_val.substr(0, port); - ri->svc_port = atoi(arg_val.substr(port + 1).c_str()); - } - } else if (arg.compare(0, 6, "server") == 0) { - std::string::size_type port = arg_val.find_first_of(':'); - - if (port == std::string::npos) { - ri->hipes_server = arg_val; - } else { - ri->hipes_server = arg_val.substr(0, port); - ri->hipes_port = atoi(arg_val.substr(port + 1).c_str()); - } - } else if (arg.compare(0, 14, "active_timeout") == 0) { - ri->active_timeout = atoi(arg_val.c_str()); - } else if (arg.compare(0, 19, "no_activity_timeout") == 0) { - ri->no_activity_timeout = atoi(arg_val.c_str()); - } else if (arg.compare(0, 15, "connect_timeout") == 0) { - ri->connect_timeout = atoi(arg_val.c_str()); - } else if (arg.compare(0, 11, "dns_timeout") == 0) { - ri->dns_timeout = atoi(arg_val.c_str()); - } else { - TSError("[hipes] Unknown url_remap option: %s", argv[ix]); - } - } - } - - return TS_SUCCESS; -} - -void -TSRemapDeleteInstance(void *ih) -{ - HIPESService *ri = static_cast(ih); - - delete ri; -} - -/////////////////////////////////////////////////////////////////////////////// -// This is the main "entry" point for the plugin, called for every request. -// -TSRemapStatus -TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) -{ - const char *slash; - char *ptr; - HIPESService *h_conf = (HIPESService *)ih; - - char new_query[MAX_PATH_SIZE]; - int new_query_size; - char redirect_url[MAX_REDIRECT_URL]; - int redirect_url_size; - - if (nullptr == h_conf) { - TSDebug(PLUGIN_NAME, "Falling back to default URL on URL remap without rules"); - return TSREMAP_NO_REMAP; - } - - int param_len = 0; - const char *param = TSUrlHttpParamsGet(rri->requestBufp, rri->requestUrl, ¶m_len); - - // Make sure we have a matrix parameter, anything without is a bogus request. - if (param_len <= 0) { - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_BAD_REQUEST); - return TSREMAP_NO_REMAP; - } - - if (param_len > MAX_PATH_SIZE) { - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_REQUEST_URI_TOO_LONG); - return TSREMAP_NO_REMAP; - } - - // If there is a '/' in the matrix parameters, we know there are multiple service requests in - // the incoming URL, so nibble off the first one, and pass the rest as a HIPES URL to the service. - if ((slash = static_cast(memchr(param, '/', param_len)))) { - char svc_url[MAX_PATH_SIZE + 1]; - char svc_url_esc[MAX_PATH_SIZE + 1]; - int len, query_len; - - // Create the escaped URL, which gets passed over to the service as a url= param. - char port[6]; - snprintf(port, 6, ":%d", h_conf->hipes_port); - - if (h_conf->hipes_port != 80) { - len = 8 + h_conf->hipes_server.size() + strlen(port) + (param_len - (slash - param) - 1); - } else { - len = 8 + h_conf->hipes_server.size() + (param_len - (slash - param) - 1); - } - if (len > MAX_PATH_SIZE) { - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_REQUEST_URI_TOO_LONG); - return TSREMAP_NO_REMAP; - } - if (h_conf->hipes_port != 80) { - snprintf(svc_url, MAX_PATH_SIZE, "http://%s%s/%.*s", h_conf->hipes_server.c_str(), port, len, slash + 1); - } else { - snprintf(svc_url, MAX_PATH_SIZE, "http://%s/%.*s", h_conf->hipes_server.c_str(), len, slash + 1); - } - TSDebug(PLUGIN_NAME, "Service URL is %s", svc_url); - - len = escapify_url(svc_url, len, svc_url_esc, MAX_PATH_SIZE); - if (len < 0) { - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_BAD_REQUEST); - return TSREMAP_NO_REMAP; - } - TSDebug(PLUGIN_NAME, "Escaped service URL is %s(%d)", svc_url_esc, len); - - // Prepare the new query arguments, make sure it fits - if (((slash - param) + 2 + (int)h_conf->url_param.size() + len) > MAX_PATH_SIZE) { - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_REQUEST_URI_TOO_LONG); - return TSREMAP_NO_REMAP; - } - - query_len = (slash - param); - memcpy(new_query, param, query_len); - ptr = new_query; - while ((ptr = static_cast(memchr(ptr, ';', (new_query + query_len) - ptr)))) { - *ptr = '&'; - } - new_query[query_len++] = '&'; - memcpy(new_query + query_len, h_conf->url_param.c_str(), h_conf->url_param.size()); - query_len += h_conf->url_param.size(); - new_query[query_len++] = '='; - - memcpy(new_query + query_len, svc_url_esc, len); - new_query_size = query_len + len; - TSDebug(PLUGIN_NAME, "New query is %.*s(%d)", new_query_size, new_query, new_query_size); - } else { - // This is the "final" step in this HIPES URL, so don't point back to HIPES (or we'll never leave) - new_query_size = param_len; - memcpy(new_query, param, param_len); - ptr = new_query; - while ((ptr = static_cast(memchr(ptr, ';', (new_query + new_query_size) - ptr)))) { - *ptr = '&'; - } - TSDebug(PLUGIN_NAME, "New query is %.*s(%d)", new_query_size, new_query, new_query_size); - } - - // Test if we should redirect or not - bool do_redirect = false; - int redirect_flag = h_conf->default_redirect_flag; - char *pos = new_query; - - while (pos && (pos = (char *)memchr(pos, '_', new_query_size - (pos - new_query)))) { - if (pos) { - ++pos; - if ((new_query_size - (pos - new_query)) < 10) { // redirect=n - pos = nullptr; - } else { - if ((*pos == 'r') && (!strncmp(pos, "redirect=", 9))) { - redirect_flag = *(pos + 9) - '0'; - if ((redirect_flag < 0) || (redirect_flag > 2)) { - redirect_flag = h_conf->default_redirect_flag; - } - TSDebug(PLUGIN_NAME, "Found _redirect flag in URL: %d", redirect_flag); - pos = nullptr; - } - } - } - } - - if (redirect_flag > 0) { - // Now check the incoming request header, and match up. - TSMBuffer bufp; - TSMLoc hdr_loc, field_loc; - bool has_error = false; - - if (TSHttpTxnClientReqGet(rh, &bufp, &hdr_loc) == TS_SUCCESS) { - field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, h_conf->x_hipes_header.c_str(), h_conf->x_hipes_header.size()); - if (field_loc) { - int hdr_flag; - - hdr_flag = TSMimeHdrFieldValueIntGet(bufp, hdr_loc, field_loc, 0); - // Alright, now match up this header flag with the request (or default) flag - TSDebug(PLUGIN_NAME, "Extracted %s header with value %d", h_conf->x_hipes_header.c_str(), hdr_flag); - switch (redirect_flag) { - case 1: - if (hdr_flag == 2) { - do_redirect = true; - } // Everything else is a "no" - break; - case 2: - if (hdr_flag == 2) { - do_redirect = true; - } else { - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_BAD_REQUEST); - has_error = true; - } - break; - default: - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_BAD_REQUEST); - has_error = true; - break; - } - TSHandleMLocRelease(bufp, hdr_loc, field_loc); - } else { - if (redirect_flag == 2) { - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_BAD_REQUEST); - has_error = true; - } - } - TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); - } else { - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_BAD_REQUEST); - has_error = true; - } - if (has_error) { - return TSREMAP_NO_REMAP; - } - } - - // If we redirect, just generate a 302 URL, otherwise update the RRI struct properly. - if (do_redirect) { - int len; - - if (h_conf->ssl) { - // https://:/?svc_server.size() + 6 + 1 + h_conf->path.size() + 1 + new_query_size + 1; - } else { - // http://:/?svc_server.size() + 6 + 1 + h_conf->path.size() + 1 + new_query_size + 1; - } - - if (len > MAX_REDIRECT_URL) { - TSError("[hipes] Redirect in HIPES URL too long"); - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_REQUEST_URI_TOO_LONG); - } else { - int port = -1; - - pos = redirect_url; - - // HTTP vs HTTPS - if (h_conf->ssl) { - memcpy(pos, "https://", 8); - pos += 8; - if (h_conf->svc_port != 443) { - port = h_conf->svc_port; - } - } else { - memcpy(pos, "http://", 7); - pos += 7; - if (h_conf->svc_port != 80) { - port = h_conf->svc_port; - } - } - - // Server - memcpy(pos, h_conf->svc_server.c_str(), h_conf->svc_server.size()); - pos += h_conf->svc_server.size(); - - // Port - if (port != -1) { - pos += snprintf(pos, 6, ":%d", port); - } - - // Path - *(pos++) = '/'; - if (h_conf->path.size() > 0) { - memcpy(pos, h_conf->path.c_str(), h_conf->path.size()); - pos += h_conf->path.size(); - } - - // Query - if (new_query_size > 0) { - *(pos++) = '?'; - memcpy(pos, new_query, new_query_size); - pos += new_query_size; - } - - // NULL terminate the URL. - *pos = '\0'; - - redirect_url_size = pos - redirect_url + 1; - TSDebug(PLUGIN_NAME, "Redirecting to %.*s", redirect_url_size, redirect_url); - const char *start = redirect_url; - const char *end = start + redirect_url_size; - rri->redirect = 1; - TSUrlParse(rri->requestBufp, rri->requestUrl, &start, end); - TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_MOVED_TEMPORARILY); - } - } else { // Not a redirect, so proceed normally - // Set timeouts (if requested) - if (h_conf->active_timeout > -1) { - TSDebug(PLUGIN_NAME, "Setting active timeout to %d", h_conf->active_timeout); - TSHttpTxnActiveTimeoutSet(rh, h_conf->active_timeout); - } - if (h_conf->no_activity_timeout > -1) { - TSDebug(PLUGIN_NAME, "Setting no activity timeout to %d", h_conf->no_activity_timeout); - TSHttpTxnNoActivityTimeoutSet(rh, h_conf->no_activity_timeout); - } - if (h_conf->connect_timeout > -1) { - TSDebug(PLUGIN_NAME, "Setting connect timeout to %d", h_conf->connect_timeout); - TSHttpTxnConnectTimeoutSet(rh, h_conf->connect_timeout); - } - if (h_conf->dns_timeout > -1) { - TSDebug(PLUGIN_NAME, "Setting DNS timeout to %d", h_conf->dns_timeout); - TSHttpTxnDNSTimeoutSet(rh, h_conf->dns_timeout); - } - - // Set server ... - TSUrlHostSet(rri->requestBufp, rri->requestUrl, h_conf->svc_server.c_str(), h_conf->svc_server.size()); - TSDebug(PLUGIN_NAME, "New server is %.*s", (int)h_conf->svc_server.size(), h_conf->svc_server.c_str()); - - // ... and port - TSUrlPortSet(rri->requestBufp, rri->requestUrl, h_conf->svc_port); - TSDebug(PLUGIN_NAME, "New port is %d", h_conf->svc_port); - - // Update the path - TSUrlPathSet(rri->requestBufp, rri->requestUrl, h_conf->path.c_str(), h_conf->path.size()); - TSDebug(PLUGIN_NAME, "New path is %.*s", (int)h_conf->path.size(), h_conf->path.c_str()); - - // Enable SSL? - if (h_conf->ssl) { - TSUrlSchemeSet(rri->requestBufp, rri->requestUrl, "https", 5); - } - - // Clear previous matrix params - TSUrlHttpParamsSet(rri->requestBufp, rri->requestUrl, "", 0); - - // set query - TSUrlHttpQuerySet(rri->requestBufp, rri->requestUrl, new_query, new_query_size); - } - - // Step 3: Profit - return TSREMAP_DID_REMAP; -} - -/* - local variables: - mode: C++ - indent-tabs-mode: nil - c-basic-offset: 2 - c-comment-only-line-offset: 0 - c-file-offsets: ((statement-block-intro . +) - (label . 0) - (statement-cont . +) - - end: - - Indent with: /usr/bin/indent -ncs -nut -npcs -l 120 logstats.cc -*/ From 87f8765af7b638280572bb657d1b61f2b23a86cd Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 19 Jul 2019 13:41:49 -0600 Subject: [PATCH 5/7] Removes the memcached_remap plugin. See #5395 --- configure.ac | 6 - plugins/Makefile.am | 4 - plugins/experimental/memcached_remap/AUTHORS | 7 - .../experimental/memcached_remap/Changelog | 5 - .../experimental/memcached_remap/Makefile.inc | 24 --- plugins/experimental/memcached_remap/README | 113 ----------- .../memcached_remap/memcached_remap.cc | 192 ------------------ .../experimental/memcached_remap/sample.py | 33 --- 8 files changed, 384 deletions(-) delete mode 100644 plugins/experimental/memcached_remap/AUTHORS delete mode 100644 plugins/experimental/memcached_remap/Changelog delete mode 100644 plugins/experimental/memcached_remap/Makefile.inc delete mode 100644 plugins/experimental/memcached_remap/README delete mode 100644 plugins/experimental/memcached_remap/memcached_remap.cc delete mode 100755 plugins/experimental/memcached_remap/sample.py diff --git a/configure.ac b/configure.ac index 57b80cf1088..424bebe995f 100644 --- a/configure.ac +++ b/configure.ac @@ -509,12 +509,6 @@ AC_ARG_ENABLE([experimental-plugins], AC_MSG_RESULT([$enable_experimental_plugins]) AM_CONDITIONAL([BUILD_EXPERIMENTAL_PLUGINS], [ test "x${enable_experimental_plugins}" = "xyes" ]) -# -# Enable experimental/memcached_remap plugin -# -PKG_CHECK_MODULES([LIBMEMCACHED], [libmemcached >= 1.0], [have_libmemcached=yes], [have_libmemcached=no]) -AM_CONDITIONAL([BUILD_MEMCACHED_REMAP_PLUGIN], [test "x${have_libmemcached}" = "xyes"]) - # # Check Magick++ is available. Enable experimental/webp_transform plugin # diff --git a/plugins/Makefile.am b/plugins/Makefile.am index d11e100a2f8..f942ae7bf25 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -93,10 +93,6 @@ if BUILD_SSL_SESSION_REUSE_PLUGIN include experimental/ssl_session_reuse/Makefile.inc endif -if BUILD_MEMCACHED_REMAP_PLUGIN -include experimental/memcached_remap/Makefile.inc -endif - if BUILD_REMAP_STATS_PLUGIN include experimental/remap_stats/Makefile.inc endif diff --git a/plugins/experimental/memcached_remap/AUTHORS b/plugins/experimental/memcached_remap/AUTHORS deleted file mode 100644 index 945968f573e..00000000000 --- a/plugins/experimental/memcached_remap/AUTHORS +++ /dev/null @@ -1,7 +0,0 @@ -Author: opensource@navyaprabha.com - -memcached_remap plugin - -========= CREDITS =============== -Eric Balsa -Original concept via mysql_remap plugin diff --git a/plugins/experimental/memcached_remap/Changelog b/plugins/experimental/memcached_remap/Changelog deleted file mode 100644 index 0e9b96d8cee..00000000000 --- a/plugins/experimental/memcached_remap/Changelog +++ /dev/null @@ -1,5 +0,0 @@ -1.0.0 20-May-2011 - * Initial Release for 2.1.8-unstable apache release -1.0.0 14-Oct-2015 - * added autoconf changes and Makefile.am so this plugin may be built - when configure --enable-experimental-plugins is used. diff --git a/plugins/experimental/memcached_remap/Makefile.inc b/plugins/experimental/memcached_remap/Makefile.inc deleted file mode 100644 index cb893b011a2..00000000000 --- a/plugins/experimental/memcached_remap/Makefile.inc +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -experimental_memcached_remap_memcached_remap_la__CXXFLAGS = $(AM_CXXFLAGS) $(LIBMEMCACHED_CFLAGS) - -pkglib_LTLIBRARIES += experimental/memcached_remap/memcached_remap.la - -experimental_memcached_remap_memcached_remap_la_SOURCES = \ - experimental/memcached_remap/memcached_remap.cc - -experimental_memcached_remap_memcached_remap_la_LIBADD = $(LIBMEMCACHED_LIBS) diff --git a/plugins/experimental/memcached_remap/README b/plugins/experimental/memcached_remap/README deleted file mode 100644 index 248341f3086..00000000000 --- a/plugins/experimental/memcached_remap/README +++ /dev/null @@ -1,113 +0,0 @@ -Apache Traffic Server Memcached-based remap plugin -============================================== - -This is a plugin which is based on the mysql_remap code (https://github.com/apache/trafficserver/tree/master/plugins/experimental/mysql_remap) -that allows us to do dynamic reverse proxy (dynamic remap) based on the -information present in the memcached database. - - (decision via memcached) -[Browser] <----> [Apache Traffic Server] <----------> [Backend Servers] - (decision via memcached) - -NOTE: memcached is used only as decision making place. All communication -happens via HTTP between above components - -################## -# QUICK HOWTO # -################## - -The following steps are for Fedora-14 System, same will apply to your -server too (please change things accordingly, if any) - -1. Install memcached - -sudo yum install memcached python-memcached - -2. Compile and Install the memcached_remap plugin - - To build this with the ATS configure script when - --enable-experimental-plugins is used, you must - have libmemcached version 1.0 or greater installed. - -3. Start Traffic Server with logging (to debug any issues) - -/usr/local/bin/traffic_server -T "memcached_remap" - -3. Start the memcached - -sudo service memcached - -4. Start apache on port 8080 (change configs accordingly) - -sudo service httpd start - -5. Add some keys into the memcached for testing (via python shell) - -[user@ memcached_remap]$ python -Python 2.7 (r27:82500, Sep 16 2010, 18:02:00) -[GCC 4.5.1 20100907 (Red Hat 4.5.1-3)] on linux2 -Type "help", "copyright", "credits" or "license" for more information. ->>> import memcache ->>> mc = memcache.Client(['127.0.0.1:11211'], debug=0) ->>> print mc.get("http://localhost:80/") -http://localhost:8080/ ->>> print mc.get("http://localhost:80/") -http://localhost:8080/ ->>> mc.set("http://127.0.0.1:80/", "http://127.0.0.1:8080"); -True ->>> print mc.get("http://127.0.0.1:80/") -http://127.0.0.1:8080 ->>> - -6. Do sample query on localhost port 80 - -[user@ ~]$ curl -v http://127.0.0.1/ -s > /dev/null -* About to connect() to 127.0.0.1 port 80 (#0) -* Trying 127.0.0.1... connected -* Connected to 127.0.0.1 (127.0.0.1) port 80 (#0) -> GET / HTTP/1.1 -> User-Agent: curl/7.21.0 (x86_64-redhat-linux-gnu) libcurl/7.21.0 -> NSS/3.12.8.0 zlib/1.2.5 libidn/1.18 libssh2/1.2.4 -> Host: 127.0.0.1 -> Accept: */* -> -< HTTP/1.1 200 OK -< Date: Wed, 09 Mar 2011 15:48:25 GMT -< Server: ATS/2.1.6-unstable -< Last-Modified: Sat, 05 Mar 2011 16:15:13 GMT -< ETag: "618ef-38b-49dbe914322ed" -< Accept-Ranges: bytes -< Content-Length: 907 -< Content-Type: text/html; charset=UTF-8 -< Age: 0 -< Connection: keep-alive -< -{ [data not shown] -* Connection #0 to host 127.0.0.1 left intact -* Closing connection #0 - -7. Thats it ! - -################################ -# SUPPORTED KEYS IN MEMCACHED # -################################ - -The advantage of memcached is that, its a hash table. - -KEY FORMAT: [PROTOCOL]://[SERVER]:[PORT]/ --------------------------------------------- -(Even ending / is important!) - -PROTOCOL can be any word in a-z, A-Z chars -SERVER can be any string which isn't a colon character -PORT should be a number -/ should end with a slash - -VALUE FORMAT: [PROTOCOL]://[SERVER]:[PORT]/ --------------------------------------------- -(Even ending / is important!) - -PROTOCOL can be any word created from a-z, A-Z chars -SERVER can be any string which isn't a colon character -PORT should be a number -/ should end with a slash diff --git a/plugins/experimental/memcached_remap/memcached_remap.cc b/plugins/experimental/memcached_remap/memcached_remap.cc deleted file mode 100644 index e6b2834ca5c..00000000000 --- a/plugins/experimental/memcached_remap/memcached_remap.cc +++ /dev/null @@ -1,192 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include -#include -#include -#include -#include - -#include - -// global settings -static const char *PLUGIN_NAME = "memcached_remap"; - -// memcached related global variables -memcached_server_st *servers; -memcached_st *memc; - -bool -do_memcached_remap(TSCont contp, TSHttpTxn txnp) -{ - TSMBuffer reqp; - TSMLoc hdr_loc, url_loc, field_loc; - bool ret_val = false; - - const char *request_host; - int request_host_length = 0; - const char *request_scheme; - int request_scheme_length = 0; - int request_port = 80; - char ikey[1024]; - char *m_result = nullptr; - size_t oval_length; - uint32_t flags; - memcached_return_t lrc; - - if (TSHttpTxnClientReqGet((TSHttpTxn)txnp, &reqp, &hdr_loc) != TS_SUCCESS) { - TSDebug(PLUGIN_NAME, "could not get request data"); - return false; - } - - if (TSHttpHdrUrlGet(reqp, hdr_loc, &url_loc) != TS_SUCCESS) { - TSDebug(PLUGIN_NAME, "couldn't retrieve request url"); - goto release_hdr; - } - - field_loc = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST); - - if (!field_loc) { - TSDebug(PLUGIN_NAME, "couldn't retrieve request HOST header"); - goto release_url; - } - - request_host = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field_loc, -1, &request_host_length); - if (request_host == nullptr || strlen(request_host) < 1) { - TSDebug(PLUGIN_NAME, "couldn't find request HOST header"); - goto release_field; - } - - request_scheme = TSUrlSchemeGet(reqp, url_loc, &request_scheme_length); - request_port = TSUrlPortGet(reqp, url_loc); - - TSDebug(PLUGIN_NAME, " +++++MEMCACHED REMAP+++++ "); - - TSDebug(PLUGIN_NAME, "\nINCOMING REQUEST ->\n ::: from_scheme_desc: %.*s\n ::: from_hostname: %.*s\n ::: from_port: %d", - request_scheme_length, request_scheme, request_host_length, request_host, request_port); - - snprintf(ikey, 1024, "%.*s://%.*s:%d/", request_scheme_length, request_scheme, request_host_length, request_host, request_port); - - TSDebug(PLUGIN_NAME, "querying for the key %s", ikey); - m_result = memcached_get(memc, ikey, strlen(ikey), &oval_length, &flags, &lrc); - - char oscheme[1024], ohost[1024]; - int oport; - - if (lrc == MEMCACHED_SUCCESS) { - TSDebug(PLUGIN_NAME, "got the response from server : %s", m_result); - TSDebug(PLUGIN_NAME, "scanf result : %d", sscanf(m_result, "%[a-zA-Z]://%[^:]:%d", oscheme, ohost, &oport)); - if (sscanf(m_result, "%[a-zA-Z]://%[^:]:%d", oscheme, ohost, &oport) == 3) { - if (m_result) - free(m_result); - TSDebug(PLUGIN_NAME, "\nOUTGOING REQUEST ->\n ::: to_scheme_desc: %s\n ::: to_hostname: %s\n ::: to_port: %d", oscheme, ohost, - oport); - TSMimeHdrFieldValueStringSet(reqp, hdr_loc, field_loc, 0, ohost, -1); - TSUrlHostSet(reqp, url_loc, ohost, -1); - TSUrlSchemeSet(reqp, url_loc, oscheme, -1); - TSUrlPortSet(reqp, url_loc, oport); - ret_val = true; - } else { - if (m_result) - free(m_result); - goto not_found; - } - } else { - TSDebug(PLUGIN_NAME, "didn't get any response from the server %d, %d, %d", lrc, flags, (int)oval_length); - goto not_found; - } - - goto release_field; // free the result set after processed - -not_found: - // lets build up a nice 404 message for someone - if (!ret_val) { - TSHttpHdrStatusSet(reqp, hdr_loc, TS_HTTP_STATUS_NOT_FOUND); - TSHttpTxnStatusSet(txnp, TS_HTTP_STATUS_NOT_FOUND); - } -release_field: - if (field_loc) { - TSHandleMLocRelease(reqp, hdr_loc, field_loc); - } -release_url: - if (url_loc) { - TSHandleMLocRelease(reqp, hdr_loc, url_loc); - } -release_hdr: - if (hdr_loc) { - TSHandleMLocRelease(reqp, TS_NULL_MLOC, hdr_loc); - } - - return ret_val; -} - -static int -memcached_remap(TSCont contp, TSEvent event, void *edata) -{ - TSHttpTxn txnp = (TSHttpTxn)edata; - TSEvent reenable = TS_EVENT_HTTP_CONTINUE; - - if (event == TS_EVENT_HTTP_READ_REQUEST_HDR) { - TSDebug(PLUGIN_NAME, "Reading Request"); - TSSkipRemappingSet(txnp, 1); - if (!do_memcached_remap(contp, txnp)) { - reenable = TS_EVENT_HTTP_ERROR; - } - } - - TSHttpTxnReenable(txnp, reenable); - return 1; -} - -void -TSPluginInit(int argc, const char *argv[]) -{ - TSPluginRegistrationInfo info; - memcached_return_t rc; - - info.plugin_name = const_cast(PLUGIN_NAME); - info.vendor_name = const_cast("Apache Software Foundation"); - info.support_email = const_cast("dev@trafficserver.apache.org"); - - TSDebug(PLUGIN_NAME, "about to init memcached"); - if (TSPluginRegister(&info) != TS_SUCCESS) { - TSError("[memcached_remap] Plugin registration failed"); - return; - } - - memc = memcached_create(NULL); - - servers = memcached_server_list_append(NULL, "localhost", 11211, &rc); - if (rc != MEMCACHED_SUCCESS) { - TSError("[memcached_remap] Plugin registration failed while adding servers.\n"); - return; - } - - rc = memcached_server_push(memc, servers); - if (rc != MEMCACHED_SUCCESS) { - TSError("[memcached_remap] Plugin registration failed while adding to pool.\n"); - return; - } - - TSCont cont = TSContCreate(memcached_remap, TSMutexCreate()); - - TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, cont); - - TSDebug(PLUGIN_NAME, "plugin is successfully initialized [plugin mode]"); - return; -} diff --git a/plugins/experimental/memcached_remap/sample.py b/plugins/experimental/memcached_remap/sample.py deleted file mode 100755 index 01466cddbd4..00000000000 --- a/plugins/experimental/memcached_remap/sample.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/python -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Author: opensource@navyaprabha.com -# Description: Sample script to add keys to memcached for use with YTS/memcached_remap plugin - -import memcache - -# connect to local server -mc = memcache.Client(['127.0.0.1:11211'], debug=0) - -# Add couple of keys -mc.set("http://127.0.0.1:80/", "http://127.0.0.1:8080") -mc.set("http://localhost:80/", "http://localhost:8080") - -# Print the keys that are saved -print "response-1 is '%s'" % (mc.get("http://127.0.0.1:80/")) -print "response-2 is '%s'" % (mc.get("http://localhost:80/")) From 3e817cfd2efea9068df616676daa3c0db3c13535 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 19 Jul 2019 13:43:43 -0600 Subject: [PATCH 6/7] Removes the stale_while_revalidate plugin. See #5395 --- NOTICE | 5 - ci/rat-regex.txt | 1 - .../admin-guide/plugins/index.en.po | 4 - .../plugins/stale_while_revalidate.en.po | 37 - plugins/Makefile.am | 1 - .../stale_while_revalidate/Makefile.inc | 20 - .../stale_while_revalidate/Makefile.tsxs | 27 - .../stale_while_revalidate/README | 36 - .../stale_while_revalidate.c | 769 ------------------ .../stale_while_revalidate/test_server.js | 42 - 10 files changed, 942 deletions(-) delete mode 100644 doc/locale/ja/LC_MESSAGES/admin-guide/plugins/stale_while_revalidate.en.po delete mode 100644 plugins/experimental/stale_while_revalidate/Makefile.inc delete mode 100644 plugins/experimental/stale_while_revalidate/Makefile.tsxs delete mode 100644 plugins/experimental/stale_while_revalidate/README delete mode 100644 plugins/experimental/stale_while_revalidate/stale_while_revalidate.c delete mode 100755 plugins/experimental/stale_while_revalidate/test_server.js diff --git a/NOTICE b/NOTICE index d72681fe40d..9728ab2fa35 100644 --- a/NOTICE +++ b/NOTICE @@ -35,11 +35,6 @@ Copyright (C) 2014 LinkedIn ~~~ -stale_while_revalidate Plugin developed by OmniTI on behalf of Oregon Health & Science University -Copyright (C) 2012 Oregon Health & Science University - -~~~ - cache-key-genid Plugin developed by GoDaddy Copyright (C) 2013 GoDaddy Operating Company, LLC diff --git a/ci/rat-regex.txt b/ci/rat-regex.txt index 32b0f3f4c2e..553902fde1b 100644 --- a/ci/rat-regex.txt +++ b/ci/rat-regex.txt @@ -48,7 +48,6 @@ README.*$ ^hashtable\.cc$ ^logstats\.summary$ ^test_AIO.sample$ -^stale_while_revalidate.conf$ ^stats\.memo$ port\.h ^diags.log^ diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po index 76d9172f62f..187dd3eed86 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/index.en.po @@ -316,10 +316,6 @@ msgstr "" msgid "Populate request headers with SSL session information." msgstr "" -#: ../../../admin-guide/plugins/index.en.rst:189 -msgid ":doc:`Stale While Revalidate `" -msgstr "" - #: ../../../admin-guide/plugins/index.en.rst:189 msgid "Refresh content asynchronously while serving stale data." msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/stale_while_revalidate.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/stale_while_revalidate.en.po deleted file mode 100644 index 1c95ed1c77d..00000000000 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/stale_while_revalidate.en.po +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: Apache Traffic Server 6.2\n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2016-06-30 14:07+0900\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=utf-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.2.0\n" - -#: ../../../admin-guide/plugins/stale_while_revalidate.en.rst:4 -msgid "Stale While Revalidate Plugin" -msgstr "" - -#: ../../../admin-guide/plugins/stale_while_revalidate.en.rst:23 -msgid "refresh content asynchronously while serving stale data" -msgstr "" diff --git a/plugins/Makefile.am b/plugins/Makefile.am index f942ae7bf25..9818f9c00b3 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -74,7 +74,6 @@ include experimental/mp4/Makefile.inc include experimental/server_push_preload/Makefile.inc include experimental/slice/Makefile.inc include experimental/sslheaders/Makefile.inc -include experimental/stale_while_revalidate/Makefile.inc include experimental/stream_editor/Makefile.inc include experimental/system_stats/Makefile.inc include experimental/traffic_dump/Makefile.inc diff --git a/plugins/experimental/stale_while_revalidate/Makefile.inc b/plugins/experimental/stale_while_revalidate/Makefile.inc deleted file mode 100644 index 7c8fce6fe39..00000000000 --- a/plugins/experimental/stale_while_revalidate/Makefile.inc +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pkglib_LTLIBRARIES += experimental/stale_while_revalidate/stale_while_revalidate.la - -experimental_stale_while_revalidate_stale_while_revalidate_la_SOURCES = \ - experimental/stale_while_revalidate/stale_while_revalidate.c diff --git a/plugins/experimental/stale_while_revalidate/Makefile.tsxs b/plugins/experimental/stale_while_revalidate/Makefile.tsxs deleted file mode 100644 index 805c458f078..00000000000 --- a/plugins/experimental/stale_while_revalidate/Makefile.tsxs +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PLUGIN=stale_while_revalidate -TSXS?=tsxs - -all: ${PLUGIN}.c - $(TSXS) -v -o ${PLUGIN}.so $? - -install: - $(TSXS) -i -o ${PLUGIN}.so - -clean: - rm -f *.lo *.so diff --git a/plugins/experimental/stale_while_revalidate/README b/plugins/experimental/stale_while_revalidate/README deleted file mode 100644 index d85cb7e795b..00000000000 --- a/plugins/experimental/stale_while_revalidate/README +++ /dev/null @@ -1,36 +0,0 @@ -************************************************************ - THIS PLUGIN IS DEPRECATED IN FAVOR OF CHANGE TO CORE -************************************************************ - -You can still run `make && make install` in this directory -to build and install the plugin, however it will not be -built or installed by default. - -************************************************************ - -This plugin was sponsored by Oregon Health & Science University. - -Quick install: - -Make sure devel packages for traffic-server are installed. -Make sure that 'tsxs' is in your path. - - make -f Makefile.tsxs - make -f Makefile.tsxs install - -Add 'stale_while_revalidate.so' to plugin.config. - -Restart traffic-server. - -test_server.js is a test server written in JavaScript and -meant to be run under node.js. - -Logging: - -You can enable logging by adding parameters to the plugin.config -line for the plugin. - ---log-all enable all logging ---log-stale-while-revalidate enable logging of stale-while-revalidate ---log-stale-if-error enable logging of stale-if-error ---log-filename set the filename to log to (.log) diff --git a/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c b/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c deleted file mode 100644 index fdcc4669d92..00000000000 --- a/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c +++ /dev/null @@ -1,769 +0,0 @@ -/** @file - - Implements RFC 5861 (HTTP Cache-Control Extensions for Stale Content) - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tscore/ink_defs.h" -#include "ts/ts.h" -#include "ts/experimental.h" - -#define PLUGIN_NAME "stale_while_revalidate" - -static const char HTTP_VALUE_STALE_WHILE_REVALIDATE[] = "stale-while-revalidate"; -static const char HTTP_VALUE_STALE_IF_ERROR[] = "stale-if-error"; -static const char HTTP_VALUE_STALE_WARNING[] = "110 Response is stale"; - -typedef struct { - TSTextLogObject object; - bool all, stale_if_error, stale_while_revalidate; - char *filename; -} log_info_t; - -typedef struct { - void *troot; - TSMutex troot_mutex; - int txn_slot; - time_t stale_if_error_override; - log_info_t log_info; -} config_t; - -typedef struct { - time_t date, stale_while_revalidate, stale_on_error, max_age; -} CachedHeaderInfo; - -typedef struct { - char *effective_url; - TSMBuffer buf; - TSMLoc http_hdr_loc; - union { - struct sockaddr sa; - struct sockaddr_in sin; - struct sockaddr_in6 sin6; - } client_addr; -} RequestInfo; - -typedef struct { - TSMBuffer buf; - TSMLoc http_hdr_loc; - TSHttpParser parser; - bool parsed; - TSHttpStatus status; -} ResponseInfo; - -typedef struct { - TSHttpTxn txn; - TSCont main_cont; - bool async_req; - TSIOBuffer req_io_buf, resp_io_buf; - TSIOBufferReader req_io_buf_reader, resp_io_buf_reader; - TSVIO r_vio, w_vio; - TSVConn vconn; - RequestInfo *req_info; - ResponseInfo *resp_info; - time_t txn_start; - config_t *plugin_config; -} StateInfo; - -static int -xstrcmp(const void *a, const void *b) -{ - return strcmp((const char *)a, (const char *)b); -} - -static ResponseInfo * -create_response_info(void) -{ - ResponseInfo *resp_info; - - resp_info = (ResponseInfo *)TSmalloc(sizeof(ResponseInfo)); - - resp_info->buf = TSMBufferCreate(); - resp_info->http_hdr_loc = TSHttpHdrCreate(resp_info->buf); - resp_info->parser = TSHttpParserCreate(); - resp_info->parsed = false; - - return resp_info; -} - -static void -free_response_info(ResponseInfo *resp_info) -{ - TSHandleMLocRelease(resp_info->buf, TS_NULL_MLOC, resp_info->http_hdr_loc); - TSMBufferDestroy(resp_info->buf); - TSHttpParserDestroy(resp_info->parser); - TSfree(resp_info); -} - -static RequestInfo * -create_request_info(TSHttpTxn txn) -{ - RequestInfo *req_info = NULL; - char *url; - int url_len; - TSMBuffer buf; - TSMLoc loc; - - const struct sockaddr *sa; - - if (TSHttpTxnClientReqGet(txn, &buf, &loc) == TS_SUCCESS) { - req_info = (RequestInfo *)TSmalloc(sizeof(RequestInfo)); - memset(req_info, 0, sizeof(RequestInfo)); - - url = TSHttpTxnEffectiveUrlStringGet(txn, &url_len); - req_info->effective_url = TSstrndup(url, url_len); - TSfree(url); - - req_info->buf = TSMBufferCreate(); - TSHttpHdrClone(req_info->buf, buf, loc, &(req_info->http_hdr_loc)); - TSHandleMLocRelease(buf, TS_NULL_MLOC, loc); - - sa = TSHttpTxnClientAddrGet(txn); - switch (sa->sa_family) { - case AF_INET: - memcpy(&req_info->client_addr.sin, sa, sizeof(struct sockaddr_in)); - break; - case AF_INET6: - memcpy(&req_info->client_addr.sin6, sa, sizeof(struct sockaddr_in6)); - break; - default: - break; - } - } - - return req_info; -} - -static void -free_request_info(RequestInfo *req_info) -{ - TSfree(req_info->effective_url); - TSHandleMLocRelease(req_info->buf, TS_NULL_MLOC, req_info->http_hdr_loc); - TSMBufferDestroy(req_info->buf); - TSfree(req_info); -} - -static void -free_request_state(StateInfo *state) -{ -#if defined(DEBUG) - int verify = 1; -#else - int verify = TSIsDebugTagSet(PLUGIN_NAME); -#endif - - // Verify that the effective URL of this state object has been removed before we delete the state. - if (verify) { - void *ptr; - - TSMutexLock(state->plugin_config->troot_mutex); - ptr = tfind(state->req_info->effective_url, &(state->plugin_config->troot), xstrcmp); - TSMutexUnlock(state->plugin_config->troot_mutex); - - if (ptr) { - TSReleaseAssert(ptr != state->req_info->effective_url); - } - } - - if (state->resp_info) { - free_response_info(state->resp_info); - } - - free_request_info(state->req_info); - TSfree(state); -} - -static CachedHeaderInfo * -get_cached_header_info(TSHttpTxn txn) -{ - CachedHeaderInfo *chi; - TSMBuffer cr_buf; - TSMLoc cr_hdr_loc, cr_date_loc, cr_cache_control_loc, cr_cache_control_dup_loc; - int cr_cache_control_count, val_len, i; - char *value, *ptr; - - chi = (CachedHeaderInfo *)TSmalloc(sizeof(CachedHeaderInfo)); - memset(chi, 0, sizeof(CachedHeaderInfo)); - - if (TSHttpTxnCachedRespGet(txn, &cr_buf, &cr_hdr_loc) == TS_SUCCESS) { - cr_date_loc = TSMimeHdrFieldFind(cr_buf, cr_hdr_loc, TS_MIME_FIELD_DATE, TS_MIME_LEN_DATE); - if (cr_date_loc != TS_NULL_MLOC) { - TSDebug(PLUGIN_NAME, "Found a date"); - chi->date = TSMimeHdrFieldValueDateGet(cr_buf, cr_hdr_loc, cr_date_loc); - TSHandleMLocRelease(cr_buf, cr_hdr_loc, cr_date_loc); - } - - cr_cache_control_loc = TSMimeHdrFieldFind(cr_buf, cr_hdr_loc, TS_MIME_FIELD_CACHE_CONTROL, TS_MIME_LEN_CACHE_CONTROL); - - while (cr_cache_control_loc != TS_NULL_MLOC) { - TSDebug(PLUGIN_NAME, "Found cache-control"); - cr_cache_control_count = TSMimeHdrFieldValuesCount(cr_buf, cr_hdr_loc, cr_cache_control_loc); - - for (i = 0; i < cr_cache_control_count; i++) { - value = (char *)TSMimeHdrFieldValueStringGet(cr_buf, cr_hdr_loc, cr_cache_control_loc, i, &val_len); - ptr = value; - - if (strncmp(value, TS_HTTP_VALUE_MAX_AGE, TS_HTTP_LEN_MAX_AGE) == 0) { - TSDebug(PLUGIN_NAME, "Found max-age"); - ptr += TS_HTTP_LEN_MAX_AGE; - if (*ptr == '=') { - ptr++; - chi->max_age = atol(ptr); - } else { - ptr = TSstrndup(value, TS_HTTP_LEN_MAX_AGE + 2); - TSDebug(PLUGIN_NAME, "This is what I found: %s", ptr); - TSfree(ptr); - } - } else if (strncmp(value, HTTP_VALUE_STALE_WHILE_REVALIDATE, strlen(HTTP_VALUE_STALE_WHILE_REVALIDATE)) == 0) { - TSDebug(PLUGIN_NAME, "Found stale-while-revalidate"); - ptr += strlen(HTTP_VALUE_STALE_WHILE_REVALIDATE); - if (*ptr == '=') { - ptr++; - chi->stale_while_revalidate = atol(ptr); - } - } else if (strncmp(value, HTTP_VALUE_STALE_IF_ERROR, strlen(HTTP_VALUE_STALE_IF_ERROR)) == 0) { - TSDebug(PLUGIN_NAME, "Found stale-on-error"); - ptr += strlen(HTTP_VALUE_STALE_IF_ERROR); - if (*ptr == '=') { - ptr++; - chi->stale_on_error = atol(ptr); - } - } else { - TSDebug(PLUGIN_NAME, "Unknown field value"); - } - } - - cr_cache_control_dup_loc = TSMimeHdrFieldNextDup(cr_buf, cr_hdr_loc, cr_cache_control_loc); - TSHandleMLocRelease(cr_buf, cr_hdr_loc, cr_cache_control_loc); - cr_cache_control_loc = cr_cache_control_dup_loc; - } - TSHandleMLocRelease(cr_buf, TS_NULL_MLOC, cr_hdr_loc); - } - - return chi; -} - -static void -parse_response(StateInfo *state) -{ - TSIOBufferBlock block; - TSParseResult pr = TS_PARSE_CONT; - int64_t avail; - char *start; - - block = TSIOBufferReaderStart(state->resp_io_buf_reader); - - while ((pr == TS_PARSE_CONT) && (block != NULL)) { - start = (char *)TSIOBufferBlockReadStart(block, state->resp_io_buf_reader, &avail); - if (avail > 0) { - pr = TSHttpHdrParseResp(state->resp_info->parser, state->resp_info->buf, state->resp_info->http_hdr_loc, - (const char **)&start, (const char *)(start + avail)); - } - block = TSIOBufferBlockNext(block); - } - - if (pr != TS_PARSE_CONT) { - state->resp_info->status = TSHttpHdrStatusGet(state->resp_info->buf, state->resp_info->http_hdr_loc); - state->resp_info->parsed = true; - TSDebug(PLUGIN_NAME, "HTTP Status: %d", state->resp_info->status); - } -} - -static int -consume_resource(TSCont cont, TSEvent event ATS_UNUSED, void *edata ATS_UNUSED) -{ - StateInfo *state; - int64_t avail; - TSVConn vconn; - TSMLoc url_loc; - int lookup_count; - - vconn = (TSVConn)edata; - state = (StateInfo *)TSContDataGet(cont); - - switch (event) { - case TS_EVENT_VCONN_WRITE_READY: - // We shouldn't get here because we specify the exact size of the buffer. - TSDebug(PLUGIN_NAME, "Write Ready"); - break; - case TS_EVENT_VCONN_WRITE_COMPLETE: - TSDebug(PLUGIN_NAME, "Write Complete"); - // TSDebug(PLUGIN_NAME, "TSVConnShutdown()"); - // TSVConnShutdown(state->vconn, 0, 1); - // TSVIOReenable(state->w_vio); - break; - case TS_EVENT_VCONN_READ_READY: - TSDebug(PLUGIN_NAME, "Read Ready"); - - if ((state->resp_info) && !state->resp_info->parsed) { - parse_response(state); - } - - // Consume data - avail = TSIOBufferReaderAvail(state->resp_io_buf_reader); - TSIOBufferReaderConsume(state->resp_io_buf_reader, avail); - TSVIONDoneSet(state->r_vio, TSVIONDoneGet(state->r_vio) + avail); - TSVIOReenable(state->r_vio); - break; - case TS_EVENT_VCONN_READ_COMPLETE: - case TS_EVENT_VCONN_EOS: - case TS_EVENT_VCONN_INACTIVITY_TIMEOUT: - if (event == TS_EVENT_VCONN_INACTIVITY_TIMEOUT) { - TSDebug(PLUGIN_NAME, "Inactivity Timeout"); - TSVConnAbort(vconn, TS_VC_CLOSE_ABORT); - } else { - if (event == TS_EVENT_VCONN_READ_COMPLETE) { - TSDebug(PLUGIN_NAME, "Read Complete"); - } else if (event == TS_EVENT_VCONN_EOS) { - TSDebug(PLUGIN_NAME, "EOS"); - } - TSVConnClose(state->vconn); - } - - if ((state->resp_info) && !state->resp_info->parsed) { - parse_response(state); - } - - // Consume data - avail = TSIOBufferReaderAvail(state->resp_io_buf_reader); - TSIOBufferReaderConsume(state->resp_io_buf_reader, avail); - TSVIONDoneSet(state->r_vio, TSVIONDoneGet(state->r_vio) + avail); - if (state->async_req) { - TSDebug(PLUGIN_NAME, "Unlock URL"); - TSMutexLock(state->plugin_config->troot_mutex); - tdelete(state->req_info->effective_url, &(state->plugin_config->troot), xstrcmp); - TSMutexUnlock(state->plugin_config->troot_mutex); - } else { - TSDebug(PLUGIN_NAME, "In sync path. setting fresh and re-enabling"); - TSHttpTxnCacheLookupCountGet(state->txn, &lookup_count); - if (state->resp_info && ((state->resp_info->status == 500) || - ((state->resp_info->status >= 502) && (state->resp_info->status <= 504)) || lookup_count > 2)) { - TSDebug(PLUGIN_NAME, "Sending stale data as fresh"); - if (state->plugin_config->log_info.object && - (state->plugin_config->log_info.all || state->plugin_config->log_info.stale_if_error)) { - CachedHeaderInfo *chi = get_cached_header_info(state->txn); - TSTextLogObjectWrite(state->plugin_config->log_info.object, "stale-if-error: %d - %d < %d + %d %s", (int)state->txn_start, - (int)chi->date, (int)chi->max_age, (int)chi->stale_on_error, state->req_info->effective_url); - TSfree(chi); - } - TSHttpTxnHookAdd(state->txn, TS_HTTP_SEND_RESPONSE_HDR_HOOK, state->main_cont); - TSHttpTxnCacheLookupStatusSet(state->txn, TS_CACHE_LOOKUP_HIT_FRESH); - } else { - TSDebug(PLUGIN_NAME, "Attempting new cache lookup"); - if (TSHttpHdrUrlGet(state->req_info->buf, state->req_info->http_hdr_loc, &url_loc) == TS_SUCCESS) { - // ToDo: This is now completely broken, and we need a different logic here than the second cache lookup - // TSHttpTxnNewCacheLookupDo(state->txn, state->req_info->buf, url_loc); - TSHandleMLocRelease(state->req_info->buf, state->req_info->http_hdr_loc, url_loc); - } else { - TSError("[%s] Error getting the URL from the transaction", PLUGIN_NAME); - } - // TODO add txn translation hook and pass result along, maybe inside continuation? - // TSHttpTxnHookAdd(state->txn, TS_HTTP_RESPONSE_TRANSFORM_HOOK, TSTransformCreate(replace_transform, state->txn)); - } - TSHttpTxnReenable(state->txn, TS_EVENT_HTTP_CONTINUE); - } - - // XXX Free as part of StateInfo. - TSIOBufferReaderFree(state->req_io_buf_reader); - TSIOBufferDestroy(state->req_io_buf); - TSIOBufferReaderFree(state->resp_io_buf_reader); - TSIOBufferDestroy(state->resp_io_buf); - - free_request_state(state); - TSContDestroy(cont); - break; - default: - TSError("[%s] Unknown event %d", PLUGIN_NAME, event); - break; - } - - return 0; -} - -static int -fetch_resource(TSCont cont, TSEvent event ATS_UNUSED, void *edata ATS_UNUSED) -{ - StateInfo *state; - TSCont consume_cont; - TSMLoc connection_hdr_loc, connection_hdr_dup_loc; - - state = (StateInfo *)TSContDataGet(cont); - - // li = (RequestInfo *) edata; - if (state->async_req) { - TSMutex mtx = state->plugin_config->troot_mutex; - TSMutexLock(mtx); - - // If already doing async lookup lets just close shop and go home - if (tfind(state->req_info->effective_url, &(state->plugin_config->troot), xstrcmp) != NULL) { - TSDebug(PLUGIN_NAME, "Looks like an async is already in progress"); - free_request_state(state); - state = NULL; - TSContDataSet(cont, NULL); - } - // Otherwise lets do the lookup! - else { - // Lock in tree - TSDebug(PLUGIN_NAME, "Locking URL"); - tsearch(state->req_info->effective_url, &(state->plugin_config->troot), xstrcmp); - } - - TSMutexUnlock(mtx); - } - - if (state) { - TSDebug(PLUGIN_NAME, "Lets do the lookup"); - consume_cont = TSContCreate(consume_resource, TSMutexCreate()); - TSContDataSet(consume_cont, (void *)state); - - if (state->async_req) { - state->resp_info = NULL; - } else { - state->resp_info = create_response_info(); - } - - TSDebug(PLUGIN_NAME, "Set Connection: close"); - connection_hdr_loc = - TSMimeHdrFieldFind(state->req_info->buf, state->req_info->http_hdr_loc, TS_MIME_FIELD_CONNECTION, TS_MIME_LEN_CONNECTION); - - while (connection_hdr_loc != TS_NULL_MLOC) { - TSDebug(PLUGIN_NAME, "Found old Connection hdr"); - - connection_hdr_dup_loc = TSMimeHdrFieldNextDup(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc); - TSMimeHdrFieldRemove(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc); - TSMimeHdrFieldDestroy(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc); - TSHandleMLocRelease(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc); - connection_hdr_loc = connection_hdr_dup_loc; - } - - // This seems to have little effect - TSDebug(PLUGIN_NAME, "Creating Connection hdr"); - TSMimeHdrFieldCreateNamed(state->req_info->buf, state->req_info->http_hdr_loc, TS_MIME_FIELD_CONNECTION, TS_MIME_LEN_CONNECTION, - &connection_hdr_loc); - TSMimeHdrFieldValueStringInsert(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc, -1, - TS_HTTP_VALUE_CLOSE, TS_HTTP_LEN_CLOSE); - TSMimeHdrFieldAppend(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc); - TSHandleMLocRelease(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc); - - /* - TSDebug(PLUGIN_NAME, "Creating @stale_while_revalidate header"); - TSMimeHdrFieldCreateNamed(state->req_info->buf, state->req_info->http_hdr_loc, TS_MIME_FIELD_CONNECTION, TS_MIME_LEN_CONNECTION, - &connection_hdr_loc); - TSMimeHdrFieldValueStringInsert(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc, -1, - TS_HTTP_VALUE_CLOSE, TS_HTTP_LEN_CLOSE); - TSMimeHdrFieldAppend(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc); - TSHandleMLocRelease(state->req_info->buf, state->req_info->http_hdr_loc, connection_hdr_loc); - */ - - TSDebug(PLUGIN_NAME, "Create Buffers"); - state->req_io_buf = TSIOBufferCreate(); - state->req_io_buf_reader = TSIOBufferReaderAlloc(state->req_io_buf); - state->resp_io_buf = TSIOBufferCreate(); - state->resp_io_buf_reader = TSIOBufferReaderAlloc(state->resp_io_buf); - - TSHttpHdrPrint(state->req_info->buf, state->req_info->http_hdr_loc, state->req_io_buf); - TSIOBufferWrite(state->req_io_buf, "\r\n", 2); - - state->vconn = TSHttpConnectWithPluginId(&state->req_info->client_addr.sa, PLUGIN_NAME, 0); - - state->r_vio = TSVConnRead(state->vconn, consume_cont, state->resp_io_buf, INT64_MAX); - state->w_vio = - TSVConnWrite(state->vconn, consume_cont, state->req_io_buf_reader, TSIOBufferReaderAvail(state->req_io_buf_reader)); - } - - TSContDestroy(cont); - - return 0; -} - -static int -is_swr_transaction(TSHttpTxn txn) -{ - const char *tag = TSHttpTxnPluginTagGet(txn); - return tag && strcmp(tag, PLUGIN_NAME) == 0; -} - -static int -main_plugin(TSCont cont, TSEvent event, void *edata) -{ - TSHttpTxn txn = (TSHttpTxn)edata; - int status, lookup_count; - TSCont fetch_cont; - StateInfo *state; - TSMBuffer buf; - TSMLoc loc, warn_loc; - config_t *plugin_config; - - switch (event) { - case TS_EVENT_HTTP_READ_REQUEST_HDR: - - if (is_swr_transaction(txn)) { - TSHttpTxnHookAdd(txn, TS_HTTP_READ_RESPONSE_HDR_HOOK, cont); - } else { - state = TSmalloc(sizeof(StateInfo)); - memset(state, 0, sizeof(StateInfo)); - - state->plugin_config = (config_t *)TSContDataGet(cont); - state->req_info = create_request_info(txn); - if (state->req_info != NULL) { - time(&state->txn_start); - - TSHttpTxnArgSet(txn, state->plugin_config->txn_slot, (void *)state); - TSHttpTxnHookAdd(txn, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, cont); - - TSDebug(PLUGIN_NAME, "tracking state %p from txn %p for %s", state, txn, state->req_info->effective_url); - } else { - free_request_state(state); - } - } - - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - break; - - case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: - - plugin_config = (config_t *)TSContDataGet(cont); - state = (StateInfo *)TSHttpTxnArgGet(txn, plugin_config->txn_slot); - - // NOTE: It is possible to receive the cache lookup - // hook more than once if there is contention on the - // cache write lock. Just bail if we already nuked - // the StateInfo. - if (state == NULL) { - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - break; - } - - TSHttpTxnCacheLookupCountGet(txn, &lookup_count); - if (TSHttpTxnCacheLookupStatusGet(txn, &status) == TS_SUCCESS) { - TSDebug(PLUGIN_NAME, "cache status %d, count %d from txn %p state %p", status, lookup_count, txn, state); - - // Are we stale? - if (status == TS_CACHE_LOOKUP_HIT_STALE) { - CachedHeaderInfo *chi; - - TSDebug(PLUGIN_NAME, "CacheLookupStatus is STALE"); - // Get headers - chi = get_cached_header_info(txn); - - if (state->plugin_config->stale_if_error_override > chi->stale_on_error) { - chi->stale_on_error = state->plugin_config->stale_if_error_override; - } - - if ((state->txn_start - chi->date) < (chi->max_age + chi->stale_while_revalidate)) { - TSDebug(PLUGIN_NAME, "Looks like we can return fresh info and validate in the background"); - if (state->plugin_config->log_info.object && - (state->plugin_config->log_info.all || state->plugin_config->log_info.stale_while_revalidate)) { - TSTextLogObjectWrite(state->plugin_config->log_info.object, "stale-while-revalidate: %d - %d < %d + %d %s", - (int)state->txn_start, (int)chi->date, (int)chi->max_age, (int)chi->stale_while_revalidate, - state->req_info->effective_url); - } - - TSHttpTxnConfigIntSet(txn, TS_CONFIG_HTTP_INSERT_AGE_IN_RESPONSE, 1); - TSHttpTxnCacheLookupStatusSet(txn, TS_CACHE_LOOKUP_HIT_FRESH); - - TSDebug(PLUGIN_NAME, "set state as async"); - state->async_req = true; - - // Set warning header - TSHttpTxnHookAdd(txn, TS_HTTP_SEND_RESPONSE_HDR_HOOK, cont); - - fetch_cont = TSContCreate(fetch_resource, TSMutexCreate()); - TSContDataSet(fetch_cont, (void *)state); - TSContScheduleOnPool(fetch_cont, 0, TS_THREAD_POOL_NET); - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - } else if ((state->txn_start - chi->date) < (chi->max_age + chi->stale_on_error)) { - TSDebug(PLUGIN_NAME, "Looks like we can return fresh data on 500 error"); - TSHttpTxnConfigIntSet(txn, TS_CONFIG_HTTP_INSERT_AGE_IN_RESPONSE, 1); - - // lookup sync - state->async_req = false; - state->txn = txn; - state->main_cont = cont; // we need this for the warning header callback. not sure i like it, but it works. - fetch_cont = TSContCreate(fetch_resource, TSMutexCreate()); - TSContDataSet(fetch_cont, (void *)state); - TSContScheduleOnPool(fetch_cont, 0, TS_THREAD_POOL_NET); - } else { - TSDebug(PLUGIN_NAME, "No love? now: %d date: %d max-age: %d swr: %d soe: %d", (int)state->txn_start, (int)chi->date, - (int)chi->max_age, (int)chi->stale_while_revalidate, (int)chi->stale_on_error); - if (lookup_count == 1) { - TSDebug(PLUGIN_NAME, "freeing state %p from txn %p", state, txn); - TSHttpTxnArgSet(txn, state->plugin_config->txn_slot, NULL); - free_request_state(state); - } - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - } - - TSfree(chi); - } else { - TSDebug(PLUGIN_NAME, "Not Stale!"); - if (lookup_count == 1) { - TSDebug(PLUGIN_NAME, "freeing state %p from txn %p", state, txn); - TSHttpTxnArgSet(txn, state->plugin_config->txn_slot, NULL); - free_request_state(state); - } - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - } - } else { - TSDebug(PLUGIN_NAME, "Could not get CacheLookupStatus"); - if (lookup_count == 1) { - TSDebug(PLUGIN_NAME, "freeing state %p from txn %p", state, txn); - TSHttpTxnArgSet(txn, state->plugin_config->txn_slot, NULL); - free_request_state(state); - } - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - } - break; - - case TS_EVENT_HTTP_READ_RESPONSE_HDR: - // In this continuation we only hook responses for background - // requests that we issued ourselves. If the origin went away, - // we don't want to replace a stale cache object with an error. - TSAssert(is_swr_transaction(txn)); - - if (TS_SUCCESS == TSHttpTxnServerRespGet(txn, &buf, &loc)) { - switch (TSHttpHdrStatusGet(buf, loc)) { - case TS_HTTP_STATUS_INTERNAL_SERVER_ERROR: - case TS_HTTP_STATUS_BAD_GATEWAY: - case TS_HTTP_STATUS_SERVICE_UNAVAILABLE: - case TS_HTTP_STATUS_GATEWAY_TIMEOUT: - TSDebug(PLUGIN_NAME, "marking background request no-store"); - TSHttpTxnServerRespNoStoreSet(txn, 1); - break; - default: - break; - } - - TSHandleMLocRelease(buf, TS_NULL_MLOC, loc); - } - - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - break; - - case TS_EVENT_HTTP_SEND_RESPONSE_HDR: - TSDebug(PLUGIN_NAME, "set warning header"); - if (TSHttpTxnClientRespGet(txn, &buf, &loc) == TS_SUCCESS) { - TSMimeHdrFieldCreateNamed(buf, loc, TS_MIME_FIELD_WARNING, TS_MIME_LEN_WARNING, &warn_loc); - TSMimeHdrFieldValueStringInsert(buf, loc, warn_loc, -1, HTTP_VALUE_STALE_WARNING, strlen(HTTP_VALUE_STALE_WARNING)); - TSMimeHdrFieldAppend(buf, loc, warn_loc); - TSHandleMLocRelease(buf, loc, warn_loc); - TSHandleMLocRelease(buf, TS_NULL_MLOC, loc); - } else { - TSError("[%s] Error while getting response from txn", PLUGIN_NAME); - } - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - break; - default: - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - break; - } - - return 0; -} - -void -TSPluginInit(int argc, const char *argv[]) -{ - config_t *plugin_config; - TSPluginRegistrationInfo info; - TSCont main_cont; - - info.plugin_name = PLUGIN_NAME; - info.vendor_name = "Apache Software Foundation"; - info.support_email = "dev@trafficserver.apache.org"; - - if (TSPluginRegister(&info) != TS_SUCCESS) { - TSError("[%s] Plugin registration failed.\n", PLUGIN_NAME); - - return; - } else { - TSDebug(PLUGIN_NAME, "Plugin registration succeeded"); - } - - if (argc > 1) { - plugin_config = TSmalloc(sizeof(config_t)); - - plugin_config->troot = NULL; - plugin_config->troot_mutex = TSMutexCreate(); - plugin_config->stale_if_error_override = 0; - plugin_config->log_info.object = NULL; - plugin_config->log_info.all = false; - plugin_config->log_info.stale_if_error = false; - plugin_config->log_info.stale_while_revalidate = false; - plugin_config->log_info.filename = PLUGIN_NAME; - - int c; - static const struct option longopts[] = {{"log-all", no_argument, NULL, 'a'}, - {"log-stale-while-revalidate", no_argument, NULL, 'r'}, - {"log-stale-if-error", no_argument, NULL, 'e'}, - {"log-filename", required_argument, NULL, 'f'}, - {"force-stale-if-error", required_argument, NULL, 'E'}, - {NULL, 0, NULL, 0}}; - - while ((c = getopt_long(argc, (char *const *)argv, "aref:E:", longopts, NULL)) != -1) { - switch (c) { - case 'a': - plugin_config->log_info.all = true; - break; - case 'r': - plugin_config->log_info.stale_while_revalidate = true; - break; - case 'e': - plugin_config->log_info.stale_if_error = true; - break; - case 'f': - plugin_config->log_info.filename = strdup(optarg); - break; - case 'E': - plugin_config->stale_if_error_override = atoi(optarg); - break; - default: - break; - } - } - - if (plugin_config->log_info.all || plugin_config->log_info.stale_while_revalidate || plugin_config->log_info.stale_if_error) { - if (TSTextLogObjectCreate(plugin_config->log_info.filename, TS_LOG_MODE_ADD_TIMESTAMP, &(plugin_config->log_info.object)) != - TS_SUCCESS) { - TSError("[%s] Error getting the URL from the transaction", PLUGIN_NAME); - TSfree(plugin_config); - return; - } - } - - // proxy.config.http.insert_age_in_response - TSHttpTxnArgIndexReserve(PLUGIN_NAME, "txn state info", &(plugin_config->txn_slot)); - main_cont = TSContCreate(main_plugin, NULL); - TSContDataSet(main_cont, (void *)plugin_config); - TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, main_cont); - - TSDebug(PLUGIN_NAME, "Plugin Init Complete"); - } -} diff --git a/plugins/experimental/stale_while_revalidate/test_server.js b/plugins/experimental/stale_while_revalidate/test_server.js deleted file mode 100755 index 6aebc27adaa..00000000000 --- a/plugins/experimental/stale_while_revalidate/test_server.js +++ /dev/null @@ -1,42 +0,0 @@ -#! /usr/bin/env node - -/** @file - - Implements Simple HTTP test server for the stale_while_revalidate plugin - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -var http = require('http'); -var url = require('url'); - -http.createServer(function (request, response) { - console.log(Date()); - setTimeout( function (req, res) { - console.log(req.headers); - console.log(url.parse(req.url)); - res.writeHead(200, {'Content-Type': 'text/plain', 'Cache-Control': 'max-age=5, stale-while-revalidate=55'}); - //res.writeHead(500, {'Content-Type': 'text/plain', 'Cache-Control': 'max-age=5, stale-if-error=555'}); - res.end(Date() + '\n'); - console.log(Date() + '\n'); - }, 50, request, response); -}).listen(8081, '127.0.0.1'); - -console.log('Proxy running at http://127.0.0.1:8080/'); -console.log('Server running at http://127.0.0.1:8081/'); From ffa4503c0f61395aab470934efccea6ae3164c35 Mon Sep 17 00:00:00 2001 From: Leif Hedstrom Date: Fri, 19 Jul 2019 13:47:30 -0600 Subject: [PATCH 7/7] Deprecate the mysql_remap plugin. See #5395 --- doc/admin-guide/plugins/mysql_remap.en.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/admin-guide/plugins/mysql_remap.en.rst b/doc/admin-guide/plugins/mysql_remap.en.rst index f87ace02794..39ff9bf1d76 100644 --- a/doc/admin-guide/plugins/mysql_remap.en.rst +++ b/doc/admin-guide/plugins/mysql_remap.en.rst @@ -20,6 +20,12 @@ MySQL Remap Plugin specific language governing permissions and limitations under the License. +.. Note:: + + This plugin is *deprecated* as of v9.0.0, and should not be used! The issue + is around the blocking APIs used here; If this functionality is needed, you + will need to implement it using something non-blocking, such as REDIS. + This is a basic plugin for doing dynamic "remaps" from a database. It essentially rewrites the incoming request's Host header / origin server