From 5190bf79a28c19da11f4c5cb6f743accf8f36f69 Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Wed, 21 Aug 2019 16:11:48 -0700 Subject: [PATCH 1/4] cachekey: added --canonical-prefix parameter In certain use-cases when calculating the prefix (the initial value of the new cache key) we need to have the scheme, host and port in their original form from the request URI, i.e. when hosting.config is used the cache key is expected to contain a valid URI authority element used for volume selection. More details about the new parameter and its functionality can be found in doc/admin-guide/plugins/cachekey.en.rst (cherry picked from commit 0f86efc7678248c09f87ec8b99bc026456edc3f7) --- doc/admin-guide/plugins/cachekey.en.rst | 21 ++++-- plugins/cachekey/cachekey.cc | 90 ++++++++++++++++++------- plugins/cachekey/cachekey.h | 3 +- plugins/cachekey/configs.cc | 11 +++ plugins/cachekey/configs.h | 6 ++ plugins/cachekey/plugin.cc | 2 +- 6 files changed, 102 insertions(+), 31 deletions(-) diff --git a/doc/admin-guide/plugins/cachekey.en.rst b/doc/admin-guide/plugins/cachekey.en.rst index 5d8b6a0d272..c82e8e4c4f8 100644 --- a/doc/admin-guide/plugins/cachekey.en.rst +++ b/doc/admin-guide/plugins/cachekey.en.rst @@ -70,20 +70,29 @@ Cache key structure and related plugin parameters :: - Optional components | ┌─────────────────┬──────────────────┬──────────────────────┐ + Optional components | ┌─────────────────┬────────────── ───┬──────────────────────┐ (included in this order) | │ --static-prefix | --capture-prefix │ --capture-prefix-uri │ | ├─────────────────┴──────────────────┴──────────────────────┤ - Default values if no | │ /host/port | + Default values if no | │ /host/port or scheme://host:port (see the table below) | optional components | └───────────────────────────────────────────────────────────┘ configured | + ┌────────────────────┬─────────────────────────┬──────────────────────┐ + │ --canonical-prefix | default value if no │ input used for │ + │ | prefix parameters used │ --capture-prefix │ + ├────────────────────┴─────────────────────────┴──────────────────────┤ + │ fasle | /host/port | host:port | + ├────────────────────┴─────────────────────────┴──────────────────────┤ + │ true | scheme://host:port | scheme://host:port | + └──────────────────────────────────────────────┴──────────────────────┘ + + * ``--static-prefix=`` (default: empty string) - if specified and not an empty string the ```` will be added to the cache key. -* ``--capture-prefix=`` (default: empty string) - if specified and not empty then strings are captured from ``host:port`` based on the ```` and are added to the cache key. +* ``--capture-prefix=`` (default: empty string) - if specified and not empty then strings are captured based on the value of ``--canonical-prefix`` parameter (see the table above) and ```` and are added to the cache key. * ``--capture-prefix-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the cache key. * If any of the "Prefix" related plugin parameters are used together in the plugin configuration they are added to the cache key in the order shown in the diagram. -* ``--remove-prefix= default prefix */ + append(getCanonicalUrl(_buf, _url, canonicalPrefix, /* provideDefaultKey */ true), /* useSeparator */ false); CacheKeyDebug("added default prefix, key: '%s'", _key.c_str()); } } diff --git a/plugins/cachekey/cachekey.h b/plugins/cachekey/cachekey.h index 7ea058cb6cf..0922ed104ab 100644 --- a/plugins/cachekey/cachekey.h +++ b/plugins/cachekey/cachekey.h @@ -55,9 +55,10 @@ class CacheKey void append(unsigned number); void append(const String &); + void append(const String &s, bool useSeparator); void append(const char *s); void append(const char *n, unsigned s); - void appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri); + void appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri, bool canonicalPrefix); void appendPath(Pattern &pathCapture, Pattern &pathCaptureUri); void appendHeaders(const ConfigHeaders &config); void appendQuery(const ConfigQuery &config); diff --git a/plugins/cachekey/configs.cc b/plugins/cachekey/configs.cc index 2a13d7608f9..8b018c82b89 100644 --- a/plugins/cachekey/configs.cc +++ b/plugins/cachekey/configs.cc @@ -397,6 +397,8 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) {const_cast("separator"), optional_argument, nullptr, 's'}, {const_cast("uri-type"), optional_argument, nullptr, 't'}, {const_cast("capture-header"), optional_argument, nullptr, 'u'}, + {const_cast("canonical-prefix"), optional_argument, nullptr, 'v'}, + /* reserve 'z' for 'config' files */ {nullptr, 0, nullptr, 0}, }; @@ -504,6 +506,9 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) case 'u': /* capture-header */ _headers.addCapture(optarg); break; + case 'v': /* canonical-prefix */ + _canonicalPrefix = isTrue(optarg); + break; } } @@ -535,6 +540,12 @@ Configs::pathToBeRemoved() return _pathToBeRemoved; } +bool +Configs::canonicalPrefix() +{ + return _canonicalPrefix; +} + void Configs::setSeparator(const char *arg) { diff --git a/plugins/cachekey/configs.h b/plugins/cachekey/configs.h index 603ff434d2a..6fadab7457d 100644 --- a/plugins/cachekey/configs.h +++ b/plugins/cachekey/configs.h @@ -162,6 +162,11 @@ class Configs */ bool pathToBeRemoved(); + /** + * @brief keep URI scheme and authority elements. + */ + bool canonicalPrefix(); + /** * @brief set the cache key elements separator string. */ @@ -205,6 +210,7 @@ class Configs bool _prefixToBeRemoved = false; /**< @brief instructs the prefix (i.e. host:port) not to added to the cache key */ bool _pathToBeRemoved = false; /**< @brief instructs the path not to added to the cache key */ + bool _canonicalPrefix = false; /**< @brief keep the URI scheme and authority element used as input to transforming into key */ String _separator = "/"; /**< @brief a separator used to separate the cache key elements extracted from the URI */ CacheKeyUriType _uriType = REMAP; /**< @brief shows which URI the cache key will be based on */ }; diff --git a/plugins/cachekey/plugin.cc b/plugins/cachekey/plugin.cc index cf3d539f54c..7793de5b2df 100644 --- a/plugins/cachekey/plugin.cc +++ b/plugins/cachekey/plugin.cc @@ -43,7 +43,7 @@ setCacheKey(TSHttpTxn txn, Configs *config, TSRemapRequestInfo *rri = nullptr) /* Append custom prefix or the host:port */ if (!config->prefixToBeRemoved()) { - cachekey.appendPrefix(config->_prefix, config->_prefixCapture, config->_prefixCaptureUri); + cachekey.appendPrefix(config->_prefix, config->_prefixCapture, config->_prefixCaptureUri, config->canonicalPrefix()); } /* Classify User-Agent and append the class name to the cache key if matched. */ cachekey.appendUaClass(config->_classifier); From 9f5ea4af29498cd850ad64a2be2a169932a4b399 Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Fri, 23 Aug 2019 14:26:26 -0700 Subject: [PATCH 2/4] cachekey: added --key-type (for parent selection) Added ability to apply all transformations, available for modifying the cache key, to parent selection URL: * --key-type=cache_key - apply transformations to cache key * --key-type=parent_selection_url - apply transformations to parent selection URL TODO/TBD: After this change all transformations can be applied not only to the cache key but to parent selection URL as well. It would make sense to give the cachekey plugin a new, more suitable name. (cherry picked from commit af7299dd06a2a7b5e862edf5aea0693dd1bde2c1) --- doc/admin-guide/plugins/cachekey.en.rst | 169 ++++++++++++++---------- doc/admin-guide/plugins/index.en.rst | 4 +- plugins/cachekey/cachekey.cc | 69 ++++++++-- plugins/cachekey/cachekey.h | 10 +- plugins/cachekey/configs.cc | 62 ++++++++- plugins/cachekey/configs.h | 19 +++ plugins/cachekey/plugin.cc | 2 +- 7 files changed, 244 insertions(+), 91 deletions(-) diff --git a/doc/admin-guide/plugins/cachekey.en.rst b/doc/admin-guide/plugins/cachekey.en.rst index c82e8e4c4f8..35a4968f371 100644 --- a/doc/admin-guide/plugins/cachekey.en.rst +++ b/doc/admin-guide/plugins/cachekey.en.rst @@ -21,22 +21,24 @@ .. _admin-plugins-cachekey: -Cache Key Manipulation Plugin -***************************** +Cache Key and Parent Selection URL Manipulation Plugin +****************************************************** Description =========== -This plugin allows some common cache key manipulations based on various HTTP request components. It can +This plugin allows some common `cache key` or `parent selection URL` manipulations based on various HTTP request components. +Although `cache key` is used everywhere in this document, the same manipulations can be applied to `parent selection URL` +by switching `key type`_. The plugin can * sort query parameters to prevent query parameter reordering being a cache miss -* ignore specific query parameters from the cache key by name or regular expression -* ignore all query parameters from the cache key -* only use specific query parameters in the cache key by name or regular expression +* ignore specific query parameters from the `cache key` by name or regular expression +* ignore all query parameters from the `cache key` +* only use specific query parameters in the `cache key` by name or regular expression * include headers or cookies by name * capture values from the ``User-Agent`` header. * classify request using ``User-Agent`` and a list of regular expressions -* capture and replace strings from the URI and include them in the cache key +* capture and replace strings from the URI and include them in the `cache key` * do more - please find more examples below. URI type @@ -46,6 +48,17 @@ The plugin manipulates the ``remap`` URI (value set during URI remap) by default * ``--uri-type=[remap|pristine]`` (default: ``remap``) +Key type +======== + +The plugin manipulates the `cache key` by default. If `parent selection URL` manipulation is needed the following option can be used: + +* ``--key-type=[cache_key|parent_selection_url]`` (default: ``cache_key``) + +One instance of this plugin can used either for `cache key` or `parent selection URL` manupulation but never both. +If `simultaneous cache key and parent selection URL manipulation`_ is needed two separate instances of the plugin +have to be loaded for each key type. + Cache key structure and related plugin parameters ================================================= @@ -59,11 +72,11 @@ Cache key structure and related plugin parameters │ (default) | (optional) │ (optional) │ (optional) │ (default) │ (default) │ └─────────────┴──────────────┴──────────────┴──────────────┴─────────────┴─────────────┘ -* The cache key set by the cachekey plugin can be considered as devided into several sections. +* The `cache key` set by the cachekey plugin can be considered as divided into several sections. * Every section is manipulated separately by the related plugin parameters (more info in each section description below). -* "User-Agent", "Headers" and "Cookies" sections are optional and will be missing from the cache key if no related plugin parameters are used. +* "User-Agent", "Headers" and "Cookies" sections are optional and will be missing from the `cache key` if no related plugin parameters are used. * "Prefix", "Path" and "Query" sections always have default values even if no related plugin parameters are used. -* All cachekey plugin parameters are optional and if missing some of the cache key sections will be missing (the optional sections) or their values will be left to their defaults. +* All cachekey plugin parameters are optional and if missing some of the `cache key` sections will be missing (the optional sections) or their values will be left to their defaults. "Prefix" section ^^^^^^^^^^^^^^^^ @@ -81,19 +94,20 @@ Cache key structure and related plugin parameters │ --canonical-prefix | default value if no │ input used for │ │ | prefix parameters used │ --capture-prefix │ ├────────────────────┴─────────────────────────┴──────────────────────┤ - │ fasle | /host/port | host:port | + │ false | /host/port | host:port | ├────────────────────┴─────────────────────────┴──────────────────────┤ │ true | scheme://host:port | scheme://host:port | └──────────────────────────────────────────────┴──────────────────────┘ -* ``--static-prefix=`` (default: empty string) - if specified and not an empty string the ```` will be added to the cache key. -* ``--capture-prefix=`` (default: empty string) - if specified and not empty then strings are captured based on the value of ``--canonical-prefix`` parameter (see the table above) and ```` and are added to the cache key. -* ``--capture-prefix-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the cache key. -* If any of the "Prefix" related plugin parameters are used together in the plugin configuration they are added to the cache key in the order shown in the diagram. -* ``--remove-prefix=true|false|yes|no|0|1`` (default: false) - if specified the prefix elements (host, port) are not processed nor appended to the cachekey. All prefix related plugin parameters are ignored if this parameter is ``true``, ``yes`` or ``1``. +* ``--static-prefix=`` (default: empty string) - if specified and not an empty string the ```` will be added to the `cache key`. +* ``--capture-prefix=`` (default: empty string) - if specified and not empty then strings are captured from ``host:port`` based on the ```` and are added to the `cache key`. +* ``--capture-prefix-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the `cache key`. +* If any of the "Prefix" related plugin parameters are used together in the plugin configuration they are added to the `cache key` in the order shown in the diagram. +* ``--remove-prefix=:`` (default: empty string) - loads a regex patterns list from a file ````, the patterns are matched against the ``User-Agent`` header and if **not** matched ```` is added it to the key. * Multiple ``--ua-whitelist`` and ``--ua-blacklist`` can be used and the result will be defined by their order in the plugin configuration. * ``User-Agent`` regex capturing and replacement - * ``--ua-capture=`` (default: empty string) - if specified and not empty then strings are captured from the ``User-Agent`` header based on ```` (see below) and are added to the cache key. -* If any ``User-Agent`` classification and regex capturing and replacement plugin parameters are used together they are added to the cache key in the order shown in the diagram. + * ``--ua-capture=`` (default: empty string) - if specified and not empty then strings are captured from the ``User-Agent`` header based on ```` (see below) and are added to the `cache key`. +* If any ``User-Agent`` classification and regex capturing and replacement plugin parameters are used together they are added to the `cache key` in the order shown in the diagram. "Headers" section ^^^^^^^^^^^^^^^^^ @@ -126,9 +140,9 @@ Cache key structure and related plugin parameters optional components | └───────────────────┴────────────────────┘ configured | -* ``--include-headers`` (default: empty list) - comma separated list of headers to be added to the cache key. The list of headers defined by ``--include-headers`` are always sorted before adding them to the cache key. +* ``--include-headers`` (default: empty list) - comma separated list of headers to be added to the `cache key`. The list of headers defined by ``--include-headers`` are always sorted before adding them to the `cache key`. -* ``--capture-header=:`` (default: empty) - captures elements from header using and adds them to the cache key. +* ``--capture-header=:`` (default: empty) - captures elements from header using and adds them to the `cache key`. "Cookies" section ^^^^^^^^^^^^^^^^^ @@ -142,7 +156,7 @@ Cache key structure and related plugin parameters optional components | └───────────────────┘ configured | -* ``--include-cookies`` (default: empty list) - comma separated list of cookies to be added to the cache key. The list of cookies defined by ``--include-cookies`` are always sorted before adding them to the cache key. +* ``--include-cookies`` (default: empty list) - comma separated list of cookies to be added to the `cache key`. The list of cookies defined by ``--include-cookies`` are always sorted before adding them to the `cache key`. "Path" section ^^^^^^^^^^^^^^ @@ -156,19 +170,19 @@ Cache key structure and related plugin parameters optional components | └─────────────────────────────────────┘ configured | -* if no path related plugin parameters are used, the URI path string is included in the cache key. -* ``--capture-path=`` (default: empty string) - if specified and not empty then strings are captured from URI path based on the ```` and are added to the cache key. -* ``--capture-path-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the cache key. -* ``--remove-path=`` (default: empty string) - if specified and not empty then strings are captured from URI path based on the ```` and are added to the `cache key`. +* ``--capture-path-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the `cache key`. +* ``--remove-path=`` can be in the following formats - * ```` - ```` defines regex capturing groups, up to 10 captured strings based on ```` will be added to the cache key. - * ``///`` - ```` defines regex capturing groups, ```` defines a pattern where the captured strings referenced with ``$0`` ... ``$9`` will be substituted and the result will be added to the cache key. + * ```` - ```` defines regex capturing groups, up to 10 captured strings based on ```` will be added to the `cache key`. + * ``///`` - ```` defines regex capturing groups, ```` defines a pattern where the captured strings referenced with ``$0`` ... ``$9`` will be substituted and the result will be added to the `cache key`. Cache key elements separator ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* ``--separator=`` - the cache key is constructed by extracting elements from HTTP URI and headers or by using the UA classifiers and they are appended during the key construction and separated by ``/`` (by default). This options allows to override the default separator to any string (including an empty string). +* ``--separator=`` - the `cache key` is constructed by extracting elements from HTTP URI and headers or by using the UA classifiers and they are appended during the key construction and separated by ``/`` (by default). This options allows to override the default separator to any string (including an empty string). How to run the plugin @@ -300,13 +314,13 @@ HTTP request :: * Connection #0 to host 127.0.0.1 left intact * Closing connection #0 -The response header ``X-Cache-Key`` header contains the cache key: :: +The response header ``X-Cache-Key`` header contains the `cache key`: :: /www.example.com/80/popular/Mozilla/5.0/H1:v1/H2:v2/C1=v1;C2=v2/path/to/data?a=1&b=2&c=3 -The ``xdebug.so`` plugin and ``X-Debug`` request header are used just to demonstrate basic cache key troubleshooting. +The ``xdebug.so`` plugin and ``X-Debug`` request header are used just to demonstrate basic `cache key` troubleshooting. -If we add ``--static-prefix=nice_custom_prefix`` to the remap rule then the cache key would look like the following: :: +If we add ``--static-prefix=nice_custom_prefix`` to the remap rule then the `cache key` would look like the following: :: /nice_custom_prefix/popular/Mozilla/5.0/H1:v1/H2:v2/C1=v1;C2=v2/path/to/data?a=1&b=2&c=3 @@ -318,50 +332,50 @@ URI query parameters Ignore the query string (all query parameters) """""""""""""""""""""""""""""""""""""""""""""" -The following added to the remap rule will ignore the query, removing it from the cache key. :: +The following added to the remap rule will ignore the query, removing it from the `cache key`. :: @plugin=cachekey.so @pparam=--remove-all-params=true Cache key normalization by sorting the query parameters """"""""""""""""""""""""""""""""""""""""""""""""""""""" -The following will normalize the cache key by sorting the query parameters. :: +The following will normalize the `cache key` by sorting the query parameters. :: @plugin=cachekey.so @pparam=--sort-params=true -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``a=1&b=2&c=1&k=1&u=1&x=1&y=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``a=1&b=2&c=1&k=1&u=1&x=1&y=1`` Ignore (exclude) certain query parameters """"""""""""""""""""""""""""""""""""""""" -The following will make sure query parameters `a` and `b` will **not** be used when constructing the cache key. :: +The following will make sure query parameters `a` and `b` will **not** be used when constructing the `cache key`. :: @plugin=cachekey.so @pparam=--exclude-params=a,b -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&x=1&k=1&u=1&y=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&x=1&k=1&u=1&y=1`` Ignore (exclude) certain query parameters from the cache key by using regular expression (PCRE) """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -The following will make sure query parameters ``a`` and ``b`` will **not** be used when constructing the cache key. :: +The following will make sure query parameters ``a`` and ``b`` will **not** be used when constructing the `cache key`. :: @plugin=cachekey.so @pparam=--exclude-match-params=(a|b) -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&x=1&k=1&u=1&y=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&x=1&k=1&u=1&y=1`` Include only certain query parameters """"""""""""""""""""""""""""""""""""" -The following will make sure only query parameters `a` and `c` will be used when constructing the cache key and the rest will be ignored. :: +The following will make sure only query parameters `a` and `c` will be used when constructing the `cache key` and the rest will be ignored. :: @plugin=cachekey.so @pparam=--include-params=a,c -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&a=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&a=1`` Include only certain query parameters by using regular expression (PCRE) """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -The following will make sure only query parameters ``a`` and ``c`` will be used when constructing the cache key and the rest will be ignored. :: +The following will make sure only query parameters ``a`` and ``c`` will be used when constructing the `cache key` and the rest will be ignored. :: @plugin=cachekey.so @pparam=--include-match-params=(a|c) -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&a=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&a=1`` White-list + black-list certain parameters using multiple parameters in the same remap rule. """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -374,7 +388,7 @@ If the plugin is used with the following plugin parameters in the remap rule: :: @pparam=--include-params=y,c \ @pparam=--include-params=x,b -and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&b=1`` +and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&b=1`` White-list + black-list certain parameters using multiple parameters in the same remap rule and regular expressions (PCRE). """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -387,7 +401,7 @@ If the plugin is used with the following plugin parameters in the remap rule: :: @pparam=--include-match-params=(y|c) \ @pparam=--include-match-params=(x|b) -and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&b=1`` +and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&b=1`` Mixing --include-params, --exclude-params, --include-match-param and --exclude-match-param """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -400,18 +414,18 @@ If the plugin is used with the following plugin parameters in the remap rule: :: @pparam=--include-params=y,c \ @pparam=--include-match-params=(x|b) -and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&b=1`` +and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&b=1`` HTTP Headers ^^^^^^^^^^^^ Include certain headers in the cache key """""""""""""""""""""""""""""""""""""""" -The following headers ``HeaderA`` and ``HeaderB`` will be used when constructing the cache key and the rest will be ignored. :: +The following headers ``HeaderA`` and ``HeaderB`` will be used when constructing the `cache key` and the rest will be ignored. :: @plugin=cachekey.so @pparam=--include-headers=HeaderA,HeaderB -The following would capture from the ``Authorization`` header and will add the captured element to the cache key :: +The following would capture from the ``Authorization`` header and will add the captured element to the `cache key` :: @plugin=cachekey.so \ @pparam=--capture-header=Authorization:/AWS\s(?[^:]+).*/clientID:$1/" @@ -421,7 +435,7 @@ If the request looks like the following:: http://example-cdn.com/path/file Authorization: AWS MKIARYMOG51PT0DLD:DLiWQ2lyS49H4Zyx34kW0URtg6s= -Cache key would be set to:: +The `cache key` would be set to:: /example-cdn.com/80/clientID:MKIARYMOG51PTCKQ0DLD/path/file @@ -432,7 +446,7 @@ HTTP Cookies Include certain cookies in the cache key """""""""""""""""""""""""""""""""""""""" -The following headers ``CookieA`` and ``CookieB`` will be used when constructing the cache key and the rest will be ignored. :: +The following headers ``CookieA`` and ``CookieB`` will be used when constructing the `cache key` and the rest will be ignored. :: @plugin=cachekey.so @pparam=--include-headers=CookieA,CookieB @@ -446,7 +460,7 @@ If the plugin is used with the following plugin parameter in the remap rule. :: @plugin=cachekey.so @pparam=--static-prefix=static_prefix -the cache key will be prefixed with ``/static_prefix`` instead of ``host:port`` when ``--static-prefix`` is not used. +the `cache key` will be prefixed with ``/static_prefix`` instead of ``host:port`` when ``--static-prefix`` is not used. Capturing from the host:port and adding it to the prefix section """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -455,7 +469,7 @@ If the plugin is used with the following plugin parameter in the remap rule. :: @plugin=cachekey.so \ @pparam=--capture-prefix=(test_prefix).*:([^\s\/$]*) -the cache key will be prefixed with ``/test_prefix/80`` instead of ``test_prefix_371.example.com:80`` when ``--capture-prefix`` is not used. +the `cache key` will be prefixed with ``/test_prefix/80`` instead of ``test_prefix_371.example.com:80`` when ``--capture-prefix`` is not used. Capturing from the entire URI and adding it to the prefix section """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -468,7 +482,7 @@ and if the request URI is the following :: http://test_prefix_123.example.com/path/to/object?a=1&b=2&c=3 -the the cache key will be prefixed with ``/test_prefix_object`` instead of ``test_prefix_123.example.com:80`` when ``--capture-prefix-uri`` is not used. +the the `cache key` will be prefixed with ``/test_prefix_object`` instead of ``test_prefix_123.example.com:80`` when ``--capture-prefix-uri`` is not used. Combining prefix plugin parameters, i.e. --static-prefix and --capture-prefix """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -478,7 +492,7 @@ If the plugin is used with the following plugin parameters in the remap rule. :: @pparam=--capture-prefix=(test_prefix).*:([^\s\/$]*) \ @pparam=--static-prefix=static_prefix -the cache key will be prefixed with ``/static_prefix/test_prefix/80`` instead of ``test_prefix_371.example.com:80`` when either ``--capture-prefix`` nor ``--static-prefix`` are used. +the `cache key` will be prefixed with ``/static_prefix/test_prefix/80`` instead of ``test_prefix_371.example.com:80`` when either ``--capture-prefix`` nor ``--static-prefix`` are used. Path, capture and replace from the path or entire URI @@ -496,7 +510,7 @@ and the request URI is the following :: http://test_path_123.example.com/path/to/object?a=1&b=2&c=3 -then the cache key will have ``/const_path_object`` in the path section of the cache key instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. +then the `cache key` will have ``/const_path_object`` in the path section of the `cache key` instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. Capture and replace groups from whole URI for the "Path" section @@ -511,7 +525,7 @@ and the request URI is the following :: http://test_path_123.example.com/path/to/object?a=1&b=2&c=3 -the the cache key will have ``/test_path_object`` in the path section of the cache key instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. +the the `cache key` will have ``/test_path_object`` in the path section of the `cache key` instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. Combining path plugin parameters --capture-path and --capture-path-uri @@ -527,7 +541,7 @@ and the request URI is the following :: http://test_path_123.example.com/path/to/object?a=1&b=2&c=3 -the the cache key will have ``/test_path_object/const_path_object`` in the path section of the cache key instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. +the the `cache key` will have ``/test_path_object/const_path_object`` in the path section of the `cache key` instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. User-Agent capturing, replacement and classification ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -589,16 +603,16 @@ then ``browser`` will be used when constructing the key. Cacheurl plugin to cachekey plugin migration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The plugin `cachekey` was not meant to replace the cacheurl plugin in terms of having exactly the same cache key strings generated. It just allows the operator to exctract elements from the HTTP URI in the same way the `cacheurl` does (through a regular expression, please see `` above). +The plugin `cachekey` was not meant to replace the cacheurl plugin in terms of having exactly the same `cache key` strings generated. It just allows the operator to extract elements from the HTTP URI in the same way the `cacheurl` does (through a regular expression, please see `` above). -The following examples demonstrate different ways to achieve `cacheurl` compatibility on a cache key string level in order to avoid invalidation of the cache. +The following examples demonstrate different ways to achieve `cacheurl` compatibility on a `cache key` string level in order to avoid invalidation of the cache. The operator could use `--capture-path-uri`, `--capture-path`, `--capture-prefix-uri`, `--capture-prefix` to capture elements from the URI, path and authority elements. -By using `--separator=` the operator could override the default separator to an empty string `--separator=` and thus make sure there are no cache key element separators. +By using `--separator=` the operator could override the default separator to an empty string `--separator=` and thus make sure there are no `cache key` element separators. -Example 1: Let us say we have a capture definition used in `cacheurl`. Now by using `--capture-prefix-uri` one could extract elements through the same caplture definition used with `cacheurl`, remove the cache key element separator `--separator=` and by using `--capture-path-uri` could remove the URI path and by using `--remove-all-params=true` could remove the query string:: +Example 1: Let us say we have a capture definition used in `cacheurl`. Now by using `--capture-prefix-uri` one could extract elements through the same capture definition used with `cacheurl`, remove the `cache key` element separator `--separator=` and by using `--capture-path-uri` could remove the URI path and by using `--remove-all-params=true` could remove the query string:: @plugin=cachekey.so \ @pparam=--capture-prefix-uri=/.*/$0/ \ @@ -606,7 +620,7 @@ Example 1: Let us say we have a capture definition used in `cacheurl`. Now by us @pparam=--remove-all-params=true \ @pparam=--separator= -Example 2: A more efficient way would be achieved by using `--capture-prefix-uri` to capture from the URI, remove the cache key element separator `--separator=` and by using `--remove-path` to remove the URI path and `--remove-all-params=true` to remove the query string:: +Example 2: A more efficient way would be achieved by using `--capture-prefix-uri` to capture from the URI, remove the `cache key` element separator `--separator=` and by using `--remove-path` to remove the URI path and `--remove-all-params=true` to remove the query string:: @plugin=cachekey.so \ @pparam=--capture-prefix-uri=/.*/$0/ \ @@ -614,7 +628,7 @@ Example 2: A more efficient way would be achieved by using `--capture-prefix-uri @pparam=--remove-all-params=true \ @pparam=--separator= -Example 3: Same result as the above but this time by using `--capture-path-uri` to capture from the URI, remove the cache key element separator `--separator=` and by using `--remove-prefix` to remove the URI authority elements and by using `--remove-all-params=true` to remove the query string:: +Example 3: Same result as the above but this time by using `--capture-path-uri` to capture from the URI, remove the `cache key` element separator `--separator=` and by using `--remove-prefix` to remove the URI authority elements and by using `--remove-all-params=true` to remove the query string:: @plugin=cachekey.so \ @pparam=--capture-path-uri=/(.*)/$0/ \ @@ -629,3 +643,24 @@ Example 4: Let us say that we would like to capture from URI in similar to `cach @pparam=--remove-path=true \ @pparam=--sort-params=true \ @pparam=--separator= + + +Simultaneous cache key and parent selection URL manipulation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following is an example of how to manipulate both `cache key` and `parent selection URL` in the same remap rule. +For this purpose two separate instances are loaded for that remap rule: + +:: + + @plugin=cachekey.so \ + @pparam=--key-type=parent_selection_url \ + @pparam=--static-prefix=this://goes.to/parent/selection/url \ + @pparam=--canonical-prefix=true \ + @plugin=cachekey.so \ + @pparam=--key-type=cache_key \ + @pparam=--static-prefix=this://goes.to/cache/key \ + @pparam=--canonical-prefix=true + +In the example above the first instance of the plugin sets the prefix to the parent selection URI and +the second instance of the plugin sets the prefix to the cache key. diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index 4e69bcec79a..c345f52a8f6 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -75,8 +75,8 @@ Plugins that are considered stable are installed by default in |TS| releases. :doc:`Background Fetch ` Proactively fetch content from Origin in a way that it will fill the object into cache. -:doc:`Cache Key Manipulation ` - Allows some common cache key manipulations based on various HTTP request elements. +:doc:`Cache Key and Parent Selection URL Manipulation ` + Allows some common cache key or parent selection URL manipulations based on various HTTP request elements. :doc:`Cache Promotion Policies ` Allows for control over which assets should be written to cache, or not. diff --git a/plugins/cachekey/cachekey.cc b/plugins/cachekey/cachekey.cc index b7d6cccca3b..5eedc76164f 100644 --- a/plugins/cachekey/cachekey.cc +++ b/plugins/cachekey/cachekey.cc @@ -239,8 +239,8 @@ getCanonicalUrl(TSMBuffer buf, TSMLoc url, bool canonicalPrefix, bool provideDef * @param uriType type of the URI used to create the cachekey ("remap" or "pristine") * @param rri remap request info */ -CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSRemapRequestInfo *rri) - : _txn(txn), _separator(separator), _uriType(uriType) +CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, CacheKeyKeyType keyType, TSRemapRequestInfo *rri) + : _txn(txn), _separator(std::move(separator)), _uriType(uriType), _keyType(keyType) { _key.reserve(512); @@ -249,8 +249,9 @@ CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSR /* Get the URI and header to base the cachekey on. * @TODO it might make sense to add more supported URI types */ + CacheKeyDebug("setting %s from a %s plugin", getCacheKeyKeyTypeName(_keyType), _remap ? "remap" : "global"); + if (_remap) { - CacheKeyDebug("setting cache key from a remap plugin"); if (PRISTINE == _uriType) { if (TS_SUCCESS != TSHttpTxnPristineUrlGet(_txn, &_buf, &_url)) { /* Failing here is unlikely. No action seems the only reasonable thing to do from within this plug-in */ @@ -265,7 +266,6 @@ CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSR } _hdrs = rri->requestHdrp; } else { - CacheKeyDebug("setting cache key from a global plugin"); if (TS_SUCCESS != TSHttpTxnClientReqGet(_txn, &_buf, &_hdrs)) { /* Failing here is unlikely. No action seems the only reasonable thing to do from within this plug-in */ CacheKeyError("failed to get client request handle"); @@ -744,24 +744,67 @@ CacheKey::appendUaClass(Classifier &classifier) bool CacheKey::finalize() const { - bool res = true; - CacheKeyDebug("finalizing cache key '%s' from a %s plugin", _key.c_str(), (_remap ? "remap" : "global")); - if (TS_SUCCESS != TSCacheUrlSet(_txn, &(_key[0]), _key.size())) { - int len; - char *url = TSHttpTxnEffectiveUrlStringGet(_txn, &len); - if (nullptr != url) { + bool res = false; + String msg; + + CacheKeyDebug("finalizing %s '%s' from a %s plugin", getCacheKeyKeyTypeName(_keyType), _key.c_str(), + (_remap ? "remap" : "global")); + switch (_keyType) { + case CACHE_KEY: { + if (TS_SUCCESS == TSCacheUrlSet(_txn, &(_key[0]), _key.size())) { + /* Set cache key succesfully */ + msg.assign("set cache key to ").append(_key); + res = true; + } else { if (_remap) { /* Remap instance. Always runs first by design (before TS_HTTP_POST_REMAP_HOOK) */ - CacheKeyError("failed to set cache key for url %.*s", len, url); + msg.assign("failed to set cache key"); } else { /* Global instance. We would fail and get here if a per-remap instance has already set the cache key * (currently TSCacheUrlSet() can be called only once successfully). Don't error, just debug. * @todo avoid the consecutive attempts and error only on unexpected failures. */ - CacheKeyDebug("failed to set cache key for url %.*s", len, url); + msg.assign("failed to set cache key"); + } + } + } break; + case PARENT_SELECTION_URL: { + /* parent selection */ + const char *start = _key.c_str(); + const char *end = _key.c_str() + _key.length(); + TSMLoc new_url_loc; + if (TS_SUCCESS == TSUrlCreate(_buf, &new_url_loc)) { + if (TS_PARSE_DONE == TSUrlParse(_buf, new_url_loc, &start, end)) { + if (TS_SUCCESS == TSHttpTxnParentSelectionUrlSet(_txn, _buf, new_url_loc)) { + msg.assign("set parent selection URL to ").append(_key); + res = true; + } else { + msg.assign("failed to set parent selection URL"); + } + } else { + msg.assign("failed to parse parent selection URL"); } + TSHandleMLocRelease(_buf, TS_NULL_MLOC, new_url_loc); + } else { + msg.assign("failed to create parent selection URL"); + } + } break; + default: { + msg.assign("unknown target URI type"); + } break; + } + + /* Report status - debug level in case of success, error in case of failure. + * Since getting effective URI is expensive add it only in case of failure */ + if (res) { + CacheKeyDebug("%.*s", static_cast(msg.length()), msg.c_str()); + } else { + int len; + char *url = TSHttpTxnEffectiveUrlStringGet(_txn, &len); + if (nullptr != url) { + msg.append(" for url ").append(url, len); TSfree(url); } - res = false; + CacheKeyError("%.*s", static_cast(msg.length()), msg.c_str()); } return res; } diff --git a/plugins/cachekey/cachekey.h b/plugins/cachekey/cachekey.h index 0922ed104ab..0b47e85984d 100644 --- a/plugins/cachekey/cachekey.h +++ b/plugins/cachekey/cachekey.h @@ -50,7 +50,8 @@ class CacheKey { public: - CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType urlType, TSRemapRequestInfo *rri = nullptr); + CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType urlType, CacheKeyKeyType targetUrlType, + TSRemapRequestInfo *rri = nullptr); ~CacheKey(); void append(unsigned number); @@ -86,7 +87,8 @@ class CacheKey bool _valid = false; /**< @brief shows if the constructor discovered the input correctly */ bool _remap = false; /**< @brief shows if the input URI was from remap info */ - String _key; /**< @brief cache key */ - String _separator; /**< @brief a separator used to separate the cache key elements extracted from the URI */ - CacheKeyUriType _uriType; /**< @brief the URI type used as a cachekey base: pristine, remap, etc. */ + String _key; /**< @brief cache key */ + String _separator; /**< @brief a separator used to separate the cache key elements extracted from the URI */ + CacheKeyUriType _uriType = REMAP; /**< @brief the URI type used as a cachekey base: pristine, remap, etc. */ + CacheKeyKeyType _keyType = CACHE_KEY; /**< @brief the target URI type: cache key, parent selection, etc. */ }; diff --git a/plugins/cachekey/configs.cc b/plugins/cachekey/configs.cc index 8b018c82b89..932123248aa 100644 --- a/plugins/cachekey/configs.cc +++ b/plugins/cachekey/configs.cc @@ -396,8 +396,9 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) {const_cast("remove-path"), optional_argument, nullptr, 'r'}, {const_cast("separator"), optional_argument, nullptr, 's'}, {const_cast("uri-type"), optional_argument, nullptr, 't'}, - {const_cast("capture-header"), optional_argument, nullptr, 'u'}, - {const_cast("canonical-prefix"), optional_argument, nullptr, 'v'}, + {const_cast("key-type"), optional_argument, nullptr, 'u'}, + {const_cast("capture-header"), optional_argument, nullptr, 'v'}, + {const_cast("canonical-prefix"), optional_argument, nullptr, 'w'}, /* reserve 'z' for 'config' files */ {nullptr, 0, nullptr, 0}, }; @@ -503,10 +504,13 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) case 't': /* uri-type */ setUriType(optarg); break; - case 'u': /* capture-header */ + case 'u': /* key-type */ + setKeyType(optarg); + break; + case 'v': /* capture-header */ _headers.addCapture(optarg); break; - case 'v': /* canonical-prefix */ + case 'w': /* canonical-prefix */ _canonicalPrefix = isTrue(optarg); break; } @@ -578,8 +582,58 @@ Configs::setUriType(const char *arg) } } +void +Configs::setKeyType(const char *arg) +{ + if (nullptr != arg) { + if (9 == strlen(arg) && 0 == strncasecmp(arg, "cache_key", 9)) { + _keyType = CacheKeyKeyType::CACHE_KEY; + CacheKeyDebug("setting cache key"); + } else if (20 == strlen(arg) && 0 == strncasecmp(arg, "parent_selection_url", 20)) { + _keyType = CacheKeyKeyType::PARENT_SELECTION_URL; + CacheKeyDebug("setting parent selection URL"); + } else { + CacheKeyError("unrecognized key type '%s', using default 'cache_key'", arg); + } + } else { + CacheKeyError("found an empty key type, using default 'cache_key'"); + } +} + CacheKeyUriType Configs::getUriType() { return _uriType; } + +CacheKeyKeyType +Configs::getKeyType() +{ + return _keyType; +} + +const char * +getCacheKeyUriTypeName(CacheKeyUriType type) +{ + switch (type) { + case REMAP: + return "remap"; + case PRISTINE: + return "pristine"; + default: + return "unknown"; + } +} + +const char * +getCacheKeyKeyTypeName(CacheKeyKeyType type) +{ + switch (type) { + case CACHE_KEY: + return "cache key"; + case PARENT_SELECTION_URL: + return "parent selection url"; + default: + return "unknown"; + } +} diff --git a/plugins/cachekey/configs.h b/plugins/cachekey/configs.h index 6fadab7457d..e0669610434 100644 --- a/plugins/cachekey/configs.h +++ b/plugins/cachekey/configs.h @@ -33,6 +33,14 @@ enum CacheKeyUriType { PRISTINE, }; +enum CacheKeyKeyType { + CACHE_KEY, + PARENT_SELECTION_URL, +}; + +const char *getCacheKeyUriTypeName(CacheKeyUriType type); +const char *getCacheKeyKeyTypeName(CacheKeyKeyType type); + /** * @brief Plug-in configuration elements (query / headers / cookies). * @@ -182,11 +190,21 @@ class Configs */ void setUriType(const char *arg); + /** + * @brief sets the target URI Type. + */ + void setKeyType(const char *arg); + /** * @brief get URI type. */ CacheKeyUriType getUriType(); + /** + * @brief get target URI type. + */ + CacheKeyKeyType getKeyType(); + /* Make the following members public to avoid unnecessary accessors */ ConfigQuery _query; /**< @brief query parameter related configuration */ ConfigHeaders _headers; /**< @brief headers related configuration */ @@ -213,4 +231,5 @@ class Configs bool _canonicalPrefix = false; /**< @brief keep the URI scheme and authority element used as input to transforming into key */ String _separator = "/"; /**< @brief a separator used to separate the cache key elements extracted from the URI */ CacheKeyUriType _uriType = REMAP; /**< @brief shows which URI the cache key will be based on */ + CacheKeyKeyType _keyType = CACHE_KEY; /**< @brief target URI to be modified, cache key or paren selection */ }; diff --git a/plugins/cachekey/plugin.cc b/plugins/cachekey/plugin.cc index 7793de5b2df..4e208062fa5 100644 --- a/plugins/cachekey/plugin.cc +++ b/plugins/cachekey/plugin.cc @@ -39,7 +39,7 @@ static void setCacheKey(TSHttpTxn txn, Configs *config, TSRemapRequestInfo *rri = nullptr) { /* Initial cache key facility from the requested URL. */ - CacheKey cachekey(txn, config->getSeparator(), config->getUriType(), rri); + CacheKey cachekey(txn, config->getSeparator(), config->getUriType(), config->getKeyType(), rri); /* Append custom prefix or the host:port */ if (!config->prefixToBeRemoved()) { From 0134a7193a43077dde85bb43a650faf96eea5944 Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Tue, 17 Sep 2019 10:06:50 -0700 Subject: [PATCH 3/4] cachekey: allow multiple values for `--key-type` Allow multiple target types to be specified for `--key-type` so the operator can apply the same modifications to both cache key and parent selection url at the same time without chaining cachekey plugin instances. Instead of: @plugin=cachekey.so \ @pparam=--key-type=parent_selection_url \ @pparam=--remove-all-params=true @plugin=cachekey.so \ @pparam=--key-type=cache_key \ @pparam=--remove-all-params=true to write: @plugin=cachekey.so \ @pparam=--key-type=parent_selection_url,cache_key \ @pparam=--remove-all-params=true (cherry picked from commit db8cd14acede7460a5996864c52e1b206695e405) --- doc/admin-guide/plugins/cachekey.en.rst | 25 ++++++++++--- plugins/cachekey/configs.cc | 29 +++++++++------ plugins/cachekey/configs.h | 6 ++-- plugins/cachekey/plugin.cc | 48 +++++++++++++------------ 4 files changed, 70 insertions(+), 38 deletions(-) diff --git a/doc/admin-guide/plugins/cachekey.en.rst b/doc/admin-guide/plugins/cachekey.en.rst index 35a4968f371..6199c93daf3 100644 --- a/doc/admin-guide/plugins/cachekey.en.rst +++ b/doc/admin-guide/plugins/cachekey.en.rst @@ -53,11 +53,11 @@ Key type The plugin manipulates the `cache key` by default. If `parent selection URL` manipulation is needed the following option can be used: -* ``--key-type=[cache_key|parent_selection_url]`` (default: ``cache_key``) +* ``--key-type=`` (default: ``cache_key``) - list of ``cache_key`` or ``parent_selection_url``, if multiple ``--key-type`` options are specified then all values are combined together. + +An instance of this plugin can be used for applying manipulations to `cache key`, `parent selection URL` or both depending on the need. See `simultaneous cache key and parent selection URL manipulation`_ +for examples of how to apply the **same** set of manupulations to both targets with a single plugin instance or applying **diferent** sets of manipulations to each target using separate plugin instances. -One instance of this plugin can used either for `cache key` or `parent selection URL` manupulation but never both. -If `simultaneous cache key and parent selection URL manipulation`_ is needed two separate instances of the plugin -have to be loaded for each key type. Cache key structure and related plugin parameters ================================================= @@ -664,3 +664,20 @@ For this purpose two separate instances are loaded for that remap rule: In the example above the first instance of the plugin sets the prefix to the parent selection URI and the second instance of the plugin sets the prefix to the cache key. + +The **same** string manipulations can be applied to both cache key and parent selection url more concisely without chaining cachekey plugin instances by specifying multiple target types `--key-type`. + +Instead of:: + + @plugin=cachekey.so \ + @pparam=--key-type=parent_selection_url \ + @pparam=--remove-all-params=true + @plugin=cachekey.so \ + @pparam=--key-type=cache_key \ + @pparam=--remove-all-params=true + +one could write:: + + @plugin=cachekey.so \ + @pparam=--key-type=parent_selection_url,cache_key \ + @pparam=--remove-all-params=true diff --git a/plugins/cachekey/configs.cc b/plugins/cachekey/configs.cc index 932123248aa..05d93ca8fc0 100644 --- a/plugins/cachekey/configs.cc +++ b/plugins/cachekey/configs.cc @@ -529,6 +529,10 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) bool Configs::finalize() { + if (_keyTypes.empty()) { + CacheKeyDebug("setting cache key"); + _keyTypes = {CACHE_KEY}; + } return _query.finalize() && _headers.finalize() && _cookies.finalize(); } @@ -586,14 +590,19 @@ void Configs::setKeyType(const char *arg) { if (nullptr != arg) { - if (9 == strlen(arg) && 0 == strncasecmp(arg, "cache_key", 9)) { - _keyType = CacheKeyKeyType::CACHE_KEY; - CacheKeyDebug("setting cache key"); - } else if (20 == strlen(arg) && 0 == strncasecmp(arg, "parent_selection_url", 20)) { - _keyType = CacheKeyKeyType::PARENT_SELECTION_URL; - CacheKeyDebug("setting parent selection URL"); - } else { - CacheKeyError("unrecognized key type '%s', using default 'cache_key'", arg); + StringVector types; + ::commaSeparateString(types, arg); + + for (auto type : types) { + if (9 == type.length() && 0 == strncasecmp(type.c_str(), "cache_key", 9)) { + _keyTypes.insert(CacheKeyKeyType::CACHE_KEY); + CacheKeyDebug("setting cache key"); + } else if (20 == type.length() && 0 == strncasecmp(type.c_str(), "parent_selection_url", 20)) { + _keyTypes.insert(CacheKeyKeyType::PARENT_SELECTION_URL); + CacheKeyDebug("setting parent selection URL"); + } else { + CacheKeyError("unrecognized key type '%s', using default 'cache_key'", arg); + } } } else { CacheKeyError("found an empty key type, using default 'cache_key'"); @@ -606,10 +615,10 @@ Configs::getUriType() return _uriType; } -CacheKeyKeyType +CacheKeyKeyTypeSet & Configs::getKeyType() { - return _keyType; + return _keyTypes; } const char * diff --git a/plugins/cachekey/configs.h b/plugins/cachekey/configs.h index e0669610434..454fb367321 100644 --- a/plugins/cachekey/configs.h +++ b/plugins/cachekey/configs.h @@ -41,6 +41,8 @@ enum CacheKeyKeyType { const char *getCacheKeyUriTypeName(CacheKeyUriType type); const char *getCacheKeyKeyTypeName(CacheKeyKeyType type); +typedef std::set CacheKeyKeyTypeSet; + /** * @brief Plug-in configuration elements (query / headers / cookies). * @@ -203,7 +205,7 @@ class Configs /** * @brief get target URI type. */ - CacheKeyKeyType getKeyType(); + CacheKeyKeyTypeSet &getKeyType(); /* Make the following members public to avoid unnecessary accessors */ ConfigQuery _query; /**< @brief query parameter related configuration */ @@ -231,5 +233,5 @@ class Configs bool _canonicalPrefix = false; /**< @brief keep the URI scheme and authority element used as input to transforming into key */ String _separator = "/"; /**< @brief a separator used to separate the cache key elements extracted from the URI */ CacheKeyUriType _uriType = REMAP; /**< @brief shows which URI the cache key will be based on */ - CacheKeyKeyType _keyType = CACHE_KEY; /**< @brief target URI to be modified, cache key or paren selection */ + CacheKeyKeyTypeSet _keyTypes; /**< @brief target URI to be modified, cache key or paren selection */ }; diff --git a/plugins/cachekey/plugin.cc b/plugins/cachekey/plugin.cc index 4e208062fa5..a05117f1ff9 100644 --- a/plugins/cachekey/plugin.cc +++ b/plugins/cachekey/plugin.cc @@ -38,34 +38,38 @@ Configs *globalConfig = nullptr; static void setCacheKey(TSHttpTxn txn, Configs *config, TSRemapRequestInfo *rri = nullptr) { - /* Initial cache key facility from the requested URL. */ - CacheKey cachekey(txn, config->getSeparator(), config->getUriType(), config->getKeyType(), rri); + const CacheKeyKeyTypeSet &keyTypes = config->getKeyType(); - /* Append custom prefix or the host:port */ - if (!config->prefixToBeRemoved()) { - cachekey.appendPrefix(config->_prefix, config->_prefixCapture, config->_prefixCaptureUri, config->canonicalPrefix()); - } - /* Classify User-Agent and append the class name to the cache key if matched. */ - cachekey.appendUaClass(config->_classifier); + for (auto type : keyTypes) { + /* Initial cache key facility from the requested URL. */ + CacheKey cachekey(txn, config->getSeparator(), config->getUriType(), type, rri); - /* Capture from User-Agent header. */ - cachekey.appendUaCaptures(config->_uaCapture); + /* Append custom prefix or the host:port */ + if (!config->prefixToBeRemoved()) { + cachekey.appendPrefix(config->_prefix, config->_prefixCapture, config->_prefixCaptureUri, config->canonicalPrefix()); + } + /* Classify User-Agent and append the class name to the cache key if matched. */ + cachekey.appendUaClass(config->_classifier); - /* Append headers to the cache key. */ - cachekey.appendHeaders(config->_headers); + /* Capture from User-Agent header. */ + cachekey.appendUaCaptures(config->_uaCapture); - /* Append cookies to the cache key. */ - cachekey.appendCookies(config->_cookies); + /* Append headers to the cache key. */ + cachekey.appendHeaders(config->_headers); - /* Append the path to the cache key. */ - if (!config->pathToBeRemoved()) { - cachekey.appendPath(config->_pathCapture, config->_pathCaptureUri); - } - /* Append query parameters to the cache key. */ - cachekey.appendQuery(config->_query); + /* Append cookies to the cache key. */ + cachekey.appendCookies(config->_cookies); - /* Set the cache key */ - cachekey.finalize(); + /* Append the path to the cache key. */ + if (!config->pathToBeRemoved()) { + cachekey.appendPath(config->_pathCapture, config->_pathCaptureUri); + } + /* Append query parameters to the cache key. */ + cachekey.appendQuery(config->_query); + + /* Set the cache key */ + cachekey.finalize(); + } } static int From fb17e0d61dbcac50610fe26e3af19ba6fd98e1f6 Mon Sep 17 00:00:00 2001 From: Gancho Tenev Date: Thu, 19 Mar 2020 15:31:48 -0700 Subject: [PATCH 4/4] cachekey plugin docs and clang-format/tidy changes Cherry-picked some isolated cachekey related changes from the following commits to branck 8.1.x: aa58b6eb8 Ran clang-format 4cfd5a738 Ran make clang-tidy 8e451b774 Fixes spelling in plugins 7651e269d Ran clang-tidy with modernize-use-default-member-init d77cd7316 Ran clang-tidy --- plugins/cachekey/README.md | 2 +- plugins/cachekey/cachekey.cc | 1 + plugins/cachekey/configs.cc | 6 +++--- plugins/cachekey/configs.h | 10 +++++----- plugins/cachekey/pattern.cc | 10 +++++----- plugins/cachekey/pattern.h | 14 +++++++------- plugins/cachekey/plugin.cc | 4 ++-- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/plugins/cachekey/README.md b/plugins/cachekey/README.md index 91dc7252e69..6cc59898bbe 100644 --- a/plugins/cachekey/README.md +++ b/plugins/cachekey/README.md @@ -1,7 +1,7 @@ # Description This plugin allows some common cache key manipulations based on various HTTP request elements. It can -* sort query parameters to prevent query parameters reordereding from being a cache miss +* sort query parameters to prevent query parameters reordering from being a cache miss * ignore specific query parameters from the cache key by name or regular expression * ignore all query parameters from the cache key * only use specific query parameters in the cache key by name or regular expression diff --git a/plugins/cachekey/cachekey.cc b/plugins/cachekey/cachekey.cc index 5eedc76164f..5f128894bfa 100644 --- a/plugins/cachekey/cachekey.cc +++ b/plugins/cachekey/cachekey.cc @@ -23,6 +23,7 @@ #include /* strlen() */ #include /* istringstream */ +#include #include "cachekey.h" static void diff --git a/plugins/cachekey/configs.cc b/plugins/cachekey/configs.cc index 05d93ca8fc0..938ae1d5cad 100644 --- a/plugins/cachekey/configs.cc +++ b/plugins/cachekey/configs.cc @@ -183,8 +183,8 @@ ConfigElements::noIncludeExcludeRules() const ConfigElements::~ConfigElements() { - for (auto it = _captures.begin(); it != _captures.end(); it++) { - delete it->second; + for (auto &_capture : _captures) { + delete _capture.second; } } @@ -414,7 +414,7 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) for (;;) { int opt; - opt = getopt_long(argc, (char *const *)argv, "", longopt, nullptr); + opt = getopt_long(argc, const_cast(argv), "", longopt, nullptr); if (opt == -1) { break; diff --git a/plugins/cachekey/configs.h b/plugins/cachekey/configs.h index 454fb367321..e8712f18342 100644 --- a/plugins/cachekey/configs.h +++ b/plugins/cachekey/configs.h @@ -51,7 +51,7 @@ typedef std::set CacheKeyKeyTypeSet; class ConfigElements { public: - ConfigElements() : _sort(false), _remove(false), _skip(false) {} + ConfigElements() {} virtual ~ConfigElements(); void setExclude(const char *arg); void setInclude(const char *arg); @@ -92,9 +92,9 @@ class ConfigElements MultiPattern _includePatterns; MultiPattern _excludePatterns; - bool _sort; - bool _remove; - bool _skip; + bool _sort = false; + bool _remove = false; + bool _skip = false; std::map _captures; }; @@ -158,7 +158,7 @@ class Configs /** * @brief provides means for post-processing of the plugin parameters to finalize the configuration or to "cache" some of the * decisions for later use. - * @return true if succesful, false if failure. + * @return true if successful, false if failure. */ bool finalize(); diff --git a/plugins/cachekey/pattern.cc b/plugins/cachekey/pattern.cc index 27eb94ee24b..c93490291da 100644 --- a/plugins/cachekey/pattern.cc +++ b/plugins/cachekey/pattern.cc @@ -38,7 +38,7 @@ replaceString(String &str, const String &from, const String &to) } } -Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(""), _replace(false), _tokenCount(0) {} +Pattern::Pattern() : _pattern(""), _replacement("") {} /** * @brief Initializes PCRE pattern by providing the subject and replacement strings. @@ -47,18 +47,18 @@ Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(" * @return true if successful, false if failure */ bool -Pattern::init(const String &pattern, const String &replacenemt, bool replace) +Pattern::init(const String &pattern, const String &replacement, bool replace) { pcreFree(); _pattern.assign(pattern); - _replacement.assign(replacenemt); + _replacement.assign(replacement); _replace = replace; _tokenCount = 0; if (!compile()) { - CacheKeyDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacenemt.c_str()); + CacheKeyDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacement.c_str()); pcreFree(); return false; } @@ -151,7 +151,7 @@ Pattern::pcreFree() } /** - * @bried Destructor, frees PCRE related resources. + * @brief Destructor, frees PCRE related resources. */ Pattern::~Pattern() { diff --git a/plugins/cachekey/pattern.h b/plugins/cachekey/pattern.h index 876edf0a3db..2b36fd70884 100644 --- a/plugins/cachekey/pattern.h +++ b/plugins/cachekey/pattern.h @@ -45,7 +45,7 @@ class Pattern Pattern(); virtual ~Pattern(); - bool init(const String &pattern, const String &replacenemt, bool replace); + bool init(const String &pattern, const String &replacement, bool replace); bool init(const String &config); bool empty() const; bool match(const String &subject); @@ -57,16 +57,16 @@ class Pattern bool compile(); void pcreFree(); - pcre *_re; /**< @brief PCRE compiled info structure, computed during initialization */ - pcre_extra *_extra; /**< @brief PCRE study data block, computed during initialization */ + pcre *_re = nullptr; /**< @brief PCRE compiled info structure, computed during initialization */ + pcre_extra *_extra = nullptr; /**< @brief PCRE study data block, computed during initialization */ String _pattern; /**< @brief PCRE pattern string, containing PCRE patterns and capturing groups. */ String _replacement; /**< @brief PCRE replacement string, containing $0..$9 to be replaced with content of the capturing groups */ - bool _replace; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty replacement - string and no replacement needed case */ + bool _replace = false; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty + replacement string and no replacement needed case */ - int _tokenCount; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ + int _tokenCount = 0; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ int _tokens[TOKENCOUNT]; /**< @brief replacement index 0..9, since they can be used in the replacement string in any order */ int _tokenOffset[TOKENCOUNT]; /**< @brief replacement offset inside the replacement string */ }; @@ -77,7 +77,7 @@ class Pattern class MultiPattern { public: - MultiPattern(const String name = "") : _name(name) {} + MultiPattern(const String &name = "") : _name(name) {} virtual ~MultiPattern(); bool empty() const; diff --git a/plugins/cachekey/plugin.cc b/plugins/cachekey/plugin.cc index a05117f1ff9..d92c079271a 100644 --- a/plugins/cachekey/plugin.cc +++ b/plugins/cachekey/plugin.cc @@ -165,7 +165,7 @@ TSRemapNewInstance(int argc, char *argv[], void **instance, char *errBuf, int er void TSRemapDeleteInstance(void *instance) { - Configs *config = (Configs *)instance; + Configs *config = static_cast(instance); delete config; } @@ -181,7 +181,7 @@ TSRemapDeleteInstance(void *instance) TSRemapStatus TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo *rri) { - Configs *config = (Configs *)instance; + Configs *config = static_cast(instance); if (nullptr != config) { setCacheKey(txn, config, rri);