diff --git a/doc/admin-guide/plugins/regex_revalidate.en.rst b/doc/admin-guide/plugins/regex_revalidate.en.rst index 18a582350f5..57e8f990460 100644 --- a/doc/admin-guide/plugins/regex_revalidate.en.rst +++ b/doc/admin-guide/plugins/regex_revalidate.en.rst @@ -78,7 +78,7 @@ Inside your revalidation rules configuration, each rule line is defined as a regular expression followed by an integer which expresses the epoch time at which the rule will expire:: - + [type MISS or default STALE] Blank lines and lines beginning with a ``#`` character are ignored. @@ -97,6 +97,16 @@ expressed as an integer of seconds since epoch (equivalent to the return value of :manpage:`time(2)`), after which the forced revalidation will no longer occur. +Type +---- + +By default any matching asset will have its cache lookup status changed +from HIT_FRESH to HIT_STALE. By adding an extra keyword MISS at the end +of a line the asset will be marked MISS instead, forcing a refetch from +the parent. *Use with care* as this will increase bandwidth to the parent. +During configuration reload, any rule which changes it type will be +reloaded and treated as a new rule. + Caveats ======= @@ -128,3 +138,6 @@ in |TS| until 6:47:27 AM on Saturday, November 14th, 2015 (UTC):: Note the escaping of the ``.`` metacharacter in the rule's regular expression. +Alternatively the following rule would case a refetch from the parent:: + + http://origin\.tld/images/foo\.jpg 1447483647 MISS diff --git a/plugins/regex_revalidate/regex_revalidate.c b/plugins/regex_revalidate/regex_revalidate.c index 2fa32c5e5e4..56241355809 100644 --- a/plugins/regex_revalidate/regex_revalidate.c +++ b/plugins/regex_revalidate/regex_revalidate.c @@ -45,12 +45,33 @@ #define LOG_ROLL_INTERVAL 86400 #define LOG_ROLL_OFFSET 0 +static char const *const RESULT_MISS = "MISS"; +static char const *const RESULT_STALE = "STALE"; +static char const *const RESULT_UNKNOWN = "UNKNOWN"; + +static char const *const +strForResult(TSCacheLookupResult const result) +{ + switch (result) { + case TS_CACHE_LOOKUP_MISS: + return RESULT_MISS; + break; + case TS_CACHE_LOOKUP_HIT_STALE: + return RESULT_STALE; + break; + default: + return RESULT_UNKNOWN; + break; + } +} + typedef struct invalidate_t { const char *regex_text; pcre *regex; pcre_extra *regex_extra; time_t epoch; time_t expiry; + TSCacheLookupResult new_result; struct invalidate_t *next; } invalidate_t; @@ -69,6 +90,7 @@ init_invalidate_t(invalidate_t *i) i->regex_extra = NULL; i->epoch = 0; i->expiry = 0; + i->new_result = TS_CACHE_LOOKUP_HIT_STALE; i->next = NULL; return i; } @@ -139,6 +161,7 @@ copy_invalidate_t(invalidate_t *i) iptr->regex_extra = pcre_study(iptr->regex, 0, &errptr); // Assuming no errors since this worked before :-/ iptr->epoch = i->epoch; iptr->expiry = i->expiry; + iptr->new_result = i->new_result; iptr->next = NULL; return iptr; } @@ -177,7 +200,8 @@ prune_config(invalidate_t **i) ilast = NULL; while (iptr) { if (difftime(iptr->expiry, now) < 0) { - TSDebug(LOG_PREFIX, "Removing %s expiry: %d now: %d", iptr->regex_text, (int)iptr->expiry, (int)now); + TSDebug(LOG_PREFIX, "Removing %s expiry: %d type: %s now: %d", iptr->regex_text, (int)iptr->expiry, + strForResult(iptr->new_result), (int)now); if (ilast) { ilast->next = iptr->next; free_invalidate_t(iptr); @@ -229,18 +253,27 @@ load_config(plugin_state_t *pstate, invalidate_t **ilist) TSDebug(LOG_PREFIX, "Could not open %s for reading", path); return false; } - config_re = pcre_compile("^([^#].+?)\\s+(\\d+)\\s*$", 0, &errptr, &erroffset, NULL); + config_re = pcre_compile("^([^#].+?)\\s+(\\d+)(\\s+(\\w+))?\\s*$", 0, &errptr, &erroffset, NULL); while (fgets(line, LINE_MAX, fs) != NULL) { ln++; TSDebug(LOG_PREFIX, "Processing: %d %s", ln, line); rc = pcre_exec(config_re, NULL, line, strlen(line), 0, 0, ovector, OVECTOR_SIZE); - if (rc == 3) { + if (3 <= rc) { i = (invalidate_t *)TSmalloc(sizeof(invalidate_t)); init_invalidate_t(i); pcre_get_substring(line, ovector, rc, 1, &i->regex_text); i->epoch = now; i->expiry = atoi(line + ovector[4]); i->regex = pcre_compile(i->regex_text, 0, &errptr, &erroffset, NULL); + if (5 == rc) { + char const *const type = line + ovector[8]; + if (0 == strncasecmp(type, RESULT_MISS, strlen(RESULT_MISS))) { + TSDebug(LOG_PREFIX, "Regex line set to result type %s: '%s'", RESULT_MISS, i->regex_text); + i->new_result = TS_CACHE_LOOKUP_MISS; + } else if (0 != strncasecmp(type, RESULT_STALE, strlen(RESULT_STALE))) { + TSDebug(LOG_PREFIX, "Unknown regex line result type '%s', using default '%s' '%s'", type, RESULT_STALE, i->regex_text); + } + } if (i->expiry <= i->epoch) { TSDebug(LOG_PREFIX, "Rule is already expired!"); free_invalidate_t(i); @@ -251,7 +284,8 @@ load_config(plugin_state_t *pstate, invalidate_t **ilist) i->regex_extra = pcre_study(i->regex, 0, &errptr); if (!*ilist) { *ilist = i; - TSDebug(LOG_PREFIX, "Created new list and Loaded %s %d %d", i->regex_text, (int)i->epoch, (int)i->expiry); + TSDebug(LOG_PREFIX, "Created new list and Loaded %s %d %d %s", i->regex_text, (int)i->epoch, (int)i->expiry, + strForResult(i->new_result)); } else { iptr = *ilist; while (1) { @@ -261,6 +295,11 @@ load_config(plugin_state_t *pstate, invalidate_t **ilist) iptr->epoch = i->epoch; iptr->expiry = i->expiry; } + if (iptr->new_result != i->new_result) { + TSDebug(LOG_PREFIX, "Resetting duplicate due to type change %s", i->regex_text); + iptr->new_result = i->new_result; + iptr->epoch = now; + } free_invalidate_t(i); i = NULL; break; @@ -272,7 +311,7 @@ load_config(plugin_state_t *pstate, invalidate_t **ilist) } if (i) { iptr->next = i; - TSDebug(LOG_PREFIX, "Loaded %s %d %d", i->regex_text, (int)i->epoch, (int)i->expiry); + TSDebug(LOG_PREFIX, "Loaded %s %d %d %s", i->regex_text, (int)i->epoch, (int)i->expiry, strForResult(i->new_result)); } } } @@ -302,9 +341,11 @@ list_config(plugin_state_t *pstate, invalidate_t *i) if (i) { iptr = i; while (iptr) { - TSDebug(LOG_PREFIX, "%s epoch: %d expiry: %d", iptr->regex_text, (int)iptr->epoch, (int)iptr->expiry); + char const *const typestr = strForResult(i->new_result); + TSDebug(LOG_PREFIX, "%s epoch: %d expiry: %d result: %s", iptr->regex_text, (int)iptr->epoch, (int)iptr->expiry, typestr); if (pstate->log) { - TSTextLogObjectWrite(pstate->log, "%s epoch: %d expiry: %d", iptr->regex_text, (int)iptr->epoch, (int)iptr->expiry); + TSTextLogObjectWrite(pstate->log, "%s epoch: %d expiry: %d result: %s", iptr->regex_text, (int)iptr->epoch, + (int)iptr->expiry, typestr); } iptr = iptr->next; } @@ -419,9 +460,9 @@ main_handler(TSCont cont, TSEvent event, void *edata) url = TSHttpTxnEffectiveUrlStringGet(txn, &url_len); } if (pcre_exec(iptr->regex, iptr->regex_extra, url, url_len, 0, 0, NULL, 0) >= 0) { - TSHttpTxnCacheLookupStatusSet(txn, TS_CACHE_LOOKUP_HIT_STALE); + TSHttpTxnCacheLookupStatusSet(txn, iptr->new_result); + TSDebug(LOG_PREFIX, "Forced revalidate - %.*s %s", url_len, url, strForResult(iptr->new_result)); iptr = NULL; - TSDebug(LOG_PREFIX, "Forced revalidate - %.*s", url_len, url); } } if (iptr) { diff --git a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate_miss.test.py b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate_miss.test.py new file mode 100644 index 00000000000..bf765ebcb9f --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate_miss.test.py @@ -0,0 +1,230 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +regex_revalidate plugin test, MISS (refetch) functionality +''' + +# Test description: +# If MISS tag encountered, should load rule as refetch instead of IMS. +# If rule switched from MISS to IMS or vice versa, rule should reset. + +Test.SkipUnless( + Condition.PluginExists('regex_revalidate.so'), + Condition.PluginExists('xdebug.so') +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager") + +# **testname is required** +#testName = "regex_reval" + +# default root +request_header_0 = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } + +response_header_0 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Cache-Control: max-age=300\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "xxx", + } + +# cache item path1 +request_header_1 = {"headers": + "GET /path1 HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } +response_header_1 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path1"\r\n' + + "Cache-Control: max-age=600,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "abc" + } + + +server.addResponse("sessionlog.json", request_header_0, response_header_0) +server.addResponse("sessionlog.json", request_header_1, response_header_1) + +# Configure ATS server +ts.Disk.plugin_config.AddLine('xdebug.so') +ts.Disk.plugin_config.AddLine( + 'regex_revalidate.so -d -c regex_revalidate.conf -l revalidate.log' +) + +regex_revalidate_conf_path = os.path.join(ts.Variables.CONFIGDIR, 'regex_revalidate.conf') +#curl_and_args = 'curl -s -D - -v -H "x-debug: x-cache" -H "Host: www.example.com"' + +path1_rule = 'path1 {}'.format(int(time.time()) + 600) + +# Define first revision for when trafficserver starts +ts.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLine( + "# Empty" +) + +ts.Disk.remap_config.AddLine( + 'map http://ats/ http://127.0.0.1:{}'.format(server.Variables.Port) +) + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'regex_revalidate', + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.response_via_str': 3, + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, +}) + +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x http://127.0.0.1:{}'.format(ts.Variables.port) + ' -H "x-debug: x-cache"' + +# 0 Test - Load cache (miss) (path1) +tr = Test.AddTestRun("Cache miss path1") +ps = tr.Processes.Default +ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +ps.StartBefore(Test.Processes.ts) +ps.Command = curl_and_args + ' http://ats/path1' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss response") +tr.StillRunningAfter = ts + +# 1 Test - Cache hit path1 +tr = Test.AddTestRun("Cache hit fresh path1") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://ats/path1' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit fresh response") +tr.StillRunningAfter = ts + +# 2 Stage - Load new regex_revalidate +tr = Test.AddTestRun("Reload config add path1") +ps = tr.Processes.Default +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLine(path1_rule + ' MISS') +# keep this for debug +tr.Disk.File(regex_revalidate_conf_path + "_tr2", typename="ats:config").AddLine(path1_rule + ' MISS') +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +ps.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +ps.Env = ts.Env +ps.ReturnCode = 0 +ps.TimeOut = 5 +tr.TimeOut = 5 + +# 3 Test - Revalidate path1 +tr = Test.AddTestRun("Revalidate MISS path1") +ps = tr.Processes.Default +tr.DelayStart = 5 +ps.Command = curl_and_args + ' http://ats/path1' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss response") +tr.StillRunningAfter = ts + +# 4 Test - Cache hit (path1) +tr = Test.AddTestRun("Cache hit fresh path1") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://ats/path1' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit fresh response") +tr.StillRunningAfter = ts + +# 5 Stage - Change from MISS to STALE, reload +tr = Test.AddTestRun("Reload config path1 STLE") +ps = tr.Processes.Default +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLine(path1_rule + ' STALE') +tr.Disk.File(regex_revalidate_conf_path + "_tr5", typename="ats:config").AddLine(path1_rule + ' STALE') +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +ps.Command = 'traffic_ctl config reload' +ps.Env = ts.Env +ps.ReturnCode = 0 +ps.TimeOut = 5 +tr.TimeOut = 5 + +# 6 Test - Cache stale +tr = Test.AddTestRun("Cache stale path1") +ps = tr.Processes.Default +tr.DelayStart = 5 +ps.Command = curl_and_args + ' http://ats/path1' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale", "expected cache hit stale response") +tr.StillRunningAfter = ts + +# 7 Stage - Switch back to MISS +tr = Test.AddTestRun("Reload config path1 MISS") +ps = tr.Processes.Default +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLine(path1_rule + ' MISS') +tr.Disk.File(regex_revalidate_conf_path + "_tr7", typename="ats:config").AddLine(path1_rule + ' MISS') +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +ps.Command = 'traffic_ctl config reload' +ps.Env = ts.Env +ps.ReturnCode = 0 +ps.TimeOut = 5 +tr.TimeOut = 5 + +# 8 Test - Cache stale +tr = Test.AddTestRun("Cache stale path1") +ps = tr.Processes.Default +tr.DelayStart = 5 +ps.Command = curl_and_args + ' http://ats/path1' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss response") +tr.StillRunningAfter = ts + +# 9 Stage - Write out same contents, ensure rule not reset +tr = Test.AddTestRun("Reload config path1 MISS again") +ps = tr.Processes.Default +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLine(path1_rule + ' MISS') +tr.Disk.File(regex_revalidate_conf_path + "_tr9", typename="ats:config").AddLine(path1_rule + ' MISSSTALE') +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +ps.Command = 'traffic_ctl config reload' +ps.Env = ts.Env +ps.ReturnCode = 0 +ps.TimeOut = 5 +tr.TimeOut = 5 + +# 8 Test - Cache stale +tr = Test.AddTestRun("Cache stale path1") +ps = tr.Processes.Default +tr.DelayStart = 5 +ps.Command = curl_and_args + ' http://ats/path1' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit response") +tr.StillRunningAfter = ts