From d85de024585a2811a8a43be3fea41493230febd3 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Mon, 2 May 2022 14:17:01 -0600 Subject: [PATCH 01/11] Add support for caching complete responses to the cache range requests plugin * Adds support for caching full object responses. * Refactored logic to be more efficient. * Added new plugin parameter to relevant docs. --- .../plugins/cache_range_requests.en.rst | 9 ++++ plugins/cache_range_requests/README.md | 21 +++++++++- .../cache_range_requests.cc | 42 ++++++++++++------- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/doc/admin-guide/plugins/cache_range_requests.en.rst b/doc/admin-guide/plugins/cache_range_requests.en.rst index e677bc528dd..fa35e832869 100644 --- a/doc/admin-guide/plugins/cache_range_requests.en.rst +++ b/doc/admin-guide/plugins/cache_range_requests.en.rst @@ -185,6 +185,15 @@ status code is reset back to 206, which leads to the object not being cached. This option is useful when used with other plugins, such as Cache Promote. +Cache Complete Responses +------------------------ + +.. option:: --cache-complete-responses +.. option:: -r + +This option causes the plugin to cache complete responses (200 OK). By default, +only 206 Partial Content responses are cached by this plugin; without this flag, +any 200 OK observed will be marked as not cacheable. Configuration examples ====================== diff --git a/plugins/cache_range_requests/README.md b/plugins/cache_range_requests/README.md index 23250bef9d7..25ee9024e1a 100644 --- a/plugins/cache_range_requests/README.md +++ b/plugins/cache_range_requests/README.md @@ -79,12 +79,13 @@ X-CRR-IMS header support Consider using the header_rewrite plugin to protect the parent from using this option as an attack vector against an origin. -Object Cacheability: +Object Cacheability + Normally objects are forced into the cache by changing the status code in the response from the upstream host from 206 to 200. The default behavior is to perform this operation blindly without checking cacheability. Add the `-v` flag to cause the plugin to ensure the object is cacheable; when it is not, - the 206 status code is restored and the object will not be cached. + the status code is not changed to 200 and the object will not be cached. Global Plugin (plugin.config): @@ -95,3 +96,19 @@ Object Cacheability: @plugin=cache_range_requests.so @pparam=--verify-cacheability @plugin=cache_range_requests.so @pparam=-v + +Caching Complete Responses + + To enable caching of complete responses, that is, a 200 OK instead of a 206 Partial + Content response, add the `-r` flag to the plugin parameters. By default, complete + responses are marked as uncacheable. + + Global Plugin (plugin.config): + + cache_range_requests.so --cache-complete-responses + cache_range_requests.so -r + + Remap Plugin (remap.config): + + @plugin=cache_range_requests.so @pparam=--cache-complete-responses + @plugin=cache_range_requests.so @pparam=-r \ No newline at end of file diff --git a/plugins/cache_range_requests/cache_range_requests.cc b/plugins/cache_range_requests/cache_range_requests.cc index 23f458a8d4a..47c33856a3d 100644 --- a/plugins/cache_range_requests/cache_range_requests.cc +++ b/plugins/cache_range_requests/cache_range_requests.cc @@ -54,6 +54,7 @@ struct pluginconfig { bool consider_ims_header{false}; bool modify_cache_key{true}; bool verify_cacheability{false}; + bool cache_complete_responses{false}; std::string ims_header; }; @@ -62,6 +63,7 @@ struct txndata { TSHttpStatus origin_status{TS_HTTP_STATUS_PARTIAL_CONTENT}; time_t ims_time{0}; bool verify_cacheability{false}; + bool cache_complete_responses{false}; }; // pluginconfig struct (global plugin only) @@ -104,6 +106,7 @@ create_pluginconfig(int argc, char *const argv[]) {const_cast("no-modify-cachekey"), no_argument, nullptr, 'n'}, {const_cast("ps-cachekey"), no_argument, nullptr, 'p'}, {const_cast("verify-cacheability"), no_argument, nullptr, 'v'}, + {const_cast("cache-complete-responses"), no_argument, nullptr, 'r'}, {nullptr, 0, nullptr, 0}, }; @@ -139,6 +142,10 @@ create_pluginconfig(int argc, char *const argv[]) DEBUG_LOG("Plugin verifies whether the object in the transaction is cacheable"); pc->verify_cacheability = true; } break; + case 'r': { + DEBUG_LOG("Plugin allows complete responses (200 OK) to be cached"); + pc->cache_complete_responses = true; + } break; default: { } break; } @@ -268,7 +275,8 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc) } } - txn_state->verify_cacheability = pc->verify_cacheability; + txn_state->verify_cacheability = pc->verify_cacheability; + txn_state->cache_complete_responses = pc->cache_complete_responses; } // remove the range request header. @@ -382,22 +390,26 @@ handle_server_read_response(TSHttpTxn txnp, txndata *const txn_state) if (TS_HTTP_STATUS_PARTIAL_CONTENT == status) { DEBUG_LOG("Got TS_HTTP_STATUS_PARTIAL_CONTENT."); - DEBUG_LOG("Set response header to TS_HTTP_STATUS_OK."); - TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_OK); - - // check if transaction is cacheable - bool const cacheable = TSHttpTxnIsCacheable(txnp, nullptr, resp_buf); - DEBUG_LOG("range is cacheable: %d", cacheable); - DEBUG_LOG("verify cacheability: %d", txn_state->verify_cacheability); - - if (txn_state->verify_cacheability && !cacheable) { - DEBUG_LOG("transaction is not cacheable; resetting status code to 206"); - TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_PARTIAL_CONTENT); + if (!txn_state->verify_cacheability || (txn_state->verify_cacheability && TSHttpTxnIsCacheable(txnp, nullptr, resp_buf))) { + // changing the status code from 206 to 200 forces the object into cache + TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_OK); + DEBUG_LOG("Set response header to TS_HTTP_STATUS_OK."); + } else { + DEBUG_LOG("Range is not cacheable"); } } else if (TS_HTTP_STATUS_OK == status) { - DEBUG_LOG("The origin does not support range requests, disabling cache write."); - if (TS_SUCCESS != TSHttpTxnCntlSet(txnp, TS_HTTP_CNTL_SERVER_NO_STORE, true)) { - DEBUG_LOG("Unable to disable cache write for this transaction."); + bool cacheable = txn_state->cache_complete_responses; + + if (cacheable && txn_state->verify_cacheability) { + DEBUG_LOG("Received a cacheable complete response from the origin; verifying cacheability"); + cacheable = TSHttpTxnIsCacheable(txnp, nullptr, resp_buf); + } + + // 200s are cached by default; only cache if configured to do so + if (!cacheable && TS_SUCCESS == TSHttpTxnCntlSet(txnp, TS_HTTP_CNTL_SERVER_NO_STORE, true)) { + DEBUG_LOG("Cache write has been disabled for this transaction."); + } else { + DEBUG_LOG("Allowing object to be cached."); } } TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc); From 1b9e1576fd5759a2c6bd542f806303be511f482b Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 4 May 2022 17:35:10 -0600 Subject: [PATCH 02/11] Revert 206 flip behavior * Status code must be flipped to 200 prior to performing cacheability check * Reverts to prior logic for the partial content case --- plugins/cache_range_requests/cache_range_requests.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/cache_range_requests/cache_range_requests.cc b/plugins/cache_range_requests/cache_range_requests.cc index 47c33856a3d..8790610cce2 100644 --- a/plugins/cache_range_requests/cache_range_requests.cc +++ b/plugins/cache_range_requests/cache_range_requests.cc @@ -389,13 +389,13 @@ handle_server_read_response(TSHttpTxn txnp, txndata *const txn_state) txn_state->origin_status = status; if (TS_HTTP_STATUS_PARTIAL_CONTENT == status) { DEBUG_LOG("Got TS_HTTP_STATUS_PARTIAL_CONTENT."); + // changing the status code from 206 to 200 forces the object into cache + TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_OK); + DEBUG_LOG("Set response header to TS_HTTP_STATUS_OK."); - if (!txn_state->verify_cacheability || (txn_state->verify_cacheability && TSHttpTxnIsCacheable(txnp, nullptr, resp_buf))) { - // changing the status code from 206 to 200 forces the object into cache - TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_OK); - DEBUG_LOG("Set response header to TS_HTTP_STATUS_OK."); - } else { - DEBUG_LOG("Range is not cacheable"); + if (txn_state->verify_cacheability && !TSHttpTxnIsCacheable(txnp, nullptr, resp_buf)) { + DEBUG_LOG("transaction is not cacheable; resetting status code to 206"); + TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_PARTIAL_CONTENT); } } else if (TS_HTTP_STATUS_OK == status) { bool cacheable = txn_state->cache_complete_responses; From 30ef52b62025532df72fc660ef0d08ed1f44e288 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 4 May 2022 17:42:31 -0600 Subject: [PATCH 03/11] Update docs to reflect actual behavior. --- plugins/cache_range_requests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/cache_range_requests/README.md b/plugins/cache_range_requests/README.md index 25ee9024e1a..381f47f4d76 100644 --- a/plugins/cache_range_requests/README.md +++ b/plugins/cache_range_requests/README.md @@ -85,7 +85,7 @@ Object Cacheability response from the upstream host from 206 to 200. The default behavior is to perform this operation blindly without checking cacheability. Add the `-v` flag to cause the plugin to ensure the object is cacheable; when it is not, - the status code is not changed to 200 and the object will not be cached. + the 206 status code is restored and the object will not be cached. Global Plugin (plugin.config): From a23ace7ca378c2c5a7c1e45dfe4eef6dc183fe96 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 25 May 2022 10:49:20 -0600 Subject: [PATCH 04/11] Adds an AuTest to validate the behavior of caching complete responses with the full plugin stack. --- ..._requests_cache_complete_responses.test.py | 274 ++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py new file mode 100644 index 00000000000..c416b8e8c66 --- /dev/null +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py @@ -0,0 +1,274 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +Test.Summary = ''' +cache_range_requests cache-complete-responses test +''' + +# Test description: +# Two rounds of testing: +# Round 1: +# - seed the cache with an object that is smaller than the slice block size +# - issue requests with various ranges and validate responses are 200s +# Round 2: +# - seed the cache with an object that is larger than the slice block size +# - issue requests with various ranges and validate responses are 206s +# Both rounds test cache miss, cache hit, and refresh hit scenarios +# Use the cachekey plugin to add the `Range` request header to the cache key +# Request content through the slice and cache_range_requests plugin with a 4MB slice block size + +Test.SkipUnless( + Condition.PluginExists('cachekey.so'), + Condition.PluginExists('cache_range_requests.so'), + Condition.PluginExists('slice.so'), + Condition.PluginExists('xdebug.so'), +) +Test.ContinueOnFail = False +Test.testName = "cache_range_requests_cache_200s" + +# Generate bodies for our responses +small_body_len = 10000 +small_body = '' +for i in range(small_body_len): + small_body += 'x' + +slice_body_len = 4*1024*1024 +slice_body = '' +for i in range(slice_body_len): + slice_body += 'x' + +# Define and configure ATS +ts = Test.MakeATSProcess("ts", command="traffic_server") + +# Define and configure origin server +server = Test.MakeOriginServer("server", lookup_key="{%UID}") + +# default root +req_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "uuid: none\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_chk = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +server.addResponse("sessionlog.json", req_chk, res_chk) + +small_req = {"headers": + "GET /obj HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "UID: SMALL\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +small_resp = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Cache-Control: max-age=1\r\n" + + "Connection: close\r\n" + + 'Etag: "772102f4-56f4bc1e6d417"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": small_body + } + +small_reval_req = {"headers": + "GET /obj HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "UID: SMALL-INM\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +small_reval_resp = {"headers": + "HTTP/1.1 304 Not Modified\r\n" + + "Cache-Control: max-age=10\r\n" + + "Connection: close\r\n" + + 'Etag: "772102f4-56f4bc1e6d417"\r\n' + + "\r\n", + "timestamp": "1469733493.993" + } + +slice_req = {"headers": + "GET /slice HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes=0-4194303\r\n" + + "Accept: */*\r\n" + + "UID: SLICE\r\n" + "\r\n", + "timestamp": "1469733493.993", + } + +slice_resp = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Cache-Control: max-age=1\r\n" + + "Content-Range: bytes 0-{}/{}\r\n".format(slice_body_len - 1, slice_body_len * 2) + "\r\n" + + "Content-Length: {}\r\n".format(slice_body_len) + "\r\n" + + "Connection: close\r\n" + + 'Etag: "872104f4-d6bcaa1e6f979"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": slice_body + } + +slice_reval_req = {"headers": + "GET /slice HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "UID: SLICE-INM\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +slice_reval_resp = {"headers": + "HTTP/1.1 304 Not Modified\r\n" + + "Cache-Control: max-age=10\r\n" + + "Connection: close\r\n" + + 'Etag: "872104f4-d6bcaa1e6f979"\r\n' + + "\r\n", + "timestamp": "1469733493.993" + } + +server.addResponse("sessionlog.json", small_req, small_resp) +server.addResponse("sessionlog.json", small_reval_req, small_reval_resp) +server.addResponse("sessionlog.json", slice_req, slice_resp) +server.addResponse("sessionlog.json", slice_reval_req, slice_reval_resp) + +# remap with slice, cachekey, and the cache range requests plugin +ts.Disk.remap_config.AddLines([ + f'map http://example.com http://127.0.0.1:{server.Variables.Port} \\' + + ' @plugin=slice.so @pparam=--blockbytes=4m \\', + ' @plugin=cachekey.so @pparam=--key-type=cache_key @pparam=--include-headers=Range @pparam=--remove-all-params=true \\', + ' @plugin=cache_range_requests.so @pparam=--no-modify-cachekey @pparam=--cache-complete-responses', +]) + +# cache debug +ts.Disk.plugin_config.AddLine('xdebug.so') + +# enable debug +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cachekey|cache_range_requests|slice', +}) + +# base cURL command +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache"'.format(ts.Variables.port) + +# Test round 1: ensure we fetch and cache objects that are returned with a 200 OK and no Content-Range when the object is smaller than the slice block size + +# 0 Test - Fetch /obj with a Range header but less than 4MB +tr = Test.AddTestRun("cache miss on /obj") +ps = tr.Processes.Default +ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +ps.StartBefore(Test.Processes.ts) +ps.Command = curl_and_args + ' -H "UID: SMALL" http://example.com/obj -r 0-5000' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss, none", "expected cache miss") +tr.StillRunningAfter = ts + +# 1 Test - Fetch /obj with a different range but less than 4MB +tr = Test.AddTestRun("cache hit-fresh on /obj") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: SMALL" http://example.com/obj -r 5001-5999' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit") +tr.StillRunningAfter = ts + +# 2 Test - Revalidate /obj with a different range but less than 4MB +tr = Test.AddTestRun("cache hit-stale on /obj") +tr.DelayStart = 2 +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: SMALL-INM" http://example.com/obj -r 0-403' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale, none", "expected cache hit stale") +tr.StillRunningAfter = ts + +# 3 Test - Fetch /obj with a different range but less than 4MB +tr = Test.AddTestRun("cache hit on /obj post revalidation") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: SMALL" http://example.com/obj -r 0-3999' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit-fresh") +tr.StillRunningAfter = ts + +# Test round 2: repeat, but ensure we have 206s and matching Content-Range headers due to a base object that exceeds the slice block size + +# 4 Test - Fetch /slice with a Range header but less than 4MB +tr = Test.AddTestRun("cache miss on /slice") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: SLICE" http://example.com/slice -r 0-5000' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 Partial Content") +ps.Streams.stdout.Content = Testers.ContainsExpression("Content-Range: bytes 0-5000/8388608", "expected Content-Range: bytes 0-5000/8388608") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss, none", "expected cache miss") +tr.StillRunningAfter = ts + +# 5 Test - Fetch /slice with a different range but less than 4MB +tr = Test.AddTestRun("cache hit-fresh on /slice") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: SLICE" http://example.com/slice -r 5001-5999' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 Partial Content") +ps.Streams.stdout.Content = Testers.ContainsExpression("Content-Range: bytes 5001-5999/8388608", "expected Content-Range: bytes 5001-5999/8388608") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit") +tr.StillRunningAfter = ts + +# 6 Test - Revalidate /slice with a different range but less than 4MB +tr = Test.AddTestRun("cache hit-stale on /slice") +tr.DelayStart = 2 +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: SLICE-INM" http://example.com/slice -r 0-403' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 Partial Content") +ps.Streams.stdout.Content = Testers.ContainsExpression("Content-Range: bytes 0-403/8388608", "expected Content-Range: bytes 0-403/8388608") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale, none", "expected cache hit stale") +tr.StillRunningAfter = ts + +# 7 Test - Fetch /slice with a different range but less than 4MB +tr = Test.AddTestRun("cache hit on /slice post revalidation") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: SLICE" http://example.com/slice -r 0-3999' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 Partial Content") +ps.Streams.stdout.Content = Testers.ContainsExpression("Content-Range: bytes 0-3999/8388608", "expected Content-Range: bytes 0-3999/8388608") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit-fresh") +tr.StillRunningAfter = ts \ No newline at end of file From 84a3fd9a7dbb7d88f8d0ceebfb87d78976dabe78 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 25 May 2022 10:49:44 -0600 Subject: [PATCH 05/11] Update docs to provide more detail on the expected use case. --- .../plugins/cache_range_requests.en.rst | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/doc/admin-guide/plugins/cache_range_requests.en.rst b/doc/admin-guide/plugins/cache_range_requests.en.rst index fa35e832869..a20b7714195 100644 --- a/doc/admin-guide/plugins/cache_range_requests.en.rst +++ b/doc/admin-guide/plugins/cache_range_requests.en.rst @@ -195,6 +195,35 @@ This option causes the plugin to cache complete responses (200 OK). By default, only 206 Partial Content responses are cached by this plugin; without this flag, any 200 OK observed will be marked as not cacheable. +This option is intended to cover the case when an origin responds with a 200 OK +when the requested range exceeds the size of the object. For example, if an object +is 500 bytes, and the requested range is for bytes 0-5000, some origins will +respond with a 206 and a `Content-Range` header, while others may respond with a +200 OK and no `Content-Range` header. The same origin that responds with a 200 OK +when the requested range exceeds the object size will serve 206s when the range is +smaller than or within the bytes of the object. + +**NOTE:** This option *should be used carefully* with full knowledge of how +cache keys are set for a given remap rule that relies on this behavior and origin +response mechanics. For example, when this option is the sole argument to +`cache_range_requests.so` and no other plugins are in use, the behavior could be +abused, especially if the origin always responds with 200 OKs. This is because +the plugin will automatically include the requested `Range` in the cache key. +This means that arbitrary ranges can be used to pollute the cache with different +combinations of ranges, which will lead to many copies of the same complete object +stored under different cache keys. + +For this reason, if the plugin is instructed to cache complete responses, `Range` +request headers coming into the remap should ideally be normalized. Normalization +can be accomplished by using the slice plugin *without* the `--ref-relative` argument +which is disabled by default. The cache key plugin can also be used to tightly control +the construction of the cache key itself. + +The preferred means of using this plugin option is with the following plugins: +- slice to normalize the requested ranges, *without* the `--ref-relative` option +- cachekey to control the cache key, including the `Range` header normalized by slice +- cache range requests with `--no-modify-cachekey` and `--cache-complete-responses` + Configuration examples ====================== From a7ccf641aaed0d606cadbd1123aada92996951e8 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 25 May 2022 10:55:03 -0600 Subject: [PATCH 06/11] Removed a trailing space from the AuTest. --- .../cache_range_requests_cache_complete_responses.test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py index c416b8e8c66..679a154dcc8 100644 --- a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py @@ -31,7 +31,7 @@ # - seed the cache with an object that is larger than the slice block size # - issue requests with various ranges and validate responses are 206s # Both rounds test cache miss, cache hit, and refresh hit scenarios -# Use the cachekey plugin to add the `Range` request header to the cache key +# Use the cachekey plugin to add the `Range` request header to the cache key # Request content through the slice and cache_range_requests plugin with a 4MB slice block size Test.SkipUnless( From c5015fb343e1b6ae4e214e5519bf6775f237fac2 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 25 May 2022 10:59:28 -0600 Subject: [PATCH 07/11] Ran autopep8 --- ..._requests_cache_complete_responses.test.py | 155 ++++++++++-------- 1 file changed, 83 insertions(+), 72 deletions(-) diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py index 679a154dcc8..822ce26a952 100644 --- a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py @@ -49,7 +49,7 @@ for i in range(small_body_len): small_body += 'x' -slice_body_len = 4*1024*1024 +slice_body_len = 4 * 1024 * 1024 slice_body = '' for i in range(slice_body_len): slice_body += 'x' @@ -81,84 +81,84 @@ server.addResponse("sessionlog.json", req_chk, res_chk) small_req = {"headers": - "GET /obj HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Accept: */*\r\n" + - "UID: SMALL\r\n" - "\r\n", - "timestamp": "1469733493.993", - "body": "" - } + "GET /obj HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "UID: SMALL\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } small_resp = {"headers": - "HTTP/1.1 200 OK\r\n" + - "Cache-Control: max-age=1\r\n" + - "Connection: close\r\n" + - 'Etag: "772102f4-56f4bc1e6d417"\r\n' + - "\r\n", - "timestamp": "1469733493.993", - "body": small_body - } + "HTTP/1.1 200 OK\r\n" + + "Cache-Control: max-age=1\r\n" + + "Connection: close\r\n" + + 'Etag: "772102f4-56f4bc1e6d417"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": small_body + } small_reval_req = {"headers": - "GET /obj HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Accept: */*\r\n" + - "UID: SMALL-INM\r\n" - "\r\n", - "timestamp": "1469733493.993", - "body": "" - } + "GET /obj HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "UID: SMALL-INM\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } small_reval_resp = {"headers": - "HTTP/1.1 304 Not Modified\r\n" + - "Cache-Control: max-age=10\r\n" + - "Connection: close\r\n" + - 'Etag: "772102f4-56f4bc1e6d417"\r\n' + - "\r\n", - "timestamp": "1469733493.993" - } + "HTTP/1.1 304 Not Modified\r\n" + + "Cache-Control: max-age=10\r\n" + + "Connection: close\r\n" + + 'Etag: "772102f4-56f4bc1e6d417"\r\n' + + "\r\n", + "timestamp": "1469733493.993" + } slice_req = {"headers": - "GET /slice HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes=0-4194303\r\n" + - "Accept: */*\r\n" + - "UID: SLICE\r\n" - "\r\n", - "timestamp": "1469733493.993", - } + "GET /slice HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes=0-4194303\r\n" + + "Accept: */*\r\n" + + "UID: SLICE\r\n" + "\r\n", + "timestamp": "1469733493.993", + } slice_resp = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Cache-Control: max-age=1\r\n" + - "Content-Range: bytes 0-{}/{}\r\n".format(slice_body_len - 1, slice_body_len * 2) + "\r\n" + - "Content-Length: {}\r\n".format(slice_body_len) + "\r\n" + - "Connection: close\r\n" + - 'Etag: "872104f4-d6bcaa1e6f979"\r\n' + - "\r\n", - "timestamp": "1469733493.993", - "body": slice_body - } + "HTTP/1.1 206 Partial Content\r\n" + + "Cache-Control: max-age=1\r\n" + + "Content-Range: bytes 0-{}/{}\r\n".format(slice_body_len - 1, slice_body_len * 2) + "\r\n" + + "Content-Length: {}\r\n".format(slice_body_len) + "\r\n" + + "Connection: close\r\n" + + 'Etag: "872104f4-d6bcaa1e6f979"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": slice_body + } slice_reval_req = {"headers": - "GET /slice HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Accept: */*\r\n" + - "UID: SLICE-INM\r\n" - "\r\n", - "timestamp": "1469733493.993", - "body": "" - } + "GET /slice HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "UID: SLICE-INM\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } slice_reval_resp = {"headers": - "HTTP/1.1 304 Not Modified\r\n" + - "Cache-Control: max-age=10\r\n" + - "Connection: close\r\n" + - 'Etag: "872104f4-d6bcaa1e6f979"\r\n' + - "\r\n", - "timestamp": "1469733493.993" - } + "HTTP/1.1 304 Not Modified\r\n" + + "Cache-Control: max-age=10\r\n" + + "Connection: close\r\n" + + 'Etag: "872104f4-d6bcaa1e6f979"\r\n' + + "\r\n", + "timestamp": "1469733493.993" + } server.addResponse("sessionlog.json", small_req, small_resp) server.addResponse("sessionlog.json", small_reval_req, small_reval_resp) @@ -185,7 +185,9 @@ # base cURL command curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache"'.format(ts.Variables.port) -# Test round 1: ensure we fetch and cache objects that are returned with a 200 OK and no Content-Range when the object is smaller than the slice block size +# Test round 1: ensure we fetch and cache objects that are returned with a +# 200 OK and no Content-Range when the object is smaller than the slice +# block size # 0 Test - Fetch /obj with a Range header but less than 4MB tr = Test.AddTestRun("cache miss on /obj") @@ -230,7 +232,8 @@ ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit-fresh") tr.StillRunningAfter = ts -# Test round 2: repeat, but ensure we have 206s and matching Content-Range headers due to a base object that exceeds the slice block size +# Test round 2: repeat, but ensure we have 206s and matching Content-Range +# headers due to a base object that exceeds the slice block size # 4 Test - Fetch /slice with a Range header but less than 4MB tr = Test.AddTestRun("cache miss on /slice") @@ -238,7 +241,9 @@ ps.Command = curl_and_args + ' -H "UID: SLICE" http://example.com/slice -r 0-5000' ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 Partial Content") -ps.Streams.stdout.Content = Testers.ContainsExpression("Content-Range: bytes 0-5000/8388608", "expected Content-Range: bytes 0-5000/8388608") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "Content-Range: bytes 0-5000/8388608", + "expected Content-Range: bytes 0-5000/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss, none", "expected cache miss") tr.StillRunningAfter = ts @@ -248,7 +253,9 @@ ps.Command = curl_and_args + ' -H "UID: SLICE" http://example.com/slice -r 5001-5999' ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 Partial Content") -ps.Streams.stdout.Content = Testers.ContainsExpression("Content-Range: bytes 5001-5999/8388608", "expected Content-Range: bytes 5001-5999/8388608") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "Content-Range: bytes 5001-5999/8388608", + "expected Content-Range: bytes 5001-5999/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit") tr.StillRunningAfter = ts @@ -259,7 +266,9 @@ ps.Command = curl_and_args + ' -H "UID: SLICE-INM" http://example.com/slice -r 0-403' ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 Partial Content") -ps.Streams.stdout.Content = Testers.ContainsExpression("Content-Range: bytes 0-403/8388608", "expected Content-Range: bytes 0-403/8388608") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "Content-Range: bytes 0-403/8388608", + "expected Content-Range: bytes 0-403/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale, none", "expected cache hit stale") tr.StillRunningAfter = ts @@ -269,6 +278,8 @@ ps.Command = curl_and_args + ' -H "UID: SLICE" http://example.com/slice -r 0-3999' ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 Partial Content") -ps.Streams.stdout.Content = Testers.ContainsExpression("Content-Range: bytes 0-3999/8388608", "expected Content-Range: bytes 0-3999/8388608") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "Content-Range: bytes 0-3999/8388608", + "expected Content-Range: bytes 0-3999/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit-fresh") -tr.StillRunningAfter = ts \ No newline at end of file +tr.StillRunningAfter = ts From 6e907aca75fca63a5879d9816cd62b2a7f37ffd2 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 25 May 2022 15:50:56 -0600 Subject: [PATCH 08/11] Added a few test cases to cover when the CRR plugin is used without slice and cachekey. --- ..._requests_cache_complete_responses.test.py | 188 +++++++++++++++++- 1 file changed, 182 insertions(+), 6 deletions(-) diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py index 822ce26a952..19283dbba5d 100644 --- a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py @@ -23,16 +23,20 @@ ''' # Test description: -# Two rounds of testing: +# Three rounds of testing: # Round 1: # - seed the cache with an object that is smaller than the slice block size # - issue requests with various ranges and validate responses are 200s # Round 2: # - seed the cache with an object that is larger than the slice block size # - issue requests with various ranges and validate responses are 206s -# Both rounds test cache miss, cache hit, and refresh hit scenarios -# Use the cachekey plugin to add the `Range` request header to the cache key -# Request content through the slice and cache_range_requests plugin with a 4MB slice block size +# The first two rounds test cache miss, cache hit, and refresh hit scenarios +# - uses the cachekey plugin to add the `Range` request header to the cache key +# - requests content through the slice and cache_range_requests plugin with a 4MB slice block size +# - demonstrates how one might normalize the `Range` header to avoid cache pollution +# The third round tests cache miss and hit scenarios without any other plugins +# - tests cache misses, then hits, using the same object but two different ranges +# - demonstrates why normalization of the `Range` header is required to prevent pollution Test.SkipUnless( Condition.PluginExists('cachekey.so'), @@ -160,12 +164,62 @@ "timestamp": "1469733493.993" } +naieve_req = {"headers": + "GET /naieve/obj HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "UID: NAIEVE\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +naieve_resp = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Cache-Control: max-age=1\r\n" + + "Connection: close\r\n" + + 'Etag: "cad04ff4-56f4bc197ceda"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": small_body + } + +naieve_reval_req = {"headers": + "GET /naieve/obj HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "UID: NAIEVE-INM\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +naieve_reval_resp = {"headers": + "HTTP/1.1 304 Not Modified\r\n" + + "Cache-Control: max-age=10\r\n" + + "Connection: close\r\n" + + 'Etag: "cad04ff4-56f4bc197ceda"\r\n' + + "\r\n", + "timestamp": "1469733493.993" + } + + server.addResponse("sessionlog.json", small_req, small_resp) server.addResponse("sessionlog.json", small_reval_req, small_reval_resp) server.addResponse("sessionlog.json", slice_req, slice_resp) server.addResponse("sessionlog.json", slice_reval_req, slice_reval_resp) +server.addResponse("sessionlog.json", naieve_req, naieve_resp) +server.addResponse("sessionlog.json", naieve_reval_req, naieve_reval_resp) -# remap with slice, cachekey, and the cache range requests plugin +# remap with the cache range requests plugin only +# this is a "naieve" configuration due to the lack of range normalization performed at remap time by slice +# this config should only be used if ranges have been reliably normalized by the requestor (either the client itself or a cache) +ts.Disk.remap_config.AddLines([ + f'map http://example.com/naieve http://127.0.0.1:{server.Variables.Port}/naieve \\' + + ' @plugin=cache_range_requests.so @pparam=--cache-complete-responses', +]) + +# remap with slice, cachekey, and the cache range requests plugin to ensure range normalization and cache keys are correct ts.Disk.remap_config.AddLines([ f'map http://example.com http://127.0.0.1:{server.Variables.Port} \\' + ' @plugin=slice.so @pparam=--blockbytes=4m \\', @@ -183,7 +237,7 @@ }) # base cURL command -curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache"'.format(ts.Variables.port) +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache, x-cache-key"'.format(ts.Variables.port) # Test round 1: ensure we fetch and cache objects that are returned with a # 200 OK and no Content-Range when the object is smaller than the slice @@ -199,6 +253,9 @@ ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss, none", "expected cache miss") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/obj", + "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts # 1 Test - Fetch /obj with a different range but less than 4MB @@ -209,6 +266,9 @@ ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/obj", + "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts # 2 Test - Revalidate /obj with a different range but less than 4MB @@ -220,6 +280,9 @@ ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale, none", "expected cache hit stale") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/obj", + "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts # 3 Test - Fetch /obj with a different range but less than 4MB @@ -230,6 +293,9 @@ ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit-fresh") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/obj", + "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts # Test round 2: repeat, but ensure we have 206s and matching Content-Range @@ -245,6 +311,9 @@ "Content-Range: bytes 0-5000/8388608", "expected Content-Range: bytes 0-5000/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss, none", "expected cache miss") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/slice", + "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts # 5 Test - Fetch /slice with a different range but less than 4MB @@ -257,6 +326,9 @@ "Content-Range: bytes 5001-5999/8388608", "expected Content-Range: bytes 5001-5999/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/slice", + "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts # 6 Test - Revalidate /slice with a different range but less than 4MB @@ -270,6 +342,9 @@ "Content-Range: bytes 0-403/8388608", "expected Content-Range: bytes 0-403/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale, none", "expected cache hit stale") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/slice", + "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts # 7 Test - Fetch /slice with a different range but less than 4MB @@ -282,4 +357,105 @@ "Content-Range: bytes 0-3999/8388608", "expected Content-Range: bytes 0-3999/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit-fresh") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/slice", + "expected cache key with bytes 0-4194303") +tr.StillRunningAfter = ts + +# Test round 3: test behavior of the cache range requests plugin when caching complete ranges *without* the slice and cachekey plugins +# this tests the "naieve" case that requires range normalization to be +# performed by the requestor and demonstrates how the cache can be +# polluted without normalization + +# 8 Test - Fetch /naieve/obj with a Range header +tr = Test.AddTestRun("cache miss on /naieve/obj") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 0-5000' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "expected cache key with bytes 0-5000") +tr.StillRunningAfter = ts + +# 9 Test - Fetch /naieve/obj with the same Range header +tr = Test.AddTestRun("cache hit-fresh on /naieve/obj") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 0-5000' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "expected cache key with bytes 0-5000") +tr.StillRunningAfter = ts + +# 10 Test - Revalidate /naieve/obj with the same Range header +tr = Test.AddTestRun("cache hit-stale on /naieve/obj") +tr.DelayStart = 2 +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: NAIEVE-INM" http://example.com/naieve/obj -r 0-5000' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale", "expected cache hit stale") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "expected cache key with bytes 0-5000") +tr.StillRunningAfter = ts + +# 11 Test - Fetch /naieve/obj with the same Range header +tr = Test.AddTestRun("cache hit on /naieve/obj post revalidation") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 0-5000' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit-fresh") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "expected cache key with bytes 0-5000") +tr.StillRunningAfter = ts + +# 9 Test - Fetch /naieve/obj with a *different* Range header; note the cache key changes and is a miss for the same object +tr = Test.AddTestRun("cache miss on /naieve/obj") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 444-777' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=444-777", + "expected cache key with bytes 444-777") +tr.StillRunningAfter = ts + +# 10 Test - Fetch /naieve/obj with the prior Range header; now a cache hit but we've effectively cached /naieve/obj twice +# this is why a Range normalization strategy should _always_ be employed when using `--cache-complete-responses` +tr = Test.AddTestRun("cache miss on /naieve/obj") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 444-777' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit", "expected cache hit-fresh") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=444-777", + "expected cache key with bytes 444-777") +tr.StillRunningAfter = ts + +# 11 Test - Fetch /naieve/obj with the original Range header (0-5000); still a cache hit +tr = Test.AddTestRun("cache hit on /naieve/obj after requesting with a different Range") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 0-5000' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("200 OK", "expected 200 OK") +ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit-fresh") +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "expected cache key with bytes 0-5000") tr.StillRunningAfter = ts From 13091ab518880adbf7f674e0208aad51cd218da5 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 25 May 2022 15:54:40 -0600 Subject: [PATCH 09/11] Fix test case numbering. --- .../cache_range_requests_cache_complete_responses.test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py index 19283dbba5d..7b39b584923 100644 --- a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py @@ -420,7 +420,7 @@ "expected cache key with bytes 0-5000") tr.StillRunningAfter = ts -# 9 Test - Fetch /naieve/obj with a *different* Range header; note the cache key changes and is a miss for the same object +# 12 Test - Fetch /naieve/obj with a *different* Range header; note the cache key changes and is a miss for the same object tr = Test.AddTestRun("cache miss on /naieve/obj") ps = tr.Processes.Default ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 444-777' @@ -433,7 +433,7 @@ "expected cache key with bytes 444-777") tr.StillRunningAfter = ts -# 10 Test - Fetch /naieve/obj with the prior Range header; now a cache hit but we've effectively cached /naieve/obj twice +# 13 Test - Fetch /naieve/obj with the prior Range header; now a cache hit but we've effectively cached /naieve/obj twice # this is why a Range normalization strategy should _always_ be employed when using `--cache-complete-responses` tr = Test.AddTestRun("cache miss on /naieve/obj") ps = tr.Processes.Default @@ -447,7 +447,7 @@ "expected cache key with bytes 444-777") tr.StillRunningAfter = ts -# 11 Test - Fetch /naieve/obj with the original Range header (0-5000); still a cache hit +# 14 Test - Fetch /naieve/obj with the original Range header (0-5000); still a cache hit tr = Test.AddTestRun("cache hit on /naieve/obj after requesting with a different Range") ps = tr.Processes.Default ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 0-5000' From 28c029df6d0c571007e4273c81826e5658849f3c Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Wed, 25 May 2022 15:57:55 -0600 Subject: [PATCH 10/11] Fix test name/comment. --- .../cache_range_requests_cache_complete_responses.test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py index 7b39b584923..7960d92619b 100644 --- a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py @@ -34,7 +34,7 @@ # - uses the cachekey plugin to add the `Range` request header to the cache key # - requests content through the slice and cache_range_requests plugin with a 4MB slice block size # - demonstrates how one might normalize the `Range` header to avoid cache pollution -# The third round tests cache miss and hit scenarios without any other plugins +# The third round tests cache miss, hit, and refresh hit scenarios without any other plugins # - tests cache misses, then hits, using the same object but two different ranges # - demonstrates why normalization of the `Range` header is required to prevent pollution @@ -435,7 +435,7 @@ # 13 Test - Fetch /naieve/obj with the prior Range header; now a cache hit but we've effectively cached /naieve/obj twice # this is why a Range normalization strategy should _always_ be employed when using `--cache-complete-responses` -tr = Test.AddTestRun("cache miss on /naieve/obj") +tr = Test.AddTestRun("cache hit on /naieve/obj") ps = tr.Processes.Default ps.Command = curl_and_args + ' -H "UID: NAIEVE" http://example.com/naieve/obj -r 444-777' ps.ReturnCode = 0 From 204fa7a08fe73a1bcd930fd726eb2b999810a444 Mon Sep 17 00:00:00 2001 From: Jeff Elsloo Date: Fri, 27 May 2022 08:48:42 -0600 Subject: [PATCH 11/11] Update AuTest check on cachekey to use non-greedy regex to work in the CI sandbox. --- ..._requests_cache_complete_responses.test.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py index 7960d92619b..371ed64873f 100644 --- a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py @@ -254,7 +254,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss, none", "expected cache miss") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/obj", + "X-Cache-Key: /.*?/Range:bytes=0-4194303/obj", "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts @@ -267,7 +267,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/obj", + "X-Cache-Key: /.*?/Range:bytes=0-4194303/obj", "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts @@ -281,7 +281,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale, none", "expected cache hit stale") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/obj", + "X-Cache-Key: /.*?/Range:bytes=0-4194303/obj", "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts @@ -294,7 +294,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit-fresh") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/obj", + "X-Cache-Key: /.*?/Range:bytes=0-4194303/obj", "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts @@ -312,7 +312,7 @@ "expected Content-Range: bytes 0-5000/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss, none", "expected cache miss") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/slice", + "X-Cache-Key: /.*?/Range:bytes=0-4194303/slice", "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts @@ -327,7 +327,7 @@ "expected Content-Range: bytes 5001-5999/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/slice", + "X-Cache-Key: /.*?/Range:bytes=0-4194303/slice", "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts @@ -343,7 +343,7 @@ "expected Content-Range: bytes 0-403/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale, none", "expected cache hit stale") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/slice", + "X-Cache-Key: /.*?/Range:bytes=0-4194303/slice", "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts @@ -358,7 +358,7 @@ "expected Content-Range: bytes 0-3999/8388608") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh, none", "expected cache hit-fresh") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: /127.0.0.1/2005/Range:bytes=0-4194303/slice", + "X-Cache-Key: /.*?/Range:bytes=0-4194303/slice", "expected cache key with bytes 0-4194303") tr.StillRunningAfter = ts @@ -376,7 +376,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "X-Cache-Key: http://.*?/naieve/obj-bytes=0-5000", "expected cache key with bytes 0-5000") tr.StillRunningAfter = ts @@ -389,7 +389,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "X-Cache-Key: http://.*?/naieve/obj-bytes=0-5000", "expected cache key with bytes 0-5000") tr.StillRunningAfter = ts @@ -403,7 +403,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale", "expected cache hit stale") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "X-Cache-Key: http://.*?/naieve/obj-bytes=0-5000", "expected cache key with bytes 0-5000") tr.StillRunningAfter = ts @@ -416,7 +416,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit-fresh") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "X-Cache-Key: http://.*?/naieve/obj-bytes=0-5000", "expected cache key with bytes 0-5000") tr.StillRunningAfter = ts @@ -429,7 +429,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=444-777", + "X-Cache-Key: http://.*?/naieve/obj-bytes=444-777", "expected cache key with bytes 444-777") tr.StillRunningAfter = ts @@ -443,7 +443,7 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit", "expected cache hit-fresh") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=444-777", + "X-Cache-Key: http://.*?/naieve/obj-bytes=444-777", "expected cache key with bytes 444-777") tr.StillRunningAfter = ts @@ -456,6 +456,6 @@ ps.Streams.stdout.Content = Testers.ExcludesExpression("Content-Range:", "expected no Content-Range header") ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh", "expected cache hit-fresh") ps.Streams.stdout.Content = Testers.ContainsExpression( - "X-Cache-Key: http://127.0.0.1:2005/naieve/obj-bytes=0-5000", + "X-Cache-Key: http://.*?/naieve/obj-bytes=0-5000", "expected cache key with bytes 0-5000") tr.StillRunningAfter = ts