diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2bac6a9d069..5601108801f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -56,6 +56,10 @@ option(ENABLE_JEMALLOC "Use jemalloc (default OFF)")
 option(ENABLE_LUAJIT   "Use LuaJIT (default OFF)")
 option(ENABLE_MIMALLOC "Use mimalloc (default OFF)")
 option(ENABLE_DOCS "Build docs (default OFF)")
+option(ENABLE_DISK_FAILURE_TESTS "Build disk failure tests (enables AIO fault injection, default OFF)" OFF)
+if(ENABLE_DISK_FAILURE_TESTS)
+    add_compile_definitions("AIO_FAULT_INJECTION")
+endif()
 option(ENABLE_AUTEST "Setup autest (default OFF)")
 option(ENABLE_BENCHMARKS "Build benchmarks (default OFF)")
 
diff --git a/configure.ac b/configure.ac
index e2b5ad7e72e..9b84930f59a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -592,6 +592,19 @@ AC_ARG_ENABLE([32bit-build],
 )
 AC_MSG_RESULT([$enable_32bit])
 
+#
+# Check if disk failure handling tests should be built.
+# This means compiling disk failure simulation code into ATS.
+#
+AC_MSG_CHECKING([whether to build disk failure handling tests])
+AC_ARG_ENABLE([disk-failure-tests],
+  [AS_HELP_STRING([--enable-disk-failure-tests],[Build disk failure tests])],
+  [],
+  [enable_disk_failure_tests=no]
+)
+AC_MSG_RESULT([$enable_disk_failure_tests])
+AM_CONDITIONAL([ENABLE_DISK_FAILURE_TESTS], [ test "x${enable_disk_failure_tests}" = "xyes" ])
+
 
 #
 # Installation directories
@@ -1103,6 +1116,12 @@ elif test "x${enable_tsan}" = "xstatic"; then
   TS_ADDTO(AM_CXXFLAGS, [-fsanitize=thread -static-libtsan])
 fi
 
+# Build for disk failure simulation
+if test "x${enable_disk_failure_tests}" = "xyes"; then
+  TS_ADDTO(AM_CFLAGS, [-DAIO_FAULT_INJECTION])
+  TS_ADDTO(AM_CXXFLAGS, [-DAIO_FAULT_INJECTION])
+fi
+
 # Checks for pointer size.
 # TODO: Later this is irrelevant, and we should just bail on 32-bit platforms always
 AC_CHECK_SIZEOF([void*])
diff --git a/iocore/aio/AIO.cc b/iocore/aio/AIO.cc
index 62bfd4c04ac..9cd6446d940 100644
--- a/iocore/aio/AIO.cc
+++ b/iocore/aio/AIO.cc
@@ -32,6 +32,10 @@
 
 #include "P_AIO.h"
 
+#ifdef AIO_FAULT_INJECTION
+#include "AIO_fault_injection.h"
+#endif
+
 #define MAX_DISKS_POSSIBLE 100
 
 // globals
@@ -442,9 +446,19 @@ cache_op(AIOCallbackInternal *op)
     while (a->aio_nbytes - res > 0) {
       do {
         if (read) {
+#ifdef AIO_FAULT_INJECTION
+          err = aioFaultInjection.pread(a->aio_fildes, (static_cast<char *>(a->aio_buf)) + res, a->aio_nbytes - res,
+                                        a->aio_offset + res);
+#else
           err = pread(a->aio_fildes, (static_cast<char *>(a->aio_buf)) + res, a->aio_nbytes - res, a->aio_offset + res);
+#endif
         } else {
+#ifdef AIO_FAULT_INJECTION
+          err = aioFaultInjection.pwrite(a->aio_fildes, (static_cast<char *>(a->aio_buf)) + res, a->aio_nbytes - res,
+                                         a->aio_offset + res);
+#else
           err = pwrite(a->aio_fildes, (static_cast<char *>(a->aio_buf)) + res, a->aio_nbytes - res, a->aio_offset + res);
+#endif
         }
       } while ((err < 0) && (errno == EINTR || errno == ENOBUFS || errno == ENOMEM));
       if (err <= 0) {
diff --git a/iocore/aio/AIO_fault_injection.cc b/iocore/aio/AIO_fault_injection.cc
new file mode 100644
index 00000000000..9bf6dbd8f47
--- /dev/null
+++ b/iocore/aio/AIO_fault_injection.cc
@@ -0,0 +1,122 @@
+/** @file
+
+A mechanism to simulate disk failure by injecting faults in userspace.
+
+@section license License
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "AIO_fault_injection.h"
+#include "I_Lock.h"
+#include <mutex>
+
+AIOFaultInjection aioFaultInjection;
+
+void
+AIOFaultInjection::_decrement_op_count(int fd)
+{
+  auto it = _state_by_fd.find(fd);
+  if (it != _state_by_fd.end()) {
+    it->second.op_count--;
+  }
+}
+
+AIOFaultInjection::IOFault
+AIOFaultInjection::_op_result(int fd)
+{
+  auto it = _faults_by_fd.find(fd);
+  if (it != _faults_by_fd.end()) {
+    std::size_t op_count = _state_by_fd[fd].op_count++;
+    auto results         = it->second;
+    if (results.find(op_count) != results.end()) {
+      return results[op_count];
+    }
+  }
+
+  return IOFault{0, false};
+}
+
+void
+AIOFaultInjection::inject_fault(const char *path_regex, int op_index, IOFault fault)
+{
+  std::lock_guard<std::mutex> lock{_mutex};
+  _faults_by_regex[path_regex][op_index] = fault;
+}
+
+int
+AIOFaultInjection::open(const char *pathname, int flags, mode_t mode)
+{
+  std::lock_guard<std::mutex> lock{_mutex};
+  std::filesystem::path abspath = std::filesystem::absolute(pathname);
+  int fd                        = ::open(pathname, flags, mode);
+  if (fd >= 0) {
+    for (auto &[re_str, faults] : _faults_by_regex) {
+      std::regex re{re_str};
+      std::cmatch m;
+      if (std::regex_match(abspath.c_str(), m, re)) {
+        _faults_by_fd.insert_or_assign(fd, faults);
+      }
+    }
+  }
+
+  return fd;
+}
+
+ssize_t
+AIOFaultInjection::pread(int fd, void *buf, size_t nbytes, off_t offset)
+{
+  std::lock_guard<std::mutex> lock{_mutex};
+  IOFault result = _op_result(fd);
+  ssize_t ret    = 0;
+  if (result.skip_io) {
+    ink_release_assert(result.err_no != 0);
+  } else {
+    ret = ::pread(fd, buf, nbytes, offset);
+    if (ret < 0 && (errno == EINTR || errno == ENOBUFS || errno == ENOMEM)) {
+      // caller will retry
+      _decrement_op_count(fd);
+    }
+  }
+  if (result.err_no != 0) {
+    errno = result.err_no;
+    ret   = -1;
+  }
+  return ret;
+}
+
+ssize_t
+AIOFaultInjection::pwrite(int fd, const void *buf, size_t n, off_t offset)
+{
+  std::lock_guard<std::mutex> lock{_mutex};
+  IOFault result = _op_result(fd);
+  ssize_t ret    = 0;
+  if (result.skip_io) {
+    ink_release_assert(result.err_no != 0);
+  } else {
+    ret = ::pwrite(fd, buf, n, offset);
+    if (ret < 0 && (errno == EINTR || errno == ENOBUFS || errno == ENOMEM)) {
+      // caller will retry
+      _decrement_op_count(fd);
+    }
+  }
+  if (result.err_no != 0) {
+    errno = result.err_no;
+    ret   = -1;
+  }
+  return ret;
+}
diff --git a/iocore/aio/AIO_fault_injection.h b/iocore/aio/AIO_fault_injection.h
new file mode 100644
index 00000000000..b237d1a6be7
--- /dev/null
+++ b/iocore/aio/AIO_fault_injection.h
@@ -0,0 +1,72 @@
+/** @file
+
+  A mechanism to simulate disk failure by injecting faults in userspace.
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#pragma once
+
+#include "I_Continuation.h"
+#include "I_Lock.h"
+#include "tscore/ink_assert.h"
+#include "tscore/ink_mutex.h"
+#include <cerrno>
+#include <fcntl.h>
+#include <filesystem>
+#include <regex>
+#include <unistd.h>
+#include <unordered_map>
+#include <vector>
+
+// We need a way to simulate failures determininstically to test disk
+// initialization
+
+static constexpr auto TAG{"fault"};
+
+class AIOFaultInjection
+{
+  struct IOFault {
+    int err_no;
+    bool skip_io;
+  };
+  using IOFaults = std::unordered_map<int, IOFault>;
+
+  struct IOFaultState {
+    std::size_t op_count = 0;
+  };
+
+  std::unordered_map<std::string, IOFaults> _faults_by_regex;
+  std::unordered_map<int, IOFaults &> _faults_by_fd;
+  std::unordered_map<int, IOFaultState> _state_by_fd;
+
+  void _decrement_op_count(int fd);
+  IOFault _op_result(int fd);
+
+  std::mutex _mutex;
+
+public:
+  void inject_fault(const char *path_regex, int op_index, IOFault fault);
+
+  int open(const char *pathname, int flags, mode_t mode);
+  ssize_t pread(int fd, void *buf, size_t nbytes, off_t offset);
+  ssize_t pwrite(int fd, const void *buf, size_t n, off_t offset);
+};
+
+extern AIOFaultInjection aioFaultInjection;
diff --git a/iocore/aio/CMakeLists.txt b/iocore/aio/CMakeLists.txt
index ceb77b26d55..f39e4be2ba4 100644
--- a/iocore/aio/CMakeLists.txt
+++ b/iocore/aio/CMakeLists.txt
@@ -19,7 +19,8 @@
 add_library(aio STATIC)
 add_library(ts::aio ALIAS aio)
 
-target_sources(aio PRIVATE AIO.cc Inline.cc)
+target_sources(aio PRIVATE AIO.cc Inline.cc AIO_fault_injection.cc)
+
 target_include_directories(aio
     PUBLIC
         ${CMAKE_CURRENT_SOURCE_DIR}
diff --git a/iocore/aio/Makefile.am b/iocore/aio/Makefile.am
index f816f97298b..c0babc8da23 100644
--- a/iocore/aio/Makefile.am
+++ b/iocore/aio/Makefile.am
@@ -33,7 +33,9 @@ libinkaio_a_SOURCES = \
 	AIO.cc \
 	I_AIO.h \
 	Inline.cc \
-	P_AIO.h
+	P_AIO.h \
+	AIO_fault_injection.h \
+	AIO_fault_injection.cc
 
 test_AIO_LDFLAGS = \
 	@AM_LDFLAGS@ \
diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc
index 0346012d76c..d21e4772275 100644
--- a/iocore/cache/Cache.cc
+++ b/iocore/cache/Cache.cc
@@ -37,6 +37,10 @@
 
 #include "tscore/hugepages.h"
 
+#ifdef AIO_FAULT_INJECTION
+#include "AIO_fault_injection.h"
+#endif
+
 #include <atomic>
 
 constexpr ts::VersionNumber CACHE_DB_VERSION(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION);
@@ -600,11 +604,19 @@ CacheProcessor::start_internal(int flags)
       opts |= O_RDONLY;
     }
 
-    int fd         = open(paths[gndisks], opts, 0644);
+#ifdef AIO_FAULT_INJECTION
+    int fd = aioFaultInjection.open(paths[gndisks], opts, 0644);
+#else
+    int fd = open(paths[gndisks], opts, 0644);
+#endif
     int64_t blocks = sd->blocks;
 
     if (fd < 0 && (opts & O_CREAT)) { // Try without O_DIRECT if this is a file on filesystem, e.g. tmpfs.
+#ifdef AIO_FAULT_INJECTION
+      fd = aioFaultInjection.open(paths[gndisks], DEFAULT_CACHE_OPTIONS | O_CREAT, 0644);
+#else
       fd = open(paths[gndisks], DEFAULT_CACHE_OPTIONS | O_CREAT, 0644);
+#endif
     }
 
     if (fd >= 0) {
diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h
index 15645070f51..d1fff896265 100644
--- a/iocore/cache/I_CacheDefs.h
+++ b/iocore/cache/I_CacheDefs.h
@@ -24,6 +24,7 @@
 #pragma once
 
 #include "I_Event.h"
+#include "I_VConnection.h"
 #include "tscore/I_Version.h"
 #include "tscore/CryptoHash.h"
 
diff --git a/iocore/cache/Makefile.am b/iocore/cache/Makefile.am
index 34e65c2bbc3..63daee25d48 100644
--- a/iocore/cache/Makefile.am
+++ b/iocore/cache/Makefile.am
@@ -132,7 +132,164 @@ check_PROGRAMS = \
   test_Alternate_S_to_L_remove_L \
   test_Update_L_to_S \
   test_Update_S_to_L \
-  test_Update_header
+  test_Update_header \
+  test_Populated_Cache
+
+if ENABLE_DISK_FAILURE_TESTS
+check_PROGRAMS += \
+  test_Disk_Init_Failure_0 \
+  test_Disk_Init_Failure_1 \
+  test_Disk_Failure_5 \
+	test_Disk_Failure_6 \
+  test_Disk_Failure_7 \
+  test_Disk_Failure_8 \
+  test_Disk_Failure_9 \
+  test_Disk_Failure_10 \
+  test_Disk_Failure_11 \
+  test_Disk_Failure_12 \
+  test_Disk_Failure_13 \
+  test_Disk_Failure_14 \
+  test_Disk_Failure_15 \
+  test_Disk_Failure_16 \
+  test_Disk_Failure_17 \
+  test_Disk_Failure_18 \
+  test_Disk_Failure_19 \
+  test_Disk_Failure_20 \
+  test_Populated_Cache_Disk_Failure
+
+test_Disk_Init_Failure_0_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={0}
+test_Disk_Init_Failure_0_LDFLAGS = AM_LDFLAGS@
+test_Disk_Init_Failure_0_LDADD = $(test_LDADD)
+test_Disk_Init_Failure_0_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Init_Failure.cc
+
+test_Disk_Init_Failure_1_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={1}
+test_Disk_Init_Failure_1_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Init_Failure_1_LDADD = $(test_LDADD)
+test_Disk_Init_Failure_1_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Init_Failure.cc
+
+test_Disk_Failure_5_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={5}
+test_Disk_Failure_5_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_5_LDADD = $(test_LDADD)
+test_Disk_Failure_5_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_6_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={6}
+test_Disk_Failure_6_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_6_LDADD = $(test_LDADD)
+test_Disk_Failure_6_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_7_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={7}
+test_Disk_Failure_7_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_7_LDADD = $(test_LDADD)
+test_Disk_Failure_7_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_8_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={8}
+test_Disk_Failure_8_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_8_LDADD = $(test_LDADD)
+test_Disk_Failure_8_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_9_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={9}
+test_Disk_Failure_9_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_9_LDADD = $(test_LDADD)
+test_Disk_Failure_9_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_10_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={10}
+test_Disk_Failure_10_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_10_LDADD = $(test_LDADD)
+test_Disk_Failure_10_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_11_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={11}
+test_Disk_Failure_11_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_11_LDADD = $(test_LDADD)
+test_Disk_Failure_11_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_12_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={12}
+test_Disk_Failure_12_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_12_LDADD = $(test_LDADD)
+test_Disk_Failure_12_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_13_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={13}
+test_Disk_Failure_13_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_13_LDADD = $(test_LDADD)
+test_Disk_Failure_13_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_14_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={14}
+test_Disk_Failure_14_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_14_LDADD = $(test_LDADD)
+test_Disk_Failure_14_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_15_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={15}
+test_Disk_Failure_15_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_15_LDADD = $(test_LDADD)
+test_Disk_Failure_15_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_16_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={16}
+test_Disk_Failure_16_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_16_LDADD = $(test_LDADD)
+test_Disk_Failure_16_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_17_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={17}
+test_Disk_Failure_17_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_17_LDADD = $(test_LDADD)
+test_Disk_Failure_17_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_18_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={18}
+test_Disk_Failure_18_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_18_LDADD = $(test_LDADD)
+test_Disk_Failure_18_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_19_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={19}
+test_Disk_Failure_19_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_19_LDADD = $(test_LDADD)
+test_Disk_Failure_19_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Disk_Failure_20_CPPFLAGS = $(test_CPPFLAGS) -DFAILURE_INDICES={20}
+test_Disk_Failure_20_LDFLAGS = @AM_LDFLAGS@
+test_Disk_Failure_20_LDADD = $(test_LDADD)
+test_Disk_Failure_20_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Disk_Failure.cc
+
+test_Populated_Cache_Disk_Failure_CPPFLAGS = $(test_CPPFLAGS)
+test_Populated_Cache_Disk_Failure_LDFLAGS = @AM_LDFLAGS@
+test_Populated_Cache_Disk_Failure_LDADD = $(test_LDADD)
+test_Populated_Cache_Disk_Failure_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Populated_Cache_Disk_Failure.cc
+endif
 
 test_main_SOURCES = \
   ./test/main.cc \
@@ -216,6 +373,13 @@ test_Update_header_SOURCES = \
   $(test_main_SOURCES) \
   ./test/test_Update_header.cc
 
+test_Populated_Cache_CPPFLAGS = $(test_CPPFLAGS)
+test_Populated_Cache_LDFLAGS = @AM_LDFLAGS@
+test_Populated_Cache_LDADD = $(test_LDADD)
+test_Populated_Cache_SOURCES = \
+  $(test_main_SOURCES) \
+  ./test/test_Populated_Cache.cc
+
 include $(top_srcdir)/mk/tidy.mk
 
 clang-tidy-local: $(DIST_SOURCES)
diff --git a/iocore/cache/test/CacheTestHandler.cc b/iocore/cache/test/CacheTestHandler.cc
index 72ceceaf934..fa602a3bb5f 100644
--- a/iocore/cache/test/CacheTestHandler.cc
+++ b/iocore/cache/test/CacheTestHandler.cc
@@ -19,12 +19,13 @@
   limitations under the License.
  */
 
+#include "I_CacheDefs.h"
 #include "main.h"
 #include "CacheTestHandler.h"
 
 TestContChain::TestContChain() : Continuation(new_ProxyMutex()) {}
 
-CacheTestHandler::CacheTestHandler(size_t size, const char *url)
+CacheTestHandler::CacheTestHandler(size_t size, const char *url, bool expect_fail) : _expect_fail(expect_fail)
 {
   this->_wt = new CacheWriteTest(size, this, url);
   this->_rt = new CacheReadTest(size, this, url);
@@ -59,6 +60,15 @@ CacheTestHandler::handle_cache_event(int event, CacheTestBase *base)
     base->close();
     delete this;
     break;
+  case CACHE_EVENT_OPEN_READ_FAILED:
+  case CACHE_EVENT_OPEN_WRITE_FAILED:
+  case VC_EVENT_ERROR:
+    if (_expect_fail) {
+      base->close();
+      delete this;
+      break;
+    }
+    [[fallthrough]];
   default:
     REQUIRE(false);
     base->close();
diff --git a/iocore/cache/test/CacheTestHandler.h b/iocore/cache/test/CacheTestHandler.h
index ea8724a8e1b..c816cc123cd 100644
--- a/iocore/cache/test/CacheTestHandler.h
+++ b/iocore/cache/test/CacheTestHandler.h
@@ -67,7 +67,7 @@ class CacheTestHandler : public TestContChain
 {
 public:
   CacheTestHandler() = default;
-  CacheTestHandler(size_t size, const char *url = DEFAULT_URL);
+  CacheTestHandler(size_t size, const char *url = DEFAULT_URL, bool expect_fail = false);
 
   int start_test(int event, void *e);
 
@@ -76,6 +76,7 @@ class CacheTestHandler : public TestContChain
 protected:
   CacheTestBase *_rt = nullptr;
   CacheTestBase *_wt = nullptr;
+  bool _expect_fail  = false;
 };
 
 class TerminalTest : public CacheTestHandler
diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc
index c3f5042a825..58e1ccffac0 100644
--- a/iocore/cache/test/main.cc
+++ b/iocore/cache/test/main.cc
@@ -21,6 +21,8 @@
   limitations under the License.
  */
 
+#include "tscore/ink_config.h"
+#include <string_view>
 #define CATCH_CONFIG_MAIN
 #include "main.h"
 
@@ -49,16 +51,35 @@ temp_prefix()
     tmpdir = "/tmp";
   }
   snprintf(buffer, sizeof(buffer), "%s/cachetest.XXXXXX", tmpdir);
+  ink_assert(cache_vols == 1 || cache_vols == 2);
   auto prefix = swoc::file::path(mkdtemp(buffer));
   bool result = swoc::file::create_directories(prefix / "var" / "trafficserver", err, 0755);
   if (!result) {
     Dbg(dbg_ctl_cache_test, "Failed to create directories for test: %s(%s)", prefix.c_str(), err.message().c_str());
   }
   ink_assert(result);
+  if (cache_vols == 2) {
+    result = swoc::file::create_directories(prefix / "var" / "trafficserver2", err, 0755);
+    if (!result) {
+      Dbg(dbg_ctl_cache_test, "Failed to create directories for test: %s(%s)", prefix.c_str(), err.message().c_str());
+    }
+  }
+  ink_assert(result);
 
   return prefix.string();
 }
 
+// Populate the temporary directory with pre-made cache files
+static void
+populate_cache(const swoc::file::path &prefix)
+{
+  swoc::file::path src_path{TS_ABS_TOP_SRCDIR};
+  std::error_code ec;
+  ink_assert(cache_vols == 2);
+  swoc::file::copy(src_path / "iocore/cache/test/var/trafficserver/cache.db", prefix / "var/trafficserver/", ec);
+  swoc::file::copy(src_path / "iocore/cache/test/var/trafficserver2/cache.db", prefix / "var/trafficserver2/", ec);
+}
+
 void
 test_done()
 {
@@ -106,7 +127,11 @@ struct EventProcessorListener : Catch::TestEventListenerBase {
     diags()->show_location = SHOW_LOCATION_DEBUG;
 
     mime_init();
-    Layout::create(temp_prefix());
+    swoc::file::path prefix = temp_prefix();
+    Layout::create(prefix.view());
+    if (reuse_existing_cache) {
+      populate_cache(prefix);
+    }
     RecProcessInit();
     LibRecordsConfigInit();
     ink_net_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE));
diff --git a/iocore/cache/test/main.h b/iocore/cache/test/main.h
index 6ceb55e6f9c..1478ccd1418 100644
--- a/iocore/cache/test/main.h
+++ b/iocore/cache/test/main.h
@@ -233,3 +233,7 @@ class CacheReadTest : public CacheTestBase
   IOBufferReader *_reader = nullptr;
   OverridableHttpConfigParams params;
 };
+
+// Does the test use stored cache files, or initialize new files?
+extern int cache_vols;
+extern bool reuse_existing_cache;
diff --git a/iocore/cache/test/storage.config b/iocore/cache/test/storage.config
index d86334e356c..e3c34e9153d 100644
Binary files a/iocore/cache/test/storage.config and b/iocore/cache/test/storage.config differ
diff --git a/iocore/cache/test/test_Alternate_L_to_S.cc b/iocore/cache/test/test_Alternate_L_to_S.cc
index b3f263884da..c8f8e1554a3 100644
--- a/iocore/cache/test/test_Alternate_L_to_S.cc
+++ b/iocore/cache/test/test_Alternate_L_to_S.cc
@@ -24,6 +24,9 @@
 #define LARGE_FILE 10 * 1024 * 1024
 #define SMALL_FILE 10 * 1024
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 #include "main.h"
 
 class CacheAltReadAgain : public CacheTestHandler
diff --git a/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc b/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc
index 5abac622335..c0195805b42 100644
--- a/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc
+++ b/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc
@@ -26,6 +26,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 // delete dir
 Dir dir = {};
 
diff --git a/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc b/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc
index 31e9ce812d8..938eda8b8d4 100644
--- a/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc
+++ b/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc
@@ -26,6 +26,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 // delete dir
 Dir dir = {};
 
diff --git a/iocore/cache/test/test_Alternate_S_to_L.cc b/iocore/cache/test/test_Alternate_S_to_L.cc
index 9a67488406d..6195eedce27 100644
--- a/iocore/cache/test/test_Alternate_S_to_L.cc
+++ b/iocore/cache/test/test_Alternate_S_to_L.cc
@@ -26,6 +26,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 class CacheAltReadAgain : public CacheTestHandler
 {
 public:
diff --git a/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc b/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc
index 27a5a39335f..57d71c732c8 100644
--- a/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc
+++ b/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc
@@ -26,6 +26,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 // delete dir
 Dir dir = {};
 
diff --git a/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc b/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc
index b2d275608ed..151ea9bcb92 100644
--- a/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc
+++ b/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc
@@ -26,6 +26,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 // delete dir
 Dir dir = {};
 
diff --git a/iocore/cache/test/test_Cache.cc b/iocore/cache/test/test_Cache.cc
index 63284f51c54..364eaf9995a 100644
--- a/iocore/cache/test/test_Cache.cc
+++ b/iocore/cache/test/test_Cache.cc
@@ -26,6 +26,9 @@
 #define LARGE_FILE 10 * 1024 * 1024
 #define SMALL_FILE 10 * 1024
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 class CacheCommInit : public CacheInit
 {
 public:
diff --git a/iocore/cache/test/test_Disk_Failure.cc b/iocore/cache/test/test_Disk_Failure.cc
new file mode 100644
index 00000000000..94a2f92d63c
--- /dev/null
+++ b/iocore/cache/test/test_Disk_Failure.cc
@@ -0,0 +1,73 @@
+/** @file
+
+  A brief file description
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include "main.h"
+
+#define LARGE_FILE 10 * 1024 * 1024
+#define SMALL_FILE 10 * 1024
+
+#ifndef AIO_FAULT_INJECTION
+#error Must define AIO_FAULT_INJECTION!
+#endif
+#include "AIO_fault_injection.h"
+
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+extern int gndisks;
+
+class CacheCommInit : public CacheInit
+{
+public:
+  CacheCommInit() {}
+  int
+  cache_init_success_callback(int event, void *e) override
+  {
+    // We initialize two disks and inject failure in one.  Ensure that one disk
+    // remains.
+    REQUIRE(gndisks == 1);
+    CacheTestHandler *h  = new CacheTestHandler(LARGE_FILE, DEFAULT_URL, true);
+    CacheTestHandler *h2 = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com", true);
+    ;
+    TerminalTest *tt = new TerminalTest;
+    h->add(h2);
+    h->add(tt);
+    this_ethread()->schedule_imm(h);
+    delete this;
+    return 0;
+  }
+};
+
+TEST_CASE("Disk fail after initialization", "cache")
+{
+  std::vector<int> indices = FAILURE_INDICES;
+  for (const int i : indices) {
+    aioFaultInjection.inject_fault(".*/var/trafficserver/cache.db", i, {.err_no = EIO, .skip_io = true});
+  }
+  init_cache(256 * 1024 * 1024);
+  cache_config_max_disk_errors = 1;
+  // large write test
+  CacheCommInit *init = new CacheCommInit;
+
+  this_ethread()->schedule_imm(init);
+  this_thread()->execute();
+}
diff --git a/iocore/cache/test/test_Disk_Init_Failure.cc b/iocore/cache/test/test_Disk_Init_Failure.cc
new file mode 100644
index 00000000000..f10b73f6175
--- /dev/null
+++ b/iocore/cache/test/test_Disk_Init_Failure.cc
@@ -0,0 +1,72 @@
+/** @file
+
+  A brief file description
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include "main.h"
+
+#define LARGE_FILE 10 * 1024 * 1024
+#define SMALL_FILE 10 * 1024
+
+#ifndef AIO_FAULT_INJECTION
+#error Must define AIO_FAULT_INJECTION!
+#endif
+#include "AIO_fault_injection.h"
+
+int cache_vols            = 2;
+bool reuse_existing_cache = false;
+extern int gndisks;
+
+class CacheCommInit : public CacheInit
+{
+public:
+  CacheCommInit() {}
+  int
+  cache_init_success_callback(int event, void *e) override
+  {
+    // We initialize two disks and inject failure in one.  Ensure that one disk
+    // remains if the fault is during initialization.
+    REQUIRE(gndisks == 1);
+
+    CacheTestHandler *h  = new CacheTestHandler(LARGE_FILE);
+    CacheTestHandler *h2 = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com");
+    TerminalTest *tt     = new TerminalTest;
+    h->add(h2);
+    h->add(tt);
+    this_ethread()->schedule_imm(h);
+    delete this;
+    return 0;
+  }
+};
+
+TEST_CASE("Cache disk initialization fail", "cache")
+{
+  std::vector<int> indices = FAILURE_INDICES;
+  for (const int i : indices) {
+    aioFaultInjection.inject_fault(".*/var/trafficserver2/cache.db", i, {.err_no = EIO, .skip_io = true});
+  }
+  init_cache(256 * 1024 * 1024);
+  // large write test
+  CacheCommInit *init = new CacheCommInit;
+
+  this_ethread()->schedule_imm(init);
+  this_thread()->execute();
+}
diff --git a/iocore/cache/test/test_Populated_Cache.cc b/iocore/cache/test/test_Populated_Cache.cc
new file mode 100644
index 00000000000..abfdd472fc2
--- /dev/null
+++ b/iocore/cache/test/test_Populated_Cache.cc
@@ -0,0 +1,58 @@
+/** @file
+
+  A brief file description
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include "main.h"
+
+#define LARGE_FILE 10 * 1024 * 1024
+#define SMALL_FILE 10 * 1024
+
+int cache_vols            = 2;
+bool reuse_existing_cache = true;
+
+class CacheCommInit : public CacheInit
+{
+public:
+  CacheCommInit() {}
+  int
+  cache_init_success_callback(int event, void *e) override
+  {
+    CacheTestHandler *h  = new CacheTestHandler(LARGE_FILE, "http://www.example.com");
+    CacheTestHandler *h2 = new CacheTestHandler(SMALL_FILE, "http://www.scw12.com");
+    TerminalTest *tt     = new TerminalTest;
+    h->add(h2);
+    h->add(tt);
+    this_ethread()->schedule_imm(h);
+    delete this;
+    return 0;
+  }
+};
+
+TEST_CASE("cache write -> read", "cache")
+{
+  init_cache(256 * 1024 * 1024);
+  // large write test
+  CacheCommInit *init = new CacheCommInit;
+
+  this_ethread()->schedule_imm(init);
+  this_thread()->execute();
+}
diff --git a/iocore/cache/test/test_Populated_Cache_Disk_Failure.cc b/iocore/cache/test/test_Populated_Cache_Disk_Failure.cc
new file mode 100644
index 00000000000..3cea4c92b87
--- /dev/null
+++ b/iocore/cache/test/test_Populated_Cache_Disk_Failure.cc
@@ -0,0 +1,63 @@
+/** @file
+
+  A brief file description
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include "main.h"
+
+#define LARGE_FILE 10 * 1024 * 1024
+#define SMALL_FILE 10 * 1024
+
+int cache_vols            = 2;
+bool reuse_existing_cache = true;
+#ifndef AIO_FAULT_INJECTION
+#error Must define AIO_FAULT_INJECTION!
+#endif
+#include "AIO_fault_injection.h"
+
+class CacheCommInit : public CacheInit
+{
+public:
+  CacheCommInit() {}
+  int
+  cache_init_success_callback(int event, void *e) override
+  {
+    CacheTestHandler *h  = new CacheTestHandler(LARGE_FILE, "http://www.example.com");
+    CacheTestHandler *h2 = new CacheTestHandler(SMALL_FILE, "http://www.scw12.com");
+    TerminalTest *tt     = new TerminalTest;
+    h->add(h2);
+    h->add(tt);
+    this_ethread()->schedule_imm(h);
+    delete this;
+    return 0;
+  }
+};
+
+TEST_CASE("cache write -> read", "cache")
+{
+  aioFaultInjection.inject_fault(".*/var/trafficserver2/cache.db", 0, {.err_no = EIO, .skip_io = false});
+  init_cache(256 * 1024 * 1024);
+  // large write test
+  CacheCommInit *init = new CacheCommInit;
+
+  this_ethread()->schedule_imm(init);
+  this_thread()->execute();
+}
diff --git a/iocore/cache/test/test_RWW.cc b/iocore/cache/test/test_RWW.cc
index 7f7f826a54f..2b6b3fe48a8 100644
--- a/iocore/cache/test/test_RWW.cc
+++ b/iocore/cache/test/test_RWW.cc
@@ -28,6 +28,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 namespace
 {
 
diff --git a/iocore/cache/test/test_Update_L_to_S.cc b/iocore/cache/test/test_Update_L_to_S.cc
index 9b4ea8e75fb..a549ccac893 100644
--- a/iocore/cache/test/test_Update_L_to_S.cc
+++ b/iocore/cache/test/test_Update_L_to_S.cc
@@ -26,6 +26,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 class CacheUpdateReadAgain : public CacheTestHandler
 {
 public:
diff --git a/iocore/cache/test/test_Update_S_to_L.cc b/iocore/cache/test/test_Update_S_to_L.cc
index 5b0f6d16faa..906d93fa2ef 100644
--- a/iocore/cache/test/test_Update_S_to_L.cc
+++ b/iocore/cache/test/test_Update_S_to_L.cc
@@ -26,6 +26,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 class CacheUpdateReadAgain : public CacheTestHandler
 {
 public:
diff --git a/iocore/cache/test/test_Update_header.cc b/iocore/cache/test/test_Update_header.cc
index 4a9c1183a88..ce6b7995cd7 100644
--- a/iocore/cache/test/test_Update_header.cc
+++ b/iocore/cache/test/test_Update_header.cc
@@ -26,6 +26,9 @@
 
 #include "main.h"
 
+int cache_vols            = 1;
+bool reuse_existing_cache = false;
+
 class CacheUpdateReadAgain : public CacheTestHandler
 {
 public:
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index 4e659b014f1..ad351fd85fa 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -87,6 +87,18 @@ macro(add_stubbed_test name)
 endmacro()
 
 add_cache_test(Cache ${CMAKE_SOURCE_DIR}/iocore/cache/test/test_Cache.cc)
+add_cache_test(Populated_Cache ${CMAKE_SOURCE_DIR}/iocore/cache/test/test_Populated_Cache.cc)
+if(ENABLE_DISK_FAILURE_TESTS)
+foreach(i RANGE 1)
+    add_cache_test(Disk_Init_Failure_${i} ${CMAKE_SOURCE_DIR}/iocore/cache/test/test_Disk_Init_Failure.cc)
+    target_compile_definitions(Disk_Init_Failure_${i} PUBLIC FAILURE_INDICES={${i}})
+endforeach()
+foreach(i RANGE 5 20)
+    add_cache_test(Disk_Failure_${i} ${CMAKE_SOURCE_DIR}/iocore/cache/test/test_Disk_Failure.cc)
+    target_compile_definitions(Disk_Failure_${i} PUBLIC FAILURE_INDICES={${i}})
+endforeach()
+add_cache_test(Populated_Cache_Disk_Failure ${CMAKE_SOURCE_DIR}/iocore/cache/test/test_Populated_Cache_Disk_Failure.cc)
+endif()
 add_cache_test(RWW ${CMAKE_SOURCE_DIR}/iocore/cache/test/test_RWW.cc)
 add_cache_test(Alternate_L_to_S ${CMAKE_SOURCE_DIR}/iocore/cache/test/test_Alternate_L_to_S.cc)
 add_cache_test(Alternate_S_to_L ${CMAKE_SOURCE_DIR}/iocore/cache/test/test_Alternate_S_to_L.cc)