diff --git a/src/runtime/c_runtime_api.cc b/src/runtime/c_runtime_api.cc
index 0164b1bc4d39..6d65c755918a 100644
--- a/src/runtime/c_runtime_api.cc
+++ b/src/runtime/c_runtime_api.cc
@@ -103,6 +103,8 @@ class DeviceAPIManager {
   std::mutex mutex_;
   // constructor
   DeviceAPIManager() { std::fill(api_.begin(), api_.end(), nullptr); }
+  // destructor
+  ~DeviceAPIManager() { std::fill(api_.begin(), api_.end(), nullptr); }
   // Global static variable.
   static DeviceAPIManager* Global() {
     static DeviceAPIManager inst;
diff --git a/src/runtime/cuda/cuda_device_api.cc b/src/runtime/cuda/cuda_device_api.cc
index 14444c92f620..a1e25cefed0f 100644
--- a/src/runtime/cuda/cuda_device_api.cc
+++ b/src/runtime/cuda/cuda_device_api.cc
@@ -209,6 +209,8 @@ class CUDADeviceAPI final : public DeviceAPI {
 
   static const std::shared_ptr<CUDADeviceAPI>& Global() {
     static std::shared_ptr<CUDADeviceAPI> inst = std::make_shared<CUDADeviceAPI>();
+    if (inst.use_count() == 0)
+      inst = std::make_shared<CUDADeviceAPI>();
     return inst;
   }