diff --git a/libcontainer/cgroups/fs/apply_raw.go b/libcontainer/cgroups/fs/apply_raw.go
index 59ce5ea6b..849b15ffa 100644
--- a/libcontainer/cgroups/fs/apply_raw.go
+++ b/libcontainer/cgroups/fs/apply_raw.go
@@ -194,11 +194,7 @@ func (m *Manager) Destroy() error {
 	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	if err := cgroups.RemovePaths(m.Paths); err != nil {
-		return err
-	}
-	m.Paths = make(map[string]string)
-	return nil
+	return cgroups.RemovePaths(m.Paths)
 }
 
 func (m *Manager) GetPaths() map[string]string {
diff --git a/libcontainer/cgroups/systemd/v1.go b/libcontainer/cgroups/systemd/v1.go
index 8bac7ca23..c7b3863e1 100644
--- a/libcontainer/cgroups/systemd/v1.go
+++ b/libcontainer/cgroups/systemd/v1.go
@@ -212,13 +212,15 @@ func (m *LegacyManager) Destroy() error {
 	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
-
 	unitName := getUnitName(m.Cgroups)
-	if err := stopUnit(unitName); err != nil {
-		return err
+
+	err := stopUnit(unitName)
+	// Both on success and on error, cleanup all the cgroups we are aware of.
+	// Some of them were created directly by Apply() and are not managed by systemd.
+	if err2 := cgroups.RemovePaths(m.Paths); err2 != nil {
+		return err2
 	}
-	m.Paths = make(map[string]string)
-	return nil
+	return err
 }
 
 func (m *LegacyManager) GetPaths() map[string]string {
diff --git a/libcontainer/cgroups/utils.go b/libcontainer/cgroups/utils.go
index 8cdd6b7ed..a967f1497 100644
--- a/libcontainer/cgroups/utils.go
+++ b/libcontainer/cgroups/utils.go
@@ -488,6 +488,7 @@ func RemovePaths(paths map[string]string) (err error) {
 			}
 		}
 		if len(paths) == 0 {
+			paths = make(map[string]string)
 			return nil
 		}
 	}
diff --git a/tests/integration/delete.bats b/tests/integration/delete.bats
index c5ed215d1..af872144c 100644
--- a/tests/integration/delete.bats
+++ b/tests/integration/delete.bats
@@ -12,24 +12,24 @@ function teardown() {
 }
 
 @test "runc delete" {
-  # run busybox detached
-  runc run -d --console-socket $CONSOLE_SOCKET test_busybox
+  runc run -d --console-socket $CONSOLE_SOCKET testbusyboxdelete
   [ "$status" -eq 0 ]
 
-  # check state
-  testcontainer test_busybox running
+  testcontainer testbusyboxdelete running
 
-  runc kill test_busybox KILL
+  runc kill testbusyboxdelete KILL
   [ "$status" -eq 0 ]
-  # wait for busybox to be in the destroyed state
-  retry 10 1 eval "__runc state test_busybox | grep -q 'stopped'"
+  retry 10 1 eval "__runc state testbusyboxdelete | grep -q 'stopped'"
 
-  # delete test_busybox
-  runc delete test_busybox
+  runc delete testbusyboxdelete
   [ "$status" -eq 0 ]
 
-  runc state test_busybox
+  runc state testbusyboxdelete
   [ "$status" -ne 0 ]
+
+  run find /sys/fs/cgroup -wholename '*testbusyboxdelete*' -type d
+  [ "$status" -eq 0 ]
+  [ "$output" = "" ] || fail "cgroup not cleaned up correctly: $output"
 }
 
 @test "runc delete --force" {