diff --git a/fs/fstest/compare.go b/fs/fstest/compare.go
index b61d8308..0d100b62 100644
--- a/fs/fstest/compare.go
+++ b/fs/fstest/compare.go
@@ -49,15 +49,7 @@ func CheckDirectoryEqual(d1, d2 string) error {
 
 	diff := diffResourceList(m1.Resources, m2.Resources)
 	if diff.HasDiff() {
-		if len(diff.Deletions) != 0 {
-			return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String())
-		}
-		// TODO: Also skip Recycle Bin contents in Windows layers which is used to store deleted files in some cases
-		for _, add := range diff.Additions {
-			if ok, _ := metadataFiles[add.Path()]; !ok {
-				return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String())
-			}
-		}
+		return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String())
 	}
 
 	return nil
diff --git a/fs/fstest/compare_windows.go b/fs/fstest/compare_windows.go
index 6b9104de..a3578199 100644
--- a/fs/fstest/compare_windows.go
+++ b/fs/fstest/compare_windows.go
@@ -17,6 +17,7 @@
 package fstest
 
 // TODO: Any more metadata files generated by Windows layers?
+// TODO: Also skip Recycle Bin contents in Windows layers which is used to store deleted files in some cases
 var metadataFiles = map[string]bool{
 	"\\System Volume Information": true,
 	"\\WcSandboxState":            true,
diff --git a/fs/fstest/continuity_util.go b/fs/fstest/continuity_util.go
index 9cbfc0b6..4d30dd01 100644
--- a/fs/fstest/continuity_util.go
+++ b/fs/fstest/continuity_util.go
@@ -42,7 +42,17 @@ type resourceListDifference struct {
 }
 
 func (l resourceListDifference) HasDiff() bool {
-	return len(l.Additions) > 0 || len(l.Deletions) > 0 || len(l.Updates) > 0
+	if len(l.Deletions) > 0 || len(l.Updates) > 0 || (len(metadataFiles) == 0 && len(l.Additions) > 0) {
+		return true
+	}
+
+	for _, add := range l.Additions {
+		if ok, _ := metadataFiles[add.Path()]; !ok {
+			return true
+		}
+	}
+
+	return false
 }
 
 func (l resourceListDifference) String() string {