diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go
index 9c5e475114..830556a1c5 100644
--- a/pkg/cache/cache.go
+++ b/pkg/cache/cache.go
@@ -287,20 +287,17 @@ func (options Options) inheritFrom(inherited Options) (*Options, error) {
 	return &combined, nil
 }
 
-func combineScheme(schemes ...*runtime.Scheme) *runtime.Scheme {
-	var out *runtime.Scheme
-	for _, sch := range schemes {
-		if sch == nil {
-			continue
-		}
-		for gvk, t := range sch.AllKnownTypes() {
-			if out == nil {
-				out = runtime.NewScheme()
-			}
-			out.AddKnownTypeWithName(gvk, reflect.New(t).Interface().(runtime.Object))
-		}
+func combineScheme(inherited *runtime.Scheme, new *runtime.Scheme) *runtime.Scheme {
+	if inherited == nil {
+		return new
+	}
+	if new == nil {
+		return inherited
+	}
+	for gvk, t := range new.AllKnownTypes() {
+		inherited.AddKnownTypeWithName(gvk, reflect.New(t).Interface().(runtime.Object))
 	}
-	return out
+	return inherited
 }
 
 func selectMapper(def, override meta.RESTMapper) meta.RESTMapper {
diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go
index 1368ea83f0..66b46907ff 100644
--- a/pkg/manager/manager_test.go
+++ b/pkg/manager/manager_test.go
@@ -38,11 +38,13 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 
 	configv1alpha1 "k8s.io/component-base/config/v1alpha1"
+	"sigs.k8s.io/controller-runtime/examples/crd/pkg"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/cache/informertest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -1156,6 +1158,45 @@ var _ = Describe("manger.Manager", func() {
 			)
 		})
 
+		Context("with custom object selector", func() {
+			It("created with BuilderWithOptions and scheme added to manager", func() {
+				opts := Options{
+					NewCache: cache.BuilderWithOptions(cache.Options{
+						ByObject: map[client.Object]cache.ByObject{
+							&corev1.Pod{}: {
+								Label: labels.SelectorFromSet(labels.Set{
+									"test-label": "true",
+								}),
+							},
+						},
+					}),
+				}
+				m, err := New(cfg, opts)
+				Expect(err).NotTo(HaveOccurred())
+
+				var wgRunnableStarted sync.WaitGroup
+				wgRunnableStarted.Add(1)
+				Expect(m.Add(RunnableFunc(func(context.Context) error {
+					defer GinkgoRecover()
+					wgRunnableStarted.Done()
+					return nil
+				}))).To(Succeed())
+
+				Expect(pkg.AddToScheme(m.GetScheme())).NotTo(HaveOccurred())
+
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
+				go func() {
+					defer GinkgoRecover()
+					Expect(m.Elected()).ShouldNot(BeClosed())
+					Expect(m.Start(ctx)).NotTo(HaveOccurred())
+				}()
+
+				<-m.Elected()
+				wgRunnableStarted.Wait()
+			})
+		})
+
 		Context("should start serving metrics", func() {
 			var listener net.Listener
 			var opts Options