[ca] Make the control API server take a security config instead of RootCA

ca/server.go

@@ -25,6 +25,13 @@ const (
 	defaultReconciliationRetryInterval = 10 * time.Second
 )
 
+// APISecurityConfigUpdater knows how to update a SecurityConfig from an api.Cluster object
+type APISecurityConfigUpdater interface {
+	UpdateRootCA(ctx context.Context, cluster *api.Cluster) error
+}
+
+var _ APISecurityConfigUpdater = &Server{}
+
 // Server is the CA and NodeCA API gRPC server.
 // TODO(aaronl): At some point we may want to have separate implementations of
 // CA, NodeCA, and other hypothetical future CA services. At the moment,
@@ -528,7 +535,7 @@ func (s *Server) isRunning() bool {
 // UpdateRootCA is called when there are cluster changes, and it ensures that the local RootCA is
 // always aware of changes in clusterExpiry and the Root CA key material - this can be called by
 // anything to update the root CA material
-func (s *Server) UpdateRootCA(ctx context.Context, cluster *api.Cluster) {
+func (s *Server) UpdateRootCA(ctx context.Context, cluster *api.Cluster) error {
 	s.mu.Lock()
 	s.joinTokens = cluster.RootCA.JoinTokens.Copy()
 	s.mu.Unlock()
@@ -575,7 +582,7 @@ func (s *Server) UpdateRootCA(ctx context.Context, cluster *api.Cluster) {
 
 		updatedRootCA, err := NewRootCA(rCA.CACert, signingCert, signingKey, expiry, intermediates)
 		if err != nil {
-			logger.WithError(err).Error("invalid Root CA object in cluster")
+			return errors.Wrap(err, "invalid Root CA object in cluster")
 		}
 
 		externalCARootPool := updatedRootCA.Pool
@@ -588,12 +595,11 @@ func (s *Server) UpdateRootCA(ctx context.Context, cluster *api.Cluster) {
 
 		// Attempt to update our local RootCA with the new parameters
 		if err := s.securityConfig.UpdateRootCA(&updatedRootCA, externalCARootPool); err != nil {
-			logger.WithError(err).Error("updating Root CA failed")
-		} else {
-			// only update the server cache if we've successfully updated the root CA
-			logger.Debug("Root CA updated successfully")
-			s.lastSeenClusterRootCA = cluster.RootCA.Copy()
+			return errors.Wrap(err, "updating Root CA failed")
 		}
+		// only update the server cache if we've successfully updated the root CA
+		logger.Debug("Root CA updated successfully")
+		s.lastSeenClusterRootCA = cluster.RootCA.Copy()
 	}
 
 	// we want to update if the external CA changed, or if the root CA changed because the root CA could affect what
@@ -633,6 +639,7 @@ func (s *Server) UpdateRootCA(ctx context.Context, cluster *api.Cluster) {
 		s.securityConfig.externalCA.UpdateURLs(cfsslURLs...)
 		s.lastSeenExternalCAs = cluster.Spec.CAConfig.Copy().ExternalCAs
 	}
+	return nil
 }
 
 // evaluateAndSignNodeCert implements the logic of which certificates to sign

ca/server_test.go

@@ -512,7 +512,7 @@ func TestCAServerUpdateRootCA(t *testing.T) {
 			externalCertSignedBy: cert,
 		},
 	} {
-		tc.CAServer.UpdateRootCA(context.Background(), testCase.clusterObj)
+		require.NoError(t, tc.CAServer.UpdateRootCA(context.Background(), testCase.clusterObj))
 
 		rootCA := tc.ServingSecurityConfig.RootCA()
 		require.Equal(t, testCase.rootCARoots, rootCA.Certs)

manager/controlapi/cluster.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/ca"
+	"github.com/docker/swarmkit/log"
 	"github.com/docker/swarmkit/manager/encryption"
 	"github.com/docker/swarmkit/manager/state/store"
 	gogotypes "github.com/gogo/protobuf/types"
@@ -104,16 +105,28 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe
 			return grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
 
 		}
+		// This ensures that we always have the latest security config, so our ca.SecurityConfig.RootCA and
+		// ca.SecurityConfig.externalCA objects are up-to-date with the current api.Cluster.RootCA and
+		// api.Cluster.Spec.ExternalCA objects, respectively.  Note that if, during this update, the cluster gets
+		// updated again with different CA info and the security config gets changed under us, that's still fine because
+		// this cluster update would fail anyway due to its version being too low on write.
+		if err := s.scu.UpdateRootCA(ctx, cluster); err != nil {
+			log.G(ctx).WithField(
+				"method", "(*controlapi.Server).UpdateCluster").WithError(err).Error("could not update security config")
+			return grpc.Errorf(codes.Internal, "could not update security config")
+		}
+		rootCA := s.securityConfig.RootCA()
+
 		cluster.Meta.Version = *request.ClusterVersion
 		cluster.Spec = *request.Spec.Copy()
 
 		expireBlacklistedCerts(cluster)
 
 		if request.Rotation.WorkerJoinToken {
-			cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(s.rootCA)
+			cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(rootCA)
 		}
 		if request.Rotation.ManagerJoinToken {
-			cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(s.rootCA)
+			cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(rootCA)
 		}
 
 		var unlockKeys []*api.EncryptionKey

manager/controlapi/cluster_test.go

@@ -32,13 +32,18 @@ func createClusterObj(id, name string, policy api.AcceptancePolicy, rootCA *ca.R
 	spec := createClusterSpec(name)
 	spec.AcceptancePolicy = policy
 
+	var key []byte
+	if s, err := rootCA.Signer(); err == nil {
+		key = s.Key
+	}
+
 	return &api.Cluster{
 		ID:   id,
 		Spec: *spec,
 		RootCA: api.RootCA{
-			CACert:     []byte("-----BEGIN CERTIFICATE-----AwEHoUQDQgAEZ4vGYkSt/kjoHbUjDx9eyO1xBVJEH2F+AwM9lACIZ414cD1qYy8u-----BEGIN CERTIFICATE-----"),
-			CAKey:      []byte("-----BEGIN EC PRIVATE KEY-----AwEHoUQDQgAEZ4vGYkSt/kjoHbUjDx9eyO1xBVJEH2F+AwM9lACIZ414cD1qYy8u-----END EC PRIVATE KEY-----"),
-			CACertHash: "hash",
+			CACert:     rootCA.Certs,
+			CAKey:      key,
+			CACertHash: rootCA.Digest.String(),
 			JoinTokens: api.JoinTokens{
 				Worker:  ca.GenerateJoinToken(rootCA),
 				Manager: ca.GenerateJoinToken(rootCA),
@@ -113,7 +118,7 @@ func TestGetCluster(t *testing.T) {
 	assert.Error(t, err)
 	assert.Equal(t, codes.NotFound, grpc.Code(err))
 
-	cluster := createCluster(t, ts, "name", "name", api.AcceptancePolicy{}, ts.Server.rootCA)
+	cluster := createCluster(t, ts, "name", "name", api.AcceptancePolicy{}, ts.Server.securityConfig.RootCA())
 	r, err := ts.Client.GetCluster(context.Background(), &api.GetClusterRequest{ClusterID: cluster.ID})
 	assert.NoError(t, err)
 	cluster.Meta.Version = r.Cluster.Meta.Version
@@ -140,7 +145,7 @@ func TestGetClusterWithSecret(t *testing.T) {
 	assert.Equal(t, codes.NotFound, grpc.Code(err))
 
 	policy := api.AcceptancePolicy{Policies: []*api.AcceptancePolicy_RoleAdmissionPolicy{{Secret: &api.AcceptancePolicy_RoleAdmissionPolicy_Secret{Data: []byte("secret")}}}}
-	cluster := createCluster(t, ts, "name", "name", policy, ts.Server.rootCA)
+	cluster := createCluster(t, ts, "name", "name", policy, ts.Server.securityConfig.RootCA())
 	r, err := ts.Client.GetCluster(context.Background(), &api.GetClusterRequest{ClusterID: cluster.ID})
 	assert.NoError(t, err)
 	cluster.Meta.Version = r.Cluster.Meta.Version
@@ -153,7 +158,7 @@ func TestGetClusterWithSecret(t *testing.T) {
 func TestUpdateCluster(t *testing.T) {
 	ts := newTestServer(t)
 	defer ts.Stop()
-	cluster := createCluster(t, ts, "name", "name", api.AcceptancePolicy{}, ts.Server.rootCA)
+	cluster := createCluster(t, ts, "name", "name", api.AcceptancePolicy{}, ts.Server.securityConfig.RootCA())
 
 	_, err := ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{})
 	assert.Error(t, err)
@@ -233,7 +238,7 @@ func TestUpdateCluster(t *testing.T) {
 func TestUpdateClusterRotateToken(t *testing.T) {
 	ts := newTestServer(t)
 	defer ts.Stop()
-	cluster := createCluster(t, ts, "name", "name", api.AcceptancePolicy{}, ts.Server.rootCA)
+	cluster := createCluster(t, ts, "name", "name", api.AcceptancePolicy{}, ts.Server.securityConfig.RootCA())
 
 	r, err := ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{
 		Filters: &api.ListClustersRequest_Filters{
@@ -317,7 +322,7 @@ func TestUpdateClusterRotateUnlockKey(t *testing.T) {
 	ts := newTestServer(t)
 	defer ts.Stop()
 	// create a cluster with extra encryption keys, to make sure they exist
-	cluster := createClusterObj("id", "name", api.AcceptancePolicy{}, ts.Server.rootCA)
+	cluster := createClusterObj("id", "name", api.AcceptancePolicy{}, ts.Server.securityConfig.RootCA())
 	expected := make(map[string]*api.EncryptionKey)
 	for i := 1; i <= 2; i++ {
 		value := fmt.Sprintf("fake%d", i)
@@ -440,13 +445,13 @@ func TestListClusters(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Empty(t, r.Clusters)
 
-	createCluster(t, ts, "id1", "name1", api.AcceptancePolicy{}, ts.Server.rootCA)
+	createCluster(t, ts, "id1", "name1", api.AcceptancePolicy{}, ts.Server.securityConfig.RootCA())
 	r, err = ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(r.Clusters))
 
-	createCluster(t, ts, "id2", "name2", api.AcceptancePolicy{}, ts.Server.rootCA)
-	createCluster(t, ts, "id3", "name3", api.AcceptancePolicy{}, ts.Server.rootCA)
+	createCluster(t, ts, "id2", "name2", api.AcceptancePolicy{}, ts.Server.securityConfig.RootCA())
+	createCluster(t, ts, "id3", "name3", api.AcceptancePolicy{}, ts.Server.securityConfig.RootCA())
 	r, err = ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
 	assert.NoError(t, err)
 	assert.Equal(t, 3, len(r.Clusters))
@@ -461,13 +466,13 @@ func TestListClustersWithSecrets(t *testing.T) {
 
 	policy := api.AcceptancePolicy{Policies: []*api.AcceptancePolicy_RoleAdmissionPolicy{{Secret: &api.AcceptancePolicy_RoleAdmissionPolicy_Secret{Alg: "bcrypt", Data: []byte("secret")}}}}
 
-	createCluster(t, ts, "id1", "name1", policy, ts.Server.rootCA)
+	createCluster(t, ts, "id1", "name1", policy, ts.Server.securityConfig.RootCA())
 	r, err = ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(r.Clusters))
 
-	createCluster(t, ts, "id2", "name2", policy, ts.Server.rootCA)
-	createCluster(t, ts, "id3", "name3", policy, ts.Server.rootCA)
+	createCluster(t, ts, "id2", "name2", policy, ts.Server.securityConfig.RootCA())
+	createCluster(t, ts, "id3", "name3", policy, ts.Server.securityConfig.RootCA())
 	r, err = ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
 	assert.NoError(t, err)
 	assert.Equal(t, 3, len(r.Clusters))

manager/controlapi/server.go

@@ -16,18 +16,21 @@ var (
 
 // Server is the Cluster API gRPC server.
 type Server struct {
-	store  *store.MemoryStore
-	raft   *raft.Node
-	rootCA *ca.RootCA
-	pg     plugingetter.PluginGetter
+	store          *store.MemoryStore
+	raft           *raft.Node
+	securityConfig *ca.SecurityConfig
+	scu            ca.APISecurityConfigUpdater
+	pg             plugingetter.PluginGetter
 }
 
 // NewServer creates a Cluster API server.
-func NewServer(store *store.MemoryStore, raft *raft.Node, rootCA *ca.RootCA, pg plugingetter.PluginGetter) *Server {
+func NewServer(store *store.MemoryStore, raft *raft.Node, securityConfig *ca.SecurityConfig,
+	scu ca.APISecurityConfigUpdater, pg plugingetter.PluginGetter) *Server {
 	return &Server{
-		store:  store,
-		raft:   raft,
-		rootCA: rootCA,
-		pg:     pg,
+		store:          store,
+		raft:           raft,
+		securityConfig: securityConfig,
+		scu:            scu,
+		pg:             pg,
 	}
 }

manager/controlapi/server_test.go

@@ -12,6 +12,7 @@ import (
 	"google.golang.org/grpc"
 
 	"github.com/docker/swarmkit/api"
+	"github.com/docker/swarmkit/ca"
 	cautils "github.com/docker/swarmkit/ca/testutils"
 	"github.com/docker/swarmkit/manager/state/store"
 	"github.com/stretchr/testify/assert"
@@ -56,11 +57,14 @@ func newTestServer(t *testing.T) *testServer {
 
 	// Create a testCA just to get a usable RootCA object
 	tc := cautils.NewTestCA(nil)
+	securityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
 	tc.Stop()
+	assert.NoError(t, err)
 
 	ts.Store = store.NewMemoryStore(&mockProposer{})
 	assert.NotNil(t, ts.Store)
-	ts.Server = NewServer(ts.Store, nil, &tc.RootCA, nil)
+
+	ts.Server = NewServer(ts.Store, nil, securityConfig, ca.NewServer(ts.Store, securityConfig), nil)
 	assert.NotNil(t, ts.Server)
 
 	temp, err := ioutil.TempFile("", "test-socket")

manager/manager.go

@@ -389,7 +389,7 @@ func (m *Manager) Run(parent context.Context) error {
 		return err
 	}
 
-	baseControlAPI := controlapi.NewServer(m.raftNode.MemoryStore(), m.raftNode, m.config.SecurityConfig.RootCA(), m.config.PluginGetter)
+	baseControlAPI := controlapi.NewServer(m.raftNode.MemoryStore(), m.raftNode, m.config.SecurityConfig, m.caserver, m.config.PluginGetter)
 	baseResourceAPI := resourceapi.New(m.raftNode.MemoryStore())
 	healthServer := health.NewHealthServer()
 	localHealthServer := health.NewHealthServer()
@@ -688,7 +688,9 @@ func (m *Manager) watchForClusterChanges(ctx context.Context) error {
 			if cluster == nil {
 				return fmt.Errorf("unable to get current cluster")
 			}
-			m.caserver.UpdateRootCA(ctx, cluster)
+			if err := m.caserver.UpdateRootCA(ctx, cluster); err != nil {
+				log.G(ctx).WithError(err).Error("could not update security config")
+			}
 			return m.updateKEK(ctx, cluster)
 		},
 		api.EventUpdateCluster{
@@ -704,7 +706,9 @@ func (m *Manager) watchForClusterChanges(ctx context.Context) error {
 			select {
 			case event := <-clusterWatch:
 				clusterEvent := event.(api.EventUpdateCluster)
-				m.caserver.UpdateRootCA(ctx, clusterEvent.Cluster)
+				if err := m.caserver.UpdateRootCA(ctx, clusterEvent.Cluster); err != nil {
+					log.G(ctx).WithError(err).Error("could not update security config")
+				}
 				m.updateKEK(ctx, clusterEvent.Cluster)
 			case <-ctx.Done():
 				clusterWatchCancel()