[manager/state] Add fernet as an option for raft encryption

api/api.pb.txt

@@ -3928,6 +3928,13 @@ file {
           66001: "NACLSecretboxSalsa20Poly1305"
         }
       }
+      value {
+        name: "FERNET_AES_128_CBC"
+        number: 2
+        options {
+          66001: "FernetAES128CBC"
+        }
+      }
     }
   }
   message_type {

api/types.proto

@@ -1035,6 +1035,7 @@ message MaybeEncryptedRecord {
 	enum Algorithm {
 		NONE = 0 [(gogoproto.enumvalue_customname) = "NotEncrypted"];
 		SECRETBOX_SALSA20_POLY1305 = 1 [(gogoproto.enumvalue_customname) = "NACLSecretboxSalsa20Poly1305"];
+		FERNET_AES_128_CBC = 2 [(gogoproto.enumvalue_customname) = "FernetAES128CBC"];
 	}
 
 	Algorithm algorithm = 1;

ca/certificates.go

@@ -29,6 +29,7 @@ import (
 	"github.com/docker/swarmkit/ca/keyutils"
 	"github.com/docker/swarmkit/ca/pkcs8"
 	"github.com/docker/swarmkit/connectionbroker"
+	"github.com/docker/swarmkit/fips"
 	"github.com/docker/swarmkit/ioutils"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -818,7 +819,7 @@ func CreateRootCA(rootCN string) (RootCA, error) {
 	}
 
 	// Convert key to PKCS#8 in FIPS mode
-	if keyutils.FIPSEnabled() {
+	if fips.Enabled() {
 		key, err = pkcs8.ConvertECPrivateKeyPEM(key)
 		if err != nil {
 			return RootCA{}, err

ca/certificates_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/docker/swarmkit/ca/pkcs8"
 	cautils "github.com/docker/swarmkit/ca/testutils"
 	"github.com/docker/swarmkit/connectionbroker"
+	"github.com/docker/swarmkit/fips"
 	"github.com/docker/swarmkit/identity"
 	"github.com/docker/swarmkit/manager/state"
 	"github.com/docker/swarmkit/manager/state/store"
@@ -94,8 +95,8 @@ func TestCreateRootCAKeyFormat(t *testing.T) {
 	require.Equal(t, "EC PRIVATE KEY", block.Type)
 
 	// Check if the CA key generated is PKCS#8 when FIPS-mode is on
-	os.Setenv(keyutils.FIPSEnvVar, "1")
-	defer os.Unsetenv(keyutils.FIPSEnvVar)
+	os.Setenv(fips.EnvVar, "1")
+	defer os.Unsetenv(fips.EnvVar)
 
 	rootCA, err = ca.CreateRootCA("rootCA")
 	require.NoError(t, err)

ca/keyutils/keyutils.go

@@ -10,22 +10,14 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"errors"
-	"os"
 
 	"github.com/cloudflare/cfssl/helpers"
 	"github.com/docker/swarmkit/ca/pkcs8"
+	"github.com/docker/swarmkit/fips"
 )
 
 var errFIPSUnsupportedKeyFormat = errors.New("unsupported key format due to FIPS compliance")
 
-// FIPSEnvVar is the environment variable which stores FIPS mode state
-const FIPSEnvVar = "GOFIPS"
-
-// FIPSEnabled returns true when FIPS mode is enabled
-func FIPSEnabled() bool {
-	return os.Getenv(FIPSEnvVar) != ""
-}
-
 // IsPKCS8 returns true if the provided der bytes is encrypted/unencrypted PKCS#8 key
 func IsPKCS8(derBytes []byte) bool {
 	if _, err := x509.ParsePKCS8PrivateKey(derBytes); err == nil {
@@ -49,7 +41,7 @@ func ParsePrivateKeyPEMWithPassword(pemBytes, password []byte) (crypto.Signer, e
 
 	if IsPKCS8(block.Bytes) {
 		return pkcs8.ParsePrivateKeyPEMWithPassword(pemBytes, password)
-	} else if FIPSEnabled() {
+	} else if fips.Enabled() {
 		return nil, errFIPSUnsupportedKeyFormat
 	}
 
@@ -59,15 +51,15 @@ func ParsePrivateKeyPEMWithPassword(pemBytes, password []byte) (crypto.Signer, e
 // IsEncryptedPEMBlock checks if a PKCS#1 or PKCS#8 PEM-block is encrypted or not
 // It returns false in FIPS mode even if PKCS#1 is encrypted
 func IsEncryptedPEMBlock(block *pem.Block) bool {
-	return pkcs8.IsEncryptedPEMBlock(block) || (!FIPSEnabled() && x509.IsEncryptedPEMBlock(block))
+	return pkcs8.IsEncryptedPEMBlock(block) || (!fips.Enabled() && x509.IsEncryptedPEMBlock(block))
 }
 
 // DecryptPEMBlock requires PKCS#1 or PKCS#8 PEM Block and password to decrypt and return unencrypted der []byte
 // It returns an error in FIPS mode when PKCS#1 PEM Block is passed.
 func DecryptPEMBlock(block *pem.Block, password []byte) ([]byte, error) {
 	if IsPKCS8(block.Bytes) {
 		return pkcs8.DecryptPEMBlock(block, password)
-	} else if FIPSEnabled() {
+	} else if fips.Enabled() {
 		return nil, errFIPSUnsupportedKeyFormat
 	}
 
@@ -79,7 +71,7 @@ func DecryptPEMBlock(block *pem.Block, password []byte) ([]byte, error) {
 func EncryptPEMBlock(data, password []byte) (*pem.Block, error) {
 	if IsPKCS8(data) {
 		return pkcs8.EncryptPEMBlock(data, password)
-	} else if FIPSEnabled() {
+	} else if fips.Enabled() {
 		return nil, errFIPSUnsupportedKeyFormat
 	}
 

ca/keyutils/keyutils_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/docker/swarmkit/fips"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -50,12 +51,12 @@ aMbljbOLAjpZS3/VnQteab4=
 )
 
 func TestFIPSEnabled(t *testing.T) {
-	os.Unsetenv(FIPSEnvVar)
-	assert.False(t, FIPSEnabled())
+	os.Unsetenv(fips.EnvVar)
+	assert.False(t, fips.Enabled())
 
-	os.Setenv(FIPSEnvVar, "1")
-	defer os.Unsetenv(FIPSEnvVar)
-	assert.True(t, FIPSEnabled())
+	os.Setenv(fips.EnvVar, "1")
+	defer os.Unsetenv(fips.EnvVar)
+	assert.True(t, fips.Enabled())
 }
 
 func TestIsPKCS8(t *testing.T) {
@@ -70,7 +71,7 @@ func TestIsPKCS8(t *testing.T) {
 
 func TestIsEncryptedPEMBlock(t *testing.T) {
 	// Disable FIPS mode
-	os.Unsetenv(FIPSEnvVar)
+	os.Unsetenv(fips.EnvVar)
 
 	// Check PKCS8 keys
 	assert.False(t, IsEncryptedPEMBlock(decryptedPKCS8Block))
@@ -81,8 +82,8 @@ func TestIsEncryptedPEMBlock(t *testing.T) {
 	assert.True(t, IsEncryptedPEMBlock(encryptedPKCS1Block))
 
 	// Enable FIPS mode
-	os.Setenv(FIPSEnvVar, "1")
-	defer os.Unsetenv(FIPSEnvVar)
+	os.Setenv(fips.EnvVar, "1")
+	defer os.Unsetenv(fips.EnvVar)
 
 	// Check PKCS8 keys again
 	assert.False(t, IsEncryptedPEMBlock(decryptedPKCS8Block))
@@ -95,7 +96,7 @@ func TestIsEncryptedPEMBlock(t *testing.T) {
 
 func TestDecryptPEMBlock(t *testing.T) {
 	// Disable FIPS mode
-	os.Unsetenv(FIPSEnvVar)
+	os.Unsetenv(fips.EnvVar)
 
 	// Check PKCS8 keys
 	_, err := DecryptPEMBlock(encryptedPKCS8Block, []byte("pony"))
@@ -114,8 +115,8 @@ func TestDecryptPEMBlock(t *testing.T) {
 	require.Equal(t, decryptedPKCS1Block.Bytes, decryptedDer)
 
 	// Enable FIPS mode
-	os.Setenv(FIPSEnvVar, "1")
-	defer os.Unsetenv(FIPSEnvVar)
+	os.Setenv(fips.EnvVar, "1")
+	defer os.Unsetenv(fips.EnvVar)
 
 	// Try to decrypt PKCS1
 	_, err = DecryptPEMBlock(encryptedPKCS1Block, []byte("ponies"))
@@ -124,7 +125,7 @@ func TestDecryptPEMBlock(t *testing.T) {
 
 func TestEncryptPEMBlock(t *testing.T) {
 	// Disable FIPS mode
-	os.Unsetenv(FIPSEnvVar)
+	os.Unsetenv(fips.EnvVar)
 
 	// Check PKCS8 keys
 	encryptedBlock, err := EncryptPEMBlock(decryptedPKCS8Block.Bytes, []byte("knock knock"))
@@ -151,8 +152,8 @@ func TestEncryptPEMBlock(t *testing.T) {
 	require.Equal(t, decryptedPKCS1Block.Bytes, decryptedDer)
 
 	// Enable FIPS mode
-	os.Setenv(FIPSEnvVar, "1")
-	defer os.Unsetenv(FIPSEnvVar)
+	os.Setenv(fips.EnvVar, "1")
+	defer os.Unsetenv(fips.EnvVar)
 
 	// Try to encrypt PKCS1
 	_, err = EncryptPEMBlock(decryptedPKCS1Block.Bytes, []byte("knock knock"))
@@ -161,7 +162,7 @@ func TestEncryptPEMBlock(t *testing.T) {
 
 func TestParsePrivateKeyPEMWithPassword(t *testing.T) {
 	// Disable FIPS mode
-	os.Unsetenv(FIPSEnvVar)
+	os.Unsetenv(fips.EnvVar)
 
 	// Check PKCS8 keys
 	_, err := ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS8), []byte("pony"))
@@ -184,8 +185,8 @@ func TestParsePrivateKeyPEMWithPassword(t *testing.T) {
 	require.NoError(t, err)
 
 	// Enable FIPS mode
-	os.Setenv(FIPSEnvVar, "1")
-	defer os.Unsetenv(FIPSEnvVar)
+	os.Setenv(fips.EnvVar, "1")
+	defer os.Unsetenv(fips.EnvVar)
 
 	// Try to parse PKCS1
 	_, err = ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS1), []byte("ponies"))

cmd/swarm-rafttool/common.go

@@ -78,7 +78,7 @@ func decryptRaftData(swarmdir, outdir, unlockKey string) error {
 	_, d := encryption.Defaults(deks.CurrentDEK)
 	if deks.PendingDEK == nil {
 		_, d2 := encryption.Defaults(deks.PendingDEK)
-		d = storage.MultiDecrypter{d, d2}
+		d = encryption.NewMultiDecrypter(d, d2)
 	}
 
 	snapDir := filepath.Join(outdir, "snap-decrypted")

cmd/swarm-rafttool/dump.go

@@ -41,7 +41,7 @@ func loadData(swarmdir, unlockKey string) (*storage.WALData, *raftpb.Snapshot, e
 		_, d := encryption.Defaults(deks.CurrentDEK)
 		if deks.PendingDEK == nil {
 			_, d2 := encryption.Defaults(deks.PendingDEK)
-			d = storage.MultiDecrypter{d, d2}
+			d = encryption.NewMultiDecrypter(d, d2)
 		}
 
 		walFactory = storage.NewWALFactory(encryption.NoopCrypter, d)

fips/fips.go

@@ -0,0 +1,11 @@
+package fips
+
+import "os"
+
+// EnvVar is the environment variable which stores FIPS mode state
+const EnvVar = "GOFIPS"
+
+// Enabled returns true when FIPS mode is enabled
+func Enabled() bool {
+	return os.Getenv(EnvVar) != ""
+}

integration/integration_test.go

@@ -20,8 +20,8 @@ import (
 	events "github.com/docker/go-events"
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/ca"
-	"github.com/docker/swarmkit/ca/keyutils"
 	cautils "github.com/docker/swarmkit/ca/testutils"
+	"github.com/docker/swarmkit/fips"
 	"github.com/docker/swarmkit/identity"
 	"github.com/docker/swarmkit/manager"
 	"github.com/docker/swarmkit/testutils"
@@ -269,12 +269,12 @@ func TestAutolockManagers(t *testing.T) {
 	t.Parallel()
 
 	// run this twice, once with root ca with pkcs1 key and then pkcs8 key
-	defer os.Unsetenv(keyutils.FIPSEnvVar)
+	defer os.Unsetenv(fips.EnvVar)
 	for _, pkcs1 := range []bool{true, false} {
 		if pkcs1 {
-			os.Unsetenv(keyutils.FIPSEnvVar)
+			os.Unsetenv(fips.EnvVar)
 		} else {
-			os.Setenv(keyutils.FIPSEnvVar, "1")
+			os.Setenv(fips.EnvVar, "1")
 		}
 
 		rootCA, err := ca.CreateRootCA("rootCN")
@@ -622,12 +622,12 @@ func TestSuccessfulRootRotation(t *testing.T) {
 	t.Parallel()
 
 	// run this twice, once with root ca with pkcs1 key and then pkcs8 key
-	defer os.Unsetenv(keyutils.FIPSEnvVar)
+	defer os.Unsetenv(fips.EnvVar)
 	for _, pkcs1 := range []bool{true, false} {
 		if pkcs1 {
-			os.Unsetenv(keyutils.FIPSEnvVar)
+			os.Unsetenv(fips.EnvVar)
 		} else {
-			os.Setenv(keyutils.FIPSEnvVar, "1")
+			os.Setenv(fips.EnvVar, "1")
 		}
 
 		rootCA, err := ca.CreateRootCA("rootCN")

manager/encryption/encryption.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/docker/swarmkit/api"
+	"github.com/docker/swarmkit/fips"
 	"github.com/gogo/protobuf/proto"
 	"github.com/pkg/errors"
 )
@@ -59,6 +60,60 @@ func (n noopCrypter) Algorithm() api.MaybeEncryptedRecord_Algorithm {
 // decrypt any data
 var NoopCrypter = noopCrypter{}
 
+// specificDecryptor represents a specific type of Decrypter, like NaclSecretbox or Fernet.
+// It does not apply to a more general decrypter like MultiDecrypter.
+type specificDecrypter interface {
+	Decrypter
+	Algorithm() api.MaybeEncryptedRecord_Algorithm
+}
+
+// MultiDecrypter is a decrypter that will attempt to decrypt with multiple decrypters.  It
+// references them by algorithm, so that only the relevant decrypters are checked instead of
+// every single one. The reason for multiple decrypters per algorithm is to support hitless
+// encryption key rotation.
+//
+// For raft encryption for instance, during an encryption key rotation, it's possible to have
+// some raft logs encrypted with the old key and some encrypted with the new key, so we need a
+// decrypter that can decrypt both.
+type MultiDecrypter struct {
+	decrypters map[api.MaybeEncryptedRecord_Algorithm][]Decrypter
+}
+
+// Decrypt tries to decrypt using any decrypters that match the given algorithm.
+func (m MultiDecrypter) Decrypt(r api.MaybeEncryptedRecord) (result []byte, err error) {
+	decrypters, ok := m.decrypters[r.Algorithm]
+	if !ok {
+		return nil, fmt.Errorf("cannot decrypt record encrypted using %s",
+			api.MaybeEncryptedRecord_Algorithm_name[int32(r.Algorithm)])
+	}
+	for _, d := range decrypters {
+		result, err = d.Decrypt(r)
+		if err == nil {
+			return
+		}
+	}
+	return
+}
+
+// NewMultiDecrypter returns a new MultiDecrypter given multiple Decrypters.  If any of
+// the Decrypters are also MultiDecrypters, they are flattened into a single map, but
+// it does not deduplicate any decrypters.
+// Note that if something is neither a MultiDecrypter nor a specificDecrypter, it is
+// ignored.
+func NewMultiDecrypter(decrypters ...Decrypter) MultiDecrypter {
+	m := MultiDecrypter{decrypters: make(map[api.MaybeEncryptedRecord_Algorithm][]Decrypter)}
+	for _, d := range decrypters {
+		if md, ok := d.(MultiDecrypter); ok {
+			for algo, dec := range md.decrypters {
+				m.decrypters[algo] = append(m.decrypters[algo], dec...)
+			}
+		} else if sd, ok := d.(specificDecrypter); ok {
+			m.decrypters[sd.Algorithm()] = append(m.decrypters[sd.Algorithm()], sd)
+		}
+	}
+	return m
+}
+
 // Decrypt turns a slice of bytes serialized as an MaybeEncryptedRecord into a slice of plaintext bytes
 func Decrypt(encryptd []byte, decrypter Decrypter) ([]byte, error) {
 	if decrypter == nil {
@@ -97,8 +152,12 @@ func Encrypt(plaintext []byte, encrypter Encrypter) ([]byte, error) {
 
 // Defaults returns a default encrypter and decrypter
 func Defaults(key []byte) (Encrypter, Decrypter) {
+	f := NewFernet(key)
+	if fips.Enabled() {
+		return f, f
+	}
 	n := NewNACLSecretbox(key)
-	return n, n
+	return n, NewMultiDecrypter(n, f)
 }
 
 // GenerateSecretKey generates a secret key that can be used for encrypting data