From 2a318675f83c5d524c34dbb99c1970fd53934035 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Wed, 14 Mar 2018 17:07:42 -0700 Subject: [PATCH 1/4] Rather than use an environment variable to determine whether something requires FIPS: (1) require that users of the keyutil package instead use a key formatter object, which could either be the default non-FIPS utility or the FIPS utility. (2) require that users that request encryption defaults specify whether FIPS compliance is needed Signed-off-by: Ying Li --- ca/keyutils/keyutils.go | 51 +++++--- ca/keyutils/keyutils_test.go | 143 ++++++++------------- cmd/swarm-rafttool/common.go | 6 +- cmd/swarm-rafttool/common_test.go | 2 +- cmd/swarm-rafttool/dump.go | 6 +- fips/fips.go | 11 -- manager/deks.go | 4 +- manager/encryption/encryption.go | 8 +- manager/encryption/encryption_test.go | 29 +---- manager/manager_test.go | 2 +- manager/state/raft/storage/storage.go | 11 +- manager/state/raft/storage/storage_test.go | 2 +- 12 files changed, 116 insertions(+), 159 deletions(-) delete mode 100644 fips/fips.go diff --git a/ca/keyutils/keyutils.go b/ca/keyutils/keyutils.go index 03874428c0..ea45aab7dd 100644 --- a/ca/keyutils/keyutils.go +++ b/ca/keyutils/keyutils.go @@ -13,10 +13,28 @@ import ( "github.com/cloudflare/cfssl/helpers" "github.com/docker/swarmkit/ca/pkcs8" - "github.com/docker/swarmkit/fips" ) -var errFIPSUnsupportedKeyFormat = errors.New("unsupported key format due to FIPS compliance") +// Formatter provides an interface for converting keys to the right format, and encrypting and decrypting keys +type Formatter interface { + ParsePrivateKeyPEMWithPassword(pemBytes, password []byte) (crypto.Signer, error) + DecryptPEMBlock(block *pem.Block, password []byte) ([]byte, error) + EncryptPEMBlock(data, password []byte) (*pem.Block, error) +} + +// ErrFIPSUnsupportedKeyFormat is returned when encryption/decryption operations are attempted on a PKCS1 key +// when FIPS mode is enabled. +var ErrFIPSUnsupportedKeyFormat = errors.New("unsupported key format due to FIPS compliance") + +// Default is the default key util, where FIPS is not required +var Default Formatter = &utils{fips: false} + +// FIPS is the key utility which enforces FIPS compliance +var FIPS Formatter = &utils{fips: true} + +type utils struct { + fips bool +} // IsPKCS8 returns true if the provided der bytes is encrypted/unencrypted PKCS#8 key func IsPKCS8(derBytes []byte) bool { @@ -31,9 +49,14 @@ func IsPKCS8(derBytes []byte) bool { }) } +// IsEncryptedPEMBlock checks if a PKCS#1 or PKCS#8 PEM-block is encrypted or not +func IsEncryptedPEMBlock(block *pem.Block) bool { + return pkcs8.IsEncryptedPEMBlock(block) || x509.IsEncryptedPEMBlock(block) +} + // ParsePrivateKeyPEMWithPassword parses an encrypted or a decrypted PKCS#1 or PKCS#8 PEM to crypto.Signer. // It returns an error in FIPS mode if PKCS#1 PEM bytes are passed. -func ParsePrivateKeyPEMWithPassword(pemBytes, password []byte) (crypto.Signer, error) { +func (u *utils) ParsePrivateKeyPEMWithPassword(pemBytes, password []byte) (crypto.Signer, error) { block, _ := pem.Decode(pemBytes) if block == nil { return nil, errors.New("Could not parse PEM") @@ -41,26 +64,20 @@ func ParsePrivateKeyPEMWithPassword(pemBytes, password []byte) (crypto.Signer, e if IsPKCS8(block.Bytes) { return pkcs8.ParsePrivateKeyPEMWithPassword(pemBytes, password) - } else if fips.Enabled() { - return nil, errFIPSUnsupportedKeyFormat + } else if u.fips { + return nil, ErrFIPSUnsupportedKeyFormat } return helpers.ParsePrivateKeyPEMWithPassword(pemBytes, password) } -// IsEncryptedPEMBlock checks if a PKCS#1 or PKCS#8 PEM-block is encrypted or not -// It returns false in FIPS mode even if PKCS#1 is encrypted -func IsEncryptedPEMBlock(block *pem.Block) bool { - return pkcs8.IsEncryptedPEMBlock(block) || (!fips.Enabled() && x509.IsEncryptedPEMBlock(block)) -} - // DecryptPEMBlock requires PKCS#1 or PKCS#8 PEM Block and password to decrypt and return unencrypted der []byte // It returns an error in FIPS mode when PKCS#1 PEM Block is passed. -func DecryptPEMBlock(block *pem.Block, password []byte) ([]byte, error) { +func (u *utils) DecryptPEMBlock(block *pem.Block, password []byte) ([]byte, error) { if IsPKCS8(block.Bytes) { return pkcs8.DecryptPEMBlock(block, password) - } else if fips.Enabled() { - return nil, errFIPSUnsupportedKeyFormat + } else if u.fips { + return nil, ErrFIPSUnsupportedKeyFormat } return x509.DecryptPEMBlock(block, password) @@ -68,11 +85,11 @@ func DecryptPEMBlock(block *pem.Block, password []byte) ([]byte, error) { // EncryptPEMBlock takes DER-format bytes and password to return an encrypted PKCS#1 or PKCS#8 PEM-block // It returns an error in FIPS mode when PKCS#1 PEM bytes are passed. -func EncryptPEMBlock(data, password []byte) (*pem.Block, error) { +func (u *utils) EncryptPEMBlock(data, password []byte) (*pem.Block, error) { if IsPKCS8(data) { return pkcs8.EncryptPEMBlock(data, password) - } else if fips.Enabled() { - return nil, errFIPSUnsupportedKeyFormat + } else if u.fips { + return nil, ErrFIPSUnsupportedKeyFormat } cipherType := x509.PEMCipherAES256 diff --git a/ca/keyutils/keyutils_test.go b/ca/keyutils/keyutils_test.go index 1e01551a3e..d0b0d455a7 100644 --- a/ca/keyutils/keyutils_test.go +++ b/ca/keyutils/keyutils_test.go @@ -2,10 +2,8 @@ package keyutils import ( "encoding/pem" - "os" "testing" - "github.com/docker/swarmkit/fips" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -50,15 +48,6 @@ aMbljbOLAjpZS3/VnQteab4= encryptedPKCS1Block, _ = pem.Decode([]byte(encryptedPKCS1)) ) -func TestFIPSEnabled(t *testing.T) { - os.Unsetenv(fips.EnvVar) - assert.False(t, fips.Enabled()) - - os.Setenv(fips.EnvVar, "1") - defer os.Unsetenv(fips.EnvVar) - assert.True(t, fips.Enabled()) -} - func TestIsPKCS8(t *testing.T) { // Check PKCS8 keys assert.True(t, IsPKCS8([]byte(decryptedPKCS8Block.Bytes))) @@ -70,125 +59,95 @@ func TestIsPKCS8(t *testing.T) { } func TestIsEncryptedPEMBlock(t *testing.T) { - // Disable FIPS mode - os.Unsetenv(fips.EnvVar) - - // Check PKCS8 keys + // Check PKCS8 assert.False(t, IsEncryptedPEMBlock(decryptedPKCS8Block)) assert.True(t, IsEncryptedPEMBlock(encryptedPKCS8Block)) - // Check PKCS1 keys + // Check PKCS1 assert.False(t, IsEncryptedPEMBlock(decryptedPKCS1Block)) assert.True(t, IsEncryptedPEMBlock(encryptedPKCS1Block)) - - // Enable FIPS mode - os.Setenv(fips.EnvVar, "1") - defer os.Unsetenv(fips.EnvVar) - - // Check PKCS8 keys again - assert.False(t, IsEncryptedPEMBlock(decryptedPKCS8Block)) - assert.True(t, IsEncryptedPEMBlock(encryptedPKCS8Block)) - - // Check PKCS1 keys again - assert.False(t, IsEncryptedPEMBlock(decryptedPKCS1Block)) - assert.False(t, IsEncryptedPEMBlock(encryptedPKCS1Block)) } func TestDecryptPEMBlock(t *testing.T) { - // Disable FIPS mode - os.Unsetenv(fips.EnvVar) - - // Check PKCS8 keys - _, err := DecryptPEMBlock(encryptedPKCS8Block, []byte("pony")) - require.Error(t, err) - - decryptedDer, err := DecryptPEMBlock(encryptedPKCS8Block, []byte("ponies")) - require.NoError(t, err) - require.Equal(t, decryptedPKCS8Block.Bytes, decryptedDer) - - // Check PKCS1 keys - _, err = DecryptPEMBlock(encryptedPKCS1Block, []byte("pony")) + // Check PKCS8 keys in both FIPS and non-FIPS mode + for _, util := range []Formatter{Default, FIPS} { + _, err := util.DecryptPEMBlock(encryptedPKCS8Block, []byte("pony")) + require.Error(t, err) + + decryptedDer, err := util.DecryptPEMBlock(encryptedPKCS8Block, []byte("ponies")) + require.NoError(t, err) + require.Equal(t, decryptedPKCS8Block.Bytes, decryptedDer) + } + + // Check PKCS1 keys in non-FIPS mode + _, err := Default.DecryptPEMBlock(encryptedPKCS1Block, []byte("pony")) require.Error(t, err) - decryptedDer, err = DecryptPEMBlock(encryptedPKCS1Block, []byte("ponies")) + decryptedDer, err := Default.DecryptPEMBlock(encryptedPKCS1Block, []byte("ponies")) require.NoError(t, err) require.Equal(t, decryptedPKCS1Block.Bytes, decryptedDer) - // Enable FIPS mode - os.Setenv(fips.EnvVar, "1") - defer os.Unsetenv(fips.EnvVar) - - // Try to decrypt PKCS1 - _, err = DecryptPEMBlock(encryptedPKCS1Block, []byte("ponies")) + // Try to decrypt PKCS1 in FIPS + _, err = FIPS.DecryptPEMBlock(encryptedPKCS1Block, []byte("ponies")) require.Error(t, err) } func TestEncryptPEMBlock(t *testing.T) { - // Disable FIPS mode - os.Unsetenv(fips.EnvVar) - - // Check PKCS8 keys - encryptedBlock, err := EncryptPEMBlock(decryptedPKCS8Block.Bytes, []byte("knock knock")) + // Check PKCS8 keys in both FIPS and non-FIPS mode + for _, util := range []Formatter{Default, FIPS} { + encryptedBlock, err := util.EncryptPEMBlock(decryptedPKCS8Block.Bytes, []byte("knock knock")) + require.NoError(t, err) + + // Try to decrypt the same encrypted block + _, err = util.DecryptPEMBlock(encryptedBlock, []byte("hey there")) + require.Error(t, err) + + decryptedDer, err := Default.DecryptPEMBlock(encryptedBlock, []byte("knock knock")) + require.NoError(t, err) + require.Equal(t, decryptedPKCS8Block.Bytes, decryptedDer) + } + + // Check PKCS1 keys in non FIPS mode + encryptedBlock, err := Default.EncryptPEMBlock(decryptedPKCS1Block.Bytes, []byte("knock knock")) require.NoError(t, err) // Try to decrypt the same encrypted block - _, err = DecryptPEMBlock(encryptedBlock, []byte("hey there")) + _, err = Default.DecryptPEMBlock(encryptedBlock, []byte("hey there")) require.Error(t, err) - decryptedDer, err := DecryptPEMBlock(encryptedBlock, []byte("knock knock")) - require.NoError(t, err) - require.Equal(t, decryptedPKCS8Block.Bytes, decryptedDer) - - // Check PKCS1 keys - encryptedBlock, err = EncryptPEMBlock(decryptedPKCS1Block.Bytes, []byte("knock knock")) - require.NoError(t, err) - - // Try to decrypt the same encrypted block - _, err = DecryptPEMBlock(encryptedBlock, []byte("hey there")) - require.Error(t, err) - - decryptedDer, err = DecryptPEMBlock(encryptedBlock, []byte("knock knock")) + decryptedDer, err := Default.DecryptPEMBlock(encryptedBlock, []byte("knock knock")) require.NoError(t, err) require.Equal(t, decryptedPKCS1Block.Bytes, decryptedDer) - // Enable FIPS mode - os.Setenv(fips.EnvVar, "1") - defer os.Unsetenv(fips.EnvVar) - // Try to encrypt PKCS1 - _, err = EncryptPEMBlock(decryptedPKCS1Block.Bytes, []byte("knock knock")) + _, err = FIPS.EncryptPEMBlock(decryptedPKCS1Block.Bytes, []byte("knock knock")) require.Error(t, err) } func TestParsePrivateKeyPEMWithPassword(t *testing.T) { - // Disable FIPS mode - os.Unsetenv(fips.EnvVar) + // Check PKCS8 keys in both FIPS and non-FIPS mode + for _, util := range []Formatter{Default, FIPS} { + _, err := util.ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS8), []byte("pony")) + require.Error(t, err) - // Check PKCS8 keys - _, err := ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS8), []byte("pony")) - require.Error(t, err) + _, err = util.ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS8), []byte("ponies")) + require.NoError(t, err) - _, err = ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS8), []byte("ponies")) - require.NoError(t, err) + _, err = util.ParsePrivateKeyPEMWithPassword([]byte(decryptedPKCS8), nil) + require.NoError(t, err) + } - _, err = ParsePrivateKeyPEMWithPassword([]byte(decryptedPKCS8), nil) - require.NoError(t, err) - - // Check PKCS1 keys - _, err = ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS1), []byte("pony")) + // Check PKCS1 keys in non-FIPS mode + _, err := Default.ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS1), []byte("pony")) require.Error(t, err) - _, err = ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS1), []byte("ponies")) + _, err = Default.ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS1), []byte("ponies")) require.NoError(t, err) - _, err = ParsePrivateKeyPEMWithPassword([]byte(decryptedPKCS1), nil) + _, err = Default.ParsePrivateKeyPEMWithPassword([]byte(decryptedPKCS1), nil) require.NoError(t, err) - // Enable FIPS mode - os.Setenv(fips.EnvVar, "1") - defer os.Unsetenv(fips.EnvVar) - - // Try to parse PKCS1 - _, err = ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS1), []byte("ponies")) + // Try to parse PKCS1 in FIPS mode + _, err = FIPS.ParsePrivateKeyPEMWithPassword([]byte(encryptedPKCS1), []byte("ponies")) require.Error(t, err) } diff --git a/cmd/swarm-rafttool/common.go b/cmd/swarm-rafttool/common.go index 532696d47b..a169b9af6e 100644 --- a/cmd/swarm-rafttool/common.go +++ b/cmd/swarm-rafttool/common.go @@ -75,9 +75,11 @@ func decryptRaftData(swarmdir, outdir, unlockKey string) error { return err } - _, d := encryption.Defaults(deks.CurrentDEK) + // always use false for FIPS, since we want to be able to decrypt logs written using + // any algorithm (not just FIPS-compatible ones) + _, d := encryption.Defaults(deks.CurrentDEK, false) if deks.PendingDEK == nil { - _, d2 := encryption.Defaults(deks.PendingDEK) + _, d2 := encryption.Defaults(deks.PendingDEK, false) d = encryption.NewMultiDecrypter(d, d2) } diff --git a/cmd/swarm-rafttool/common_test.go b/cmd/swarm-rafttool/common_test.go index ada57263fd..606110ad20 100644 --- a/cmd/swarm-rafttool/common_test.go +++ b/cmd/swarm-rafttool/common_test.go @@ -74,7 +74,7 @@ func TestDecrypt(t *testing.T) { Term: 1, }, } - e, d := encryption.Defaults(dek) + e, d := encryption.Defaults(dek, false) writeFakeRaftData(t, tempdir, &origSnapshot, storage.NewWALFactory(e, d), storage.NewSnapFactory(e, d)) outdir := filepath.Join(tempdir, "outdir") diff --git a/cmd/swarm-rafttool/dump.go b/cmd/swarm-rafttool/dump.go index e4e1196ecf..6360194d67 100644 --- a/cmd/swarm-rafttool/dump.go +++ b/cmd/swarm-rafttool/dump.go @@ -38,9 +38,11 @@ func loadData(swarmdir, unlockKey string) (*storage.WALData, *raftpb.Snapshot, e return nil, nil, err } - _, d := encryption.Defaults(deks.CurrentDEK) + // always set FIPS=false, because we want to decrypt logs stored using any + // algorithm, not just FIPS-compatible ones + _, d := encryption.Defaults(deks.CurrentDEK, false) if deks.PendingDEK == nil { - _, d2 := encryption.Defaults(deks.PendingDEK) + _, d2 := encryption.Defaults(deks.PendingDEK, false) d = encryption.NewMultiDecrypter(d, d2) } diff --git a/fips/fips.go b/fips/fips.go deleted file mode 100644 index 9fde7772ee..0000000000 --- a/fips/fips.go +++ /dev/null @@ -1,11 +0,0 @@ -package fips - -import "os" - -// EnvVar is the environment variable which stores FIPS mode state -const EnvVar = "GOFIPS" - -// Enabled returns true when FIPS mode is enabled -func Enabled() bool { - return os.Getenv(EnvVar) != "" -} diff --git a/manager/deks.go b/manager/deks.go index 4813a67d53..edb5227904 100644 --- a/manager/deks.go +++ b/manager/deks.go @@ -243,7 +243,7 @@ func (r *RaftDEKManager) MaybeUpdateKEK(candidateKEK ca.KEKData) (bool, bool, er func decodePEMHeaderValue(headerValue string, kek []byte) ([]byte, error) { var decrypter encryption.Decrypter = encryption.NoopCrypter if kek != nil { - _, decrypter = encryption.Defaults(kek) + _, decrypter = encryption.Defaults(kek, false) } valueBytes, err := base64.StdEncoding.DecodeString(headerValue) if err != nil { @@ -259,7 +259,7 @@ func decodePEMHeaderValue(headerValue string, kek []byte) ([]byte, error) { func encodePEMHeaderValue(headerValue []byte, kek []byte) (string, error) { var encrypter encryption.Encrypter = encryption.NoopCrypter if kek != nil { - encrypter, _ = encryption.Defaults(kek) + encrypter, _ = encryption.Defaults(kek, false) } encrypted, err := encryption.Encrypt(headerValue, encrypter) if err != nil { diff --git a/manager/encryption/encryption.go b/manager/encryption/encryption.go index 5b20f1ec8d..d9aad6ad84 100644 --- a/manager/encryption/encryption.go +++ b/manager/encryption/encryption.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/docker/swarmkit/api" - "github.com/docker/swarmkit/fips" "github.com/gogo/protobuf/proto" "github.com/pkg/errors" ) @@ -150,10 +149,11 @@ func Encrypt(plaintext []byte, encrypter Encrypter) ([]byte, error) { return data, nil } -// Defaults returns a default encrypter and decrypter -func Defaults(key []byte) (Encrypter, Decrypter) { +// Defaults returns a default encrypter and decrypter. If the FIPS parameter is set to +// true, the only algorithm supported on both the encrypter and decrypter will be fernet. +func Defaults(key []byte, fips bool) (Encrypter, Decrypter) { f := NewFernet(key) - if fips.Enabled() { + if fips { return f, f } n := NewNACLSecretbox(key) diff --git a/manager/encryption/encryption_test.go b/manager/encryption/encryption_test.go index c974445e3a..2556ef8abe 100644 --- a/manager/encryption/encryption_test.go +++ b/manager/encryption/encryption_test.go @@ -2,11 +2,8 @@ package encryption import ( "fmt" - "os" "testing" - "github.com/docker/swarmkit/fips" - "github.com/stretchr/testify/require" ) @@ -32,7 +29,7 @@ func TestEncryptDecrypt(t *testing.T) { require.Equal(t, msg, decrypted) // the default encrypter can produce something the default decrypter can read - encrypter, decrypter := Defaults([]byte("key")) + encrypter, decrypter := Defaults([]byte("key"), false) encrypted, err = Encrypt(msg, encrypter) require.NoError(t, err) decrypted, err = Decrypt(encrypted, decrypter) @@ -117,23 +114,19 @@ func TestMultiDecryptor(t *testing.T) { // enabled, the encrypter/decrypter is Fernet only, because FIPS only permits // (given the algorithms swarmkit supports) AES-128-CBC func TestDefaults(t *testing.T) { - oldFipsVar := os.Getenv(fips.EnvVar) - plaintext := []byte("my message") - // ensure the fips var is not set - require.NoError(t, os.Unsetenv(fips.EnvVar)) - c, d := Defaults([]byte("key")) + // encrypt something without FIPS enabled + c, d := Defaults([]byte("key"), false) ciphertext, err := Encrypt(plaintext, c) require.NoError(t, err) decrypted, err := Decrypt(ciphertext, d) require.NoError(t, err) require.Equal(t, plaintext, decrypted) - // ensure that the fips var is set - defaults should return a fernet encrypter + // with fips enabled, defaults should return a fernet encrypter // and a decrypter that can't decrypt nacl - require.NoError(t, os.Setenv(fips.EnvVar, "true")) - c, d = Defaults([]byte("key")) + c, d = Defaults([]byte("key"), true) _, err = Decrypt(ciphertext, d) require.Error(t, err) ciphertext, err = Encrypt(plaintext, c) @@ -142,18 +135,10 @@ func TestDefaults(t *testing.T) { require.NoError(t, err) require.Equal(t, plaintext, decrypted) - // unset the fips var again, and ensure we can decrypt the previous ciphertext + // without FIPS, and ensure we can decrypt the previous ciphertext // (encrypted with fernet) with the decrypter returned by defaults - require.NoError(t, os.Unsetenv(fips.EnvVar)) - _, d = Defaults([]byte("key")) + _, d = Defaults([]byte("key"), false) decrypted, err = Decrypt(ciphertext, d) require.NoError(t, err) require.Equal(t, plaintext, decrypted) - - // put the env var back - if oldFipsVar == "" { - require.NoError(t, os.Unsetenv(fips.EnvVar)) - } else { - require.NoError(t, os.Setenv(fips.EnvVar, oldFipsVar)) - } } diff --git a/manager/manager_test.go b/manager/manager_test.go index e718c95f1c..3e15605ed7 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -366,7 +366,7 @@ func TestManagerLockUnlock(t *testing.T) { require.False(t, ok) // verify that the snapshot is readable with the new DEK - encrypter, decrypter := encryption.Defaults(currentDEK) + encrypter, decrypter := encryption.Defaults(currentDEK, false) // we can't use the raftLogger, because the WALs are still locked while the raft node is up. And once we remove // the manager, they'll be deleted. snapshot, err := storage.NewSnapFactory(encrypter, decrypter).New(filepath.Join(stateDir, "raft", "snap-v3-encrypted")).Load() diff --git a/manager/state/raft/storage/storage.go b/manager/state/raft/storage/storage.go index 764e5dbc16..bbd262f37c 100644 --- a/manager/state/raft/storage/storage.go +++ b/manager/state/raft/storage/storage.go @@ -38,6 +38,9 @@ type EncryptedRaftLogger struct { StateDir string EncryptionKey []byte + // FIPS specifies whether the encryption should be FIPS-compliant + FIPS bool + // mutex is locked for writing only when we need to replace the wal object and snapshotter // object, not when we're writing snapshots or wals (in which case it's locked for reading) encoderMu sync.RWMutex @@ -53,11 +56,11 @@ func (e *EncryptedRaftLogger) BootstrapFromDisk(ctx context.Context, oldEncrypti walDir := e.walDir() snapDir := e.snapDir() - encrypter, decrypter := encryption.Defaults(e.EncryptionKey) + encrypter, decrypter := encryption.Defaults(e.EncryptionKey, e.FIPS) if oldEncryptionKeys != nil { decrypters := []encryption.Decrypter{decrypter} for _, key := range oldEncryptionKeys { - _, d := encryption.Defaults(key) + _, d := encryption.Defaults(key, e.FIPS) decrypters = append(decrypters, d) } decrypter = encryption.NewMultiDecrypter(decrypters...) @@ -141,7 +144,7 @@ func (e *EncryptedRaftLogger) BootstrapFromDisk(ctx context.Context, oldEncrypti func (e *EncryptedRaftLogger) BootstrapNew(metadata []byte) error { e.encoderMu.Lock() defer e.encoderMu.Unlock() - encrypter, decrypter := encryption.Defaults(e.EncryptionKey) + encrypter, decrypter := encryption.Defaults(e.EncryptionKey, e.FIPS) walFactory := NewWALFactory(encrypter, decrypter) for _, dirpath := range []string{filepath.Dir(e.walDir()), e.snapDir()} { @@ -184,7 +187,7 @@ func (e *EncryptedRaftLogger) RotateEncryptionKey(newKey []byte) { panic(fmt.Errorf("EncryptedRaftLogger's WAL is not a wrappedWAL")) } - wrapped.encrypter, wrapped.decrypter = encryption.Defaults(newKey) + wrapped.encrypter, wrapped.decrypter = encryption.Defaults(newKey, e.FIPS) e.snapshotter = NewSnapFactory(wrapped.encrypter, wrapped.decrypter).New(e.snapDir()) } diff --git a/manager/state/raft/storage/storage_test.go b/manager/state/raft/storage/storage_test.go index 89db461578..f192eb48c2 100644 --- a/manager/state/raft/storage/storage_test.go +++ b/manager/state/raft/storage/storage_test.go @@ -184,7 +184,7 @@ func TestMigrateToV3EncryptedForm(t *testing.T) { v3EncryptedSnapshot.Metadata.Index += 200 v3EncryptedSnapshot.Metadata.Term += 20 - encoder, decoders := encryption.Defaults(dek) + encoder, decoders := encryption.Defaults(dek, false) walFactory := NewWALFactory(encoder, decoders) snapFactory := NewSnapFactory(encoder, decoders) From 971930e9841e655cde3ed1393bd7e40b3d31c570 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Thu, 15 Mar 2018 15:55:58 -0700 Subject: [PATCH 2/4] Use the default keyutil formatter for tests, and remove fips checks for the root CA because we no longer support encrypting the root CA key, and PKCS8 vs PKCS1 only matters for fips if we encrypt. We want to keep the root key PKCS1 so that mixed version clusters will continue to work. Signed-off-by: Ying Li --- ca/certificates.go | 12 +----------- ca/certificates_test.go | 26 -------------------------- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/ca/certificates.go b/ca/certificates.go index 6bad7f9a9c..f2d3dbac55 100644 --- a/ca/certificates.go +++ b/ca/certificates.go @@ -26,10 +26,8 @@ import ( "github.com/cloudflare/cfssl/signer/local" "github.com/docker/go-events" "github.com/docker/swarmkit/api" - "github.com/docker/swarmkit/ca/keyutils" "github.com/docker/swarmkit/ca/pkcs8" "github.com/docker/swarmkit/connectionbroker" - "github.com/docker/swarmkit/fips" "github.com/docker/swarmkit/ioutils" "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -636,7 +634,7 @@ func newLocalSigner(keyBytes, certBytes []byte, certExpiry time.Duration, rootPo } // The key should not be encrypted, but it could be in PKCS8 format rather than PKCS1 - priv, err := keyutils.ParsePrivateKeyPEMWithPassword(keyBytes, nil) + priv, err := helpers.ParsePrivateKeyPEM(keyBytes) if err != nil { return nil, errors.Wrap(err, "malformed private key") } @@ -782,14 +780,6 @@ func CreateRootCA(rootCN string) (RootCA, error) { return RootCA{}, err } - // Convert key to PKCS#8 in FIPS mode - if fips.Enabled() { - key, err = pkcs8.ConvertECPrivateKeyPEM(key) - if err != nil { - return RootCA{}, err - } - } - rootCA, err := NewRootCA(cert, cert, key, DefaultNodeCertExpiration, nil) if err != nil { return RootCA{}, err diff --git a/ca/certificates_test.go b/ca/certificates_test.go index 205534a6ef..2c7895510c 100644 --- a/ca/certificates_test.go +++ b/ca/certificates_test.go @@ -29,7 +29,6 @@ import ( "github.com/docker/swarmkit/ca" cautils "github.com/docker/swarmkit/ca/testutils" "github.com/docker/swarmkit/connectionbroker" - "github.com/docker/swarmkit/fips" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/store" @@ -78,31 +77,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestCreateRootCAKeyFormat(t *testing.T) { - // Check if the CA key generated is PKCS#1 when FIPS-mode is off - rootCA, err := ca.CreateRootCA("rootCA") - require.NoError(t, err) - - s, err := rootCA.Signer() - require.NoError(t, err) - block, _ := pem.Decode(s.Key) - require.NotNil(t, block) - require.Equal(t, "EC PRIVATE KEY", block.Type) - - // Check if the CA key generated is PKCS#8 when FIPS-mode is on - os.Setenv(fips.EnvVar, "1") - defer os.Unsetenv(fips.EnvVar) - - rootCA, err = ca.CreateRootCA("rootCA") - require.NoError(t, err) - - s, err = rootCA.Signer() - require.NoError(t, err) - block, _ = pem.Decode(s.Key) - require.NotNil(t, block) - require.Equal(t, "PRIVATE KEY", block.Type) -} - func TestCreateRootCASaveRootCA(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) From 4ffb0ec22fdce25242ac835bf37b4c8d72523d15 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Thu, 15 Mar 2018 15:56:15 -0700 Subject: [PATCH 3/4] Add a key formatter parameter to the NewKeyReadWriter, and use that to encrypt and decrypt keys. It can be set using a setter function. Signed-off-by: Ying Li --- ca/keyreadwriter.go | 33 ++++++++++++++++++---------- ca/keyreadwriter_test.go | 46 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/ca/keyreadwriter.go b/ca/keyreadwriter.go index c929523f38..cf4517fff4 100644 --- a/ca/keyreadwriter.go +++ b/ca/keyreadwriter.go @@ -73,21 +73,30 @@ func (e ErrInvalidKEK) Error() string { // KeyReadWriter is an object that knows how to read and write TLS keys and certs to disk, // optionally encrypted and optionally updating PEM headers. type KeyReadWriter struct { - mu sync.Mutex - kekData KEKData - paths CertPaths - headersObj PEMKeyHeaders + mu sync.Mutex + kekData KEKData + paths CertPaths + headersObj PEMKeyHeaders + keyFormatter keyutils.Formatter } // NewKeyReadWriter creates a new KeyReadWriter func NewKeyReadWriter(paths CertPaths, kek []byte, headersObj PEMKeyHeaders) *KeyReadWriter { return &KeyReadWriter{ - kekData: KEKData{KEK: kek}, - paths: paths, - headersObj: headersObj, + kekData: KEKData{KEK: kek}, + paths: paths, + headersObj: headersObj, + keyFormatter: keyutils.Default, } } +// SetKeyFormatter sets the keyformatter with which to encrypt and decrypt keys +func (k *KeyReadWriter) SetKeyFormatter(kf keyutils.Formatter) { + k.mu.Lock() + defer k.mu.Unlock() + k.keyFormatter = kf +} + // Migrate checks to see if a temporary key file exists. Older versions of // swarmkit wrote temporary keys instead of temporary certificates, so // migrate that temporary key if it exists. We want to write temporary certificates, @@ -324,8 +333,10 @@ func (k *KeyReadWriter) readKey() (*pem.Block, error) { return nil, ErrInvalidKEK{Wrapped: x509.IncorrectPasswordError} } - derBytes, err := keyutils.DecryptPEMBlock(keyBlock, k.kekData.KEK) - if err != nil { + derBytes, err := k.keyFormatter.DecryptPEMBlock(keyBlock, k.kekData.KEK) + if err == keyutils.ErrFIPSUnsupportedKeyFormat { + return nil, err + } else if err != nil { return nil, ErrInvalidKEK{Wrapped: err} } @@ -349,7 +360,7 @@ func (k *KeyReadWriter) readKey() (*pem.Block, error) { // writing it to disk. If the kek is nil, writes it to disk unencrypted. func (k *KeyReadWriter) writeKey(keyBlock *pem.Block, kekData KEKData, pkh PEMKeyHeaders) error { if kekData.KEK != nil { - encryptedPEMBlock, err := keyutils.EncryptPEMBlock(keyBlock.Bytes, kekData.KEK) + encryptedPEMBlock, err := k.keyFormatter.EncryptPEMBlock(keyBlock.Bytes, kekData.KEK) if err != nil { return err } @@ -404,7 +415,7 @@ func (k *KeyReadWriter) DowngradeKey() error { } if k.kekData.KEK != nil { - newBlock, err = keyutils.EncryptPEMBlock(newBlock.Bytes, k.kekData.KEK) + newBlock, err = k.keyFormatter.EncryptPEMBlock(newBlock.Bytes, k.kekData.KEK) if err != nil { return err } diff --git a/ca/keyreadwriter_test.go b/ca/keyreadwriter_test.go index 4b2610a74b..b28ee2c244 100644 --- a/ca/keyreadwriter_test.go +++ b/ca/keyreadwriter_test.go @@ -436,7 +436,7 @@ func testKeyReadWriterDowngradeKeyCase(t *testing.T, tc downgradeTestCase) error require.NotNil(t, block) kek = []byte("kek") - block, err = keyutils.EncryptPEMBlock(block.Bytes, kek) + block, err = keyutils.Default.EncryptPEMBlock(block.Bytes, kek) require.NoError(t, err) key = pem.EncodeToMemory(block) @@ -517,3 +517,47 @@ func TestKeyReadWriterDowngradeKey(t *testing.T) { require.NoError(t, err) } } + +// In FIPS mode, when reading a PKCS1 encrypted key, a PKCS1 error is returned as opposed +// to any other type of invalid KEK error +func TestKeyReadWriterReadNonFIPS(t *testing.T) { + t.Parallel() + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + key, err = pkcs8.ConvertToECPrivateKeyPEM(key) + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created + + k := ca.NewKeyReadWriter(path.Node, nil, nil) + k.SetKeyFormatter(keyutils.FIPS) + + // can write an unencrypted PKCS1 key with no issues + require.NoError(t, k.Write(cert, key, nil)) + // can read the unencrypted key with no issues + readCert, readKey, err := k.Read() + require.NoError(t, err) + require.Equal(t, cert, readCert) + require.Equal(t, key, readKey) + + // cannot write an encrypted PKCS1 key + passphrase := []byte("passphrase") + require.Equal(t, keyutils.ErrFIPSUnsupportedKeyFormat, k.Write(cert, key, &ca.KEKData{KEK: passphrase})) + + k.SetKeyFormatter(keyutils.Default) + require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: passphrase})) + + // cannot read an encrypted PKCS1 key + k.SetKeyFormatter(keyutils.FIPS) + _, _, err = k.Read() + require.Equal(t, keyutils.ErrFIPSUnsupportedKeyFormat, err) + + k.SetKeyFormatter(keyutils.Default) + _, _, err = k.Read() + require.NoError(t, err) +} From 43f607a2502dcfbc8576eaa3d9c60ef057c43d55 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Tue, 27 Mar 2018 16:40:24 -0700 Subject: [PATCH 4/4] Add a FIPS bool to the node object, and propagate that boolean to the KeyReadWriter used in the node object. Signed-off-by: Ying Li --- integration/cluster.go | 10 +++--- integration/integration_test.go | 39 +++++++-------------- integration/node.go | 3 +- node/node.go | 6 ++++ node/node_test.go | 62 +++++++++++++++++++++++++++++++-- 5 files changed, 86 insertions(+), 34 deletions(-) diff --git a/integration/cluster.go b/integration/cluster.go index c08e7bbdaa..b36498b0f0 100644 --- a/integration/cluster.go +++ b/integration/cluster.go @@ -34,13 +34,14 @@ type testCluster struct { errs chan error wg sync.WaitGroup counter int + fips bool } var testnameKey struct{} // NewCluster creates new cluster to which nodes can be added. // AcceptancePolicy is set to most permissive mode on first manager node added. -func newTestCluster(testname string) *testCluster { +func newTestCluster(testname string, fips bool) *testCluster { ctx, cancel := context.WithCancel(context.Background()) ctx = context.WithValue(ctx, testnameKey, testname) c := &testCluster{ @@ -49,6 +50,7 @@ func newTestCluster(testname string) *testCluster { nodes: make(map[string]*testNode), nodesOrder: make(map[string]int), errs: make(chan error, 1024), + fips: fips, } c.api = &dummyAPI{c: c} return c @@ -92,7 +94,7 @@ func (c *testCluster) AddManager(lateBind bool, rootCA *ca.RootCA) error { // first node var n *testNode if len(c.nodes) == 0 { - node, err := newTestNode("", "", lateBind) + node, err := newTestNode("", "", lateBind, c.fips) if err != nil { return err } @@ -113,7 +115,7 @@ func (c *testCluster) AddManager(lateBind bool, rootCA *ca.RootCA) error { if err != nil { return err } - node, err := newTestNode(joinAddr, clusterInfo.RootCA.JoinTokens.Manager, false) + node, err := newTestNode(joinAddr, clusterInfo.RootCA.JoinTokens.Manager, false, c.fips) if err != nil { return err } @@ -169,7 +171,7 @@ func (c *testCluster) AddAgent() error { if err != nil { return err } - node, err := newTestNode(joinAddr, clusterInfo.RootCA.JoinTokens.Worker, false) + node, err := newTestNode(joinAddr, clusterInfo.RootCA.JoinTokens.Worker, false, c.fips) if err != nil { return err } diff --git a/integration/integration_test.go b/integration/integration_test.go index 8f1f2c29d5..81dc6f4000 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -21,7 +21,6 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" cautils "github.com/docker/swarmkit/ca/testutils" - "github.com/docker/swarmkit/fips" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/manager" "github.com/docker/swarmkit/testutils" @@ -154,7 +153,7 @@ func pollServiceReady(t *testing.T, c *testCluster, sid string, replicas int) { } func newCluster(t *testing.T, numWorker, numManager int) *testCluster { - cl := newTestCluster(t.Name()) + cl := newTestCluster(t.Name(), false) for i := 0; i < numManager; i++ { require.NoError(t, cl.AddManager(false, nil), "manager number %d", i+1) } @@ -166,8 +165,8 @@ func newCluster(t *testing.T, numWorker, numManager int) *testCluster { return cl } -func newClusterWithRootCA(t *testing.T, numWorker, numManager int, rootCA *ca.RootCA) *testCluster { - cl := newTestCluster(t.Name()) +func newClusterWithRootCA(t *testing.T, numWorker, numManager int, rootCA *ca.RootCA, fips bool) *testCluster { + cl := newTestCluster(t.Name(), fips) for i := 0; i < numManager; i++ { require.NoError(t, cl.AddManager(false, rootCA), "manager number %d", i+1) } @@ -194,7 +193,7 @@ func TestServiceCreateLateBind(t *testing.T) { numWorker, numManager := 3, 3 - cl := newTestCluster(t.Name()) + cl := newTestCluster(t.Name(), false) for i := 0; i < numManager; i++ { require.NoError(t, cl.AddManager(true, nil), "manager number %d", i+1) } @@ -268,19 +267,12 @@ func TestNodeOps(t *testing.T) { func TestAutolockManagers(t *testing.T) { t.Parallel() - // run this twice, once with root ca with pkcs1 key and then pkcs8 key - defer os.Unsetenv(fips.EnvVar) - for _, pkcs1 := range []bool{true, false} { - if pkcs1 { - os.Unsetenv(fips.EnvVar) - } else { - os.Setenv(fips.EnvVar, "1") - } - + // run this twice, once with FIPS set and once without FIPS set + for _, fips := range []bool{true, false} { rootCA, err := ca.CreateRootCA("rootCN") require.NoError(t, err) numWorker, numManager := 1, 1 - cl := newClusterWithRootCA(t, numWorker, numManager, &rootCA) + cl := newClusterWithRootCA(t, numWorker, numManager, &rootCA, fips) defer func() { require.NoError(t, cl.Stop()) }() @@ -551,7 +543,7 @@ func TestForceNewCluster(t *testing.T) { // start a new cluster with the external CA bootstrapped numWorker, numManager := 0, 1 - cl := newTestCluster(t.Name()) + cl := newTestCluster(t.Name(), false) defer func() { require.NoError(t, cl.Stop()) }() @@ -621,20 +613,13 @@ func pollRootRotationDone(t *testing.T, cl *testCluster) { func TestSuccessfulRootRotation(t *testing.T) { t.Parallel() - // run this twice, once with root ca with pkcs1 key and then pkcs8 key - defer os.Unsetenv(fips.EnvVar) - for _, pkcs1 := range []bool{true, false} { - if pkcs1 { - os.Unsetenv(fips.EnvVar) - } else { - os.Setenv(fips.EnvVar, "1") - } - + // run this twice, once with FIPS set and once without + for _, fips := range []bool{true, false} { rootCA, err := ca.CreateRootCA("rootCN") require.NoError(t, err) numWorker, numManager := 2, 3 - cl := newClusterWithRootCA(t, numWorker, numManager, &rootCA) + cl := newClusterWithRootCA(t, numWorker, numManager, &rootCA, fips) defer func() { require.NoError(t, cl.Stop()) }() @@ -858,7 +843,7 @@ func TestNodeJoinWithWrongCerts(t *testing.T) { require.NoError(t, err) for role, token := range tokens { - node, err := newTestNode(joinAddr, token, false) + node, err := newTestNode(joinAddr, token, false, false) require.NoError(t, err) nodeID := identity.NewID() require.NoError(t, diff --git a/integration/node.go b/integration/node.go index 9ea075ec6f..6b2100bfdf 100644 --- a/integration/node.go +++ b/integration/node.go @@ -54,7 +54,7 @@ func generateCerts(tmpDir string, rootCA *ca.RootCA, nodeID, role, org string, w // existing cluster. if joinAddr is empty string, then new cluster will be initialized. // It uses TestExecutor as executor. If lateBind is set, the remote API port is not // bound. If rootCA is set, this root is used to bootstrap the node's TLS certs. -func newTestNode(joinAddr, joinToken string, lateBind bool) (*testNode, error) { +func newTestNode(joinAddr, joinToken string, lateBind bool, fips bool) (*testNode, error) { tmpDir, err := ioutil.TempDir("", "swarmkit-integration-") if err != nil { return nil, err @@ -67,6 +67,7 @@ func newTestNode(joinAddr, joinToken string, lateBind bool) (*testNode, error) { StateDir: tmpDir, Executor: &agentutils.TestExecutor{}, JoinToken: joinToken, + FIPS: fips, } if !lateBind { cfg.ListenRemoteAPI = "127.0.0.1:0" diff --git a/node/node.go b/node/node.go index 1209d6de06..ab083d20ca 100644 --- a/node/node.go +++ b/node/node.go @@ -14,6 +14,8 @@ import ( "sync" "time" + "github.com/docker/swarmkit/ca/keyutils" + "github.com/boltdb/bolt" "github.com/docker/docker/pkg/plugingetter" metrics "github.com/docker/go-metrics" @@ -758,6 +760,10 @@ func (n *Node) loadSecurityConfig(ctx context.Context, paths *ca.SecurityConfigP ) krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{}) + // if FIPS is required, we want to make sure our key is stored in PKCS8 format + if n.config.FIPS { + krw.SetKeyFormatter(keyutils.FIPS) + } if err := krw.Migrate(); err != nil { return nil, nil, err } diff --git a/node/node_test.go b/node/node_test.go index 709b0d3d98..3c3d8d5b63 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -16,8 +16,10 @@ import ( agentutils "github.com/docker/swarmkit/agent/testutils" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" + "github.com/docker/swarmkit/ca/keyutils" cautils "github.com/docker/swarmkit/ca/testutils" "github.com/docker/swarmkit/identity" + "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/testutils" "github.com/pkg/errors" @@ -25,6 +27,10 @@ import ( "golang.org/x/net/context" ) +func getLoggingContext(t *testing.T) context.Context { + return log.WithLogger(context.Background(), log.L.WithField("test", t.Name())) +} + // If there is nothing on disk and no join addr, we create a new CA and a new set of TLS certs. // If AutoLockManagers is enabled, the TLS key is encrypted with a randomly generated lock key. func TestLoadSecurityConfigNewNode(t *testing.T) { @@ -148,9 +154,9 @@ func TestLoadSecurityConfigLoadFromDisk(t *testing.T) { require.Equal(t, ErrInvalidUnlockKey, err) // Invalid CA - rootCA, err = ca.CreateRootCA(ca.DefaultRootCN) + otherRootCA, err := ca.CreateRootCA(ca.DefaultRootCN) require.NoError(t, err) - require.NoError(t, ca.SaveRootCA(rootCA, paths.RootCA)) + require.NoError(t, ca.SaveRootCA(otherRootCA, paths.RootCA)) node, err = New(&Config{ StateDir: tempdir, JoinAddr: peer.Addr, @@ -160,6 +166,21 @@ func TestLoadSecurityConfigLoadFromDisk(t *testing.T) { require.NoError(t, err) _, _, err = node.loadSecurityConfig(context.Background(), paths) require.IsType(t, x509.UnknownAuthorityError{}, errors.Cause(err)) + + // Convert to PKCS1 and require FIPS + require.NoError(t, krw.DowngradeKey()) + // go back to the previous root CA + require.NoError(t, ca.SaveRootCA(rootCA, paths.RootCA)) + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + UnlockKey: []byte("passphrase"), + FIPS: true, + }) + require.NoError(t, err) + _, _, err = node.loadSecurityConfig(context.Background(), paths) + require.Equal(t, keyutils.ErrFIPSUnsupportedKeyFormat, errors.Cause(err)) } // If there is no CA, and a join addr is provided, one is downloaded from the @@ -488,3 +509,40 @@ func TestManagerFailedStartup(t *testing.T) { require.EqualError(t, node.err, "manager stopped: can't initialize raft node: attempted to join raft cluster without knowing own address") } } + +// TestFIPSConfiguration ensures that new keys will be stored in PKCS8 format. +func TestFIPSConfiguration(t *testing.T) { + ctx := getLoggingContext(t) + tmpDir, err := ioutil.TempDir("", "fips") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + paths := ca.NewConfigPaths(filepath.Join(tmpDir, "certificates")) + + // don't bother with a listening socket + cAddr := filepath.Join(tmpDir, "control.sock") + cfg := &Config{ + ListenControlAPI: cAddr, + StateDir: tmpDir, + Executor: &agentutils.TestExecutor{}, + FIPS: true, + } + node, err := New(cfg) + require.NoError(t, err) + require.NoError(t, node.Start(ctx)) + defer func() { + require.NoError(t, node.Stop(ctx)) + }() + + select { + case <-node.Ready(): + case <-time.After(5 * time.Second): + require.FailNow(t, "node did not ready in time") + } + + nodeKey, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + pemBlock, _ := pem.Decode(nodeKey) + require.NotNil(t, pemBlock) + require.True(t, keyutils.IsPKCS8(pemBlock.Bytes)) +}