diff --git a/manager/state/raft/raft_test.go b/manager/state/raft/raft_test.go index c91336b2ba..3b945ad971 100644 --- a/manager/state/raft/raft_test.go +++ b/manager/state/raft/raft_test.go @@ -18,6 +18,7 @@ import ( "golang.org/x/net/context" "github.com/Sirupsen/logrus" + "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/wal" "github.com/docker/swarmkit/api" cautils "github.com/docker/swarmkit/ca/testutils" @@ -319,10 +320,8 @@ func TestRaftLeaderLeave(t *testing.T) { raftutils.WaitForCluster(t, clockSource, newCluster) // Node1's state should be cleared - _, err = os.Stat(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted")) - require.True(t, os.IsNotExist(err)) - _, err = os.Stat(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted")) - require.True(t, os.IsNotExist(err)) + require.False(t, fileutil.Exist(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted"))) + require.False(t, fileutil.Exist(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted"))) require.Equal(t, raft.EncryptionKeys{}, nodes[1].KeyRotator.GetKeys()) // Leader should not be 1 diff --git a/manager/state/raft/storage/storage.go b/manager/state/raft/storage/storage.go index d830767ded..3bddedd025 100644 --- a/manager/state/raft/storage/storage.go +++ b/manager/state/raft/storage/storage.go @@ -160,12 +160,13 @@ func (e *EncryptedRaftLogger) BootstrapNew(metadata []byte) error { encrypter, decrypter := encryption.Defaults(e.EncryptionKey) walFactory := NewWALFactory(encrypter, decrypter) - for _, dirpath := range []string{e.walDir(), e.snapDir()} { + for _, dirpath := range []string{filepath.Dir(e.walDir()), e.snapDir()} { if err := os.MkdirAll(dirpath, 0700); err != nil { return errors.Wrapf(err, "failed to create %s", dirpath) } } var err error + // the wal directory must not already exist upon creation e.wal, err = walFactory.Create(e.walDir(), metadata) if err != nil { return errors.Wrap(err, "failed to create WAL") @@ -373,8 +374,8 @@ func (e *EncryptedRaftLogger) Clear(ctx context.Context) error { if err != nil { return err } - err = os.Rename(e.walDir(), newWALDir) - if err != nil { + os.RemoveAll(newWALDir) + if err = os.Rename(e.walDir(), newWALDir); err != nil { return err } @@ -382,8 +383,8 @@ func (e *EncryptedRaftLogger) Clear(ctx context.Context) error { if err != nil { return err } - err = os.Rename(e.snapDir(), newSnapDir) - if err != nil { + os.RemoveAll(newSnapDir) + if err := os.Rename(e.snapDir(), newSnapDir); err != nil { return err } diff --git a/manager/state/raft/storage/storage_test.go b/manager/state/raft/storage/storage_test.go index 2683d1d735..af061e30a2 100644 --- a/manager/state/raft/storage/storage_test.go +++ b/manager/state/raft/storage/storage_test.go @@ -151,9 +151,7 @@ func TestMigrateToV3EncryptedForm(t *testing.T) { writeDataTo := func(suffix string, snapshot raftpb.Snapshot, walFactory WALFactory, snapFactory SnapFactory) []raftpb.Entry { snapDir := filepath.Join(tempdir, "snap"+suffix) walDir := filepath.Join(tempdir, "wal"+suffix) - for _, dirpath := range []string{snapDir, walDir} { - require.NoError(t, os.MkdirAll(dirpath, 0755)) - } + require.NoError(t, os.MkdirAll(snapDir, 0755)) require.NoError(t, snapFactory.New(snapDir).SaveSnap(snapshot)) _, entries, _ := makeWALData(snapshot.Metadata.Index, snapshot.Metadata.Term) diff --git a/manager/state/raft/storage/walwrap.go b/manager/state/raft/storage/walwrap.go index 87009d4827..5a6c71ae61 100644 --- a/manager/state/raft/storage/walwrap.go +++ b/manager/state/raft/storage/walwrap.go @@ -9,7 +9,6 @@ import ( "sort" "strings" - "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/raft/raftpb" "github.com/coreos/etcd/wal" "github.com/coreos/etcd/wal/walpb" @@ -204,16 +203,16 @@ func MigrateWALs(ctx context.Context, oldDir, newDir string, oldFactory, newFact } oldReader.Close() + if err := os.MkdirAll(filepath.Dir(newDir), 0700); err != nil { + return errors.Wrap(err, "could not create parent directory") + } + // keep temporary wal directory so WAL initialization appears atomic tmpdirpath := filepath.Clean(newDir) + ".tmp" - if fileutil.Exist(tmpdirpath) { - if err := os.RemoveAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not remove temporary WAL directory") - } - } - if err := fileutil.CreateDirAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not create temporary WAL directory") + if err := os.RemoveAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not remove temporary WAL directory") } + defer os.RemoveAll(tmpdirpath) tmpWAL, err := newFactory.Create(tmpdirpath, waldata.Metadata) if err != nil { @@ -228,6 +227,9 @@ func MigrateWALs(ctx context.Context, oldDir, newDir string, oldFactory, newFact if err := tmpWAL.Save(waldata.HardState, waldata.Entries); err != nil { return errors.Wrap(err, "could not migrate WALs to temporary directory") } + if err := tmpWAL.Close(); err != nil { + return err + } return os.Rename(tmpdirpath, newDir) } diff --git a/manager/state/raft/storage/walwrap_test.go b/manager/state/raft/storage/walwrap_test.go index 24ca563eb5..ffe4d39107 100644 --- a/manager/state/raft/storage/walwrap_test.go +++ b/manager/state/raft/storage/walwrap_test.go @@ -40,6 +40,7 @@ func makeWALData(index uint64, term uint64) ([]byte, []raftpb.Entry, walpb.Snaps func createWithWAL(t *testing.T, w WALFactory, metadata []byte, startSnap walpb.Snapshot, entries []raftpb.Entry) string { walDir, err := ioutil.TempDir("", "waltests") require.NoError(t, err) + require.NoError(t, os.RemoveAll(walDir)) walWriter, err := w.Create(walDir, metadata) require.NoError(t, err) @@ -162,6 +163,7 @@ func TestSaveEncryptionFails(t *testing.T) { tempdir, err := ioutil.TempDir("", "waltests") require.NoError(t, err) + os.RemoveAll(tempdir) defer os.RemoveAll(tempdir) // fail encrypting one of the entries, but not the first one @@ -195,7 +197,11 @@ func TestCreateOpenInvalidDirFails(t *testing.T) { _, err := c.Create("/not/existing/directory", []byte("metadata")) require.Error(t, err) - _, err = c.Open("/not/existing/directory", walpb.Snapshot{}) + tempDir, err := ioutil.TempDir("", "test-migrate") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + _, err = c.Open(tempDir, walpb.Snapshot{}) // invalid because no WAL file require.Error(t, err) }