Merge pull request #10689 from joshcc3/master

raft: cleanup wal directory if creation fails

wal/wal.go

@@ -184,6 +184,13 @@ func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) {
 		return nil, err
 	}
 
+	var perr error
+	defer func() {
+		if perr != nil {
+			w.cleanupWAL(lg)
+		}
+	}()
+
 	// directory was renamed; sync parent dir to persist rename
 	pdir, perr := fileutil.OpenDir(filepath.Dir(w.dir))
 	if perr != nil {
@@ -208,7 +215,7 @@ func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) {
 		}
 		return nil, perr
 	}
-	if perr = pdir.Close(); err != nil {
+	if perr = pdir.Close(); perr != nil {
 		if lg != nil {
 			lg.Warn(
 				"failed to close the parent data directory file",
@@ -223,6 +230,30 @@ func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) {
 	return w, nil
 }
 
+func (w *WAL) cleanupWAL(lg *zap.Logger) {
+	var err error
+	if err = w.Close(); err != nil {
+		if lg != nil {
+			lg.Panic("failed to close WAL during cleanup", zap.Error(err))
+		} else {
+			plog.Panicf("failed to close WAL during cleanup: %v", err)
+		}
+	}
+	brokenDirName := fmt.Sprintf("%s.broken.%v", w.dir, time.Now().Format("20060102.150405.999999"))
+	if err = os.Rename(w.dir, brokenDirName); err != nil {
+		if lg != nil {
+			lg.Panic(
+				"failed to rename WAL during cleanup",
+				zap.Error(err),
+				zap.String("source-path", w.dir),
+				zap.String("rename-path", brokenDirName),
+			)
+		} else {
+			plog.Panicf("failed to rename WAL during cleanup: %v", err)
+		}
+	}
+}
+
 func (w *WAL) renameWAL(tmpdirpath string) (*WAL, error) {
 	if err := os.RemoveAll(w.dir); err != nil {
 		return nil, err

wal/wal_test.go

@@ -16,6 +16,7 @@ package wal
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"math"
@@ -23,6 +24,7 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"testing"
 
 	"go.etcd.io/etcd/v3/pkg/fileutil"
@@ -101,6 +103,37 @@ func TestCreateFailFromPollutedDir(t *testing.T) {
 	}
 }
 
+func TestWalCleanup(t *testing.T) {
+	testRoot, err := ioutil.TempDir(os.TempDir(), "waltestroot")
+	if err != nil {
+		t.Fatal(err)
+	}
+	p, err := ioutil.TempDir(testRoot, "waltest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(testRoot)
+
+	logger := zap.NewExample()
+	w, err := Create(logger, p, []byte(""))
+	if err != nil {
+		t.Fatalf("err = %v, want nil", err)
+	}
+	w.cleanupWAL(logger)
+	fnames, err := fileutil.ReadDir(testRoot)
+	if err != nil {
+		t.Fatalf("err = %v, want nil", err)
+	}
+	if len(fnames) != 1 {
+		t.Fatalf("expected 1 file under %v, got %v", testRoot, len(fnames))
+	}
+	pattern := fmt.Sprintf(`%s.broken\.[\d]{8}\.[\d]{6}\.[\d]{1,6}?`, filepath.Base(p))
+	match, _ := regexp.MatchString(pattern, fnames[0])
+	if !match {
+		t.Errorf("match = false, expected true for %v with pattern %v", fnames[0], pattern)
+	}
+}
+
 func TestCreateFailFromNoSpaceLeft(t *testing.T) {
 	p, err := ioutil.TempDir(os.TempDir(), "waltest")
 	if err != nil {