fileutil: support file extending preallocate

pkg/fileutil/preallocate.go

@@ -12,31 +12,36 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build linux
-
 package fileutil
 
-import (
-	"os"
-	"syscall"
-)
+import "os"
 
 // Preallocate tries to allocate the space for given
 // file. This operation is only supported on linux by a
 // few filesystems (btrfs, ext4, etc.).
 // If the operation is unsupported, no error will be returned.
 // Otherwise, the error encountered will be returned.
-func Preallocate(f *os.File, sizeInBytes int) error {
-	// use mode = 1 to keep size
-	// see FALLOC_FL_KEEP_SIZE
-	err := syscall.Fallocate(int(f.Fd()), 1, 0, int64(sizeInBytes))
+func Preallocate(f *os.File, sizeInBytes int64, extendFile bool) error {
+	if extendFile {
+		preallocExtend(f, sizeInBytes)
+	}
+	return preallocFixed(f, sizeInBytes)
+}
+
+func preallocExtendTrunc(f *os.File, sizeInBytes int64) error {
+	curOff, err := f.Seek(0, os.SEEK_CUR)
 	if err != nil {
-		errno, ok := err.(syscall.Errno)
-		// treat not support as nil error
-		if ok && errno == syscall.ENOTSUP {
-			return nil
-		}
 		return err
 	}
-	return nil
+	size, err := f.Seek(sizeInBytes, os.SEEK_END)
+	if err != nil {
+		return err
+	}
+	if _, err = f.Seek(curOff, os.SEEK_SET); err != nil {
+		return err
+	}
+	if sizeInBytes > size {
+		return nil
+	}
+	return f.Truncate(sizeInBytes)
 }

pkg/fileutil/preallocate_darwin.go

@@ -0,0 +1,43 @@
+// Copyright 2016 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build darwin
+
+package fileutil
+
+import (
+	"os"
+	"syscall"
+	"unsafe"
+)
+
+func preallocExtend(f *os.File, sizeInBytes int64) error {
+	if err := preallocFixed(f, sizeInBytes); err != nil {
+		return err
+	}
+	return preallocExtendTrunc(f, sizeInBytes)
+}
+
+func preallocFixed(f *os.File, sizeInBytes int64) error {
+	fstore := &syscall.Fstore_t{
+		Flags:   syscall.F_ALLOCATEALL,
+		Posmode: syscall.F_PEOFPOSMODE,
+		Length:  sizeInBytes}
+	p := unsafe.Pointer(fstore)
+	_, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_PREALLOCATE), uintptr(p))
+	if errno == 0 || errno == syscall.ENOTSUP {
+		return nil
+	}
+	return errno
+}

pkg/fileutil/preallocate_test.go

@@ -17,31 +17,31 @@ package fileutil
 import (
 	"io/ioutil"
 	"os"
-	"runtime"
 	"testing"
 )
 
-func TestPreallocate(t *testing.T) {
-	if runtime.GOOS != "linux" {
-		t.Skipf("skip testPreallocate, OS = %s", runtime.GOOS)
-	}
-
-	p, err := ioutil.TempDir(os.TempDir(), "preallocateTest")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(p)
-
-	f, err := ioutil.TempFile(p, "")
-	if err != nil {
+func TestPreallocateExtend(t *testing.T) { runPreallocTest(t, testPreallocateExtend) }
+func testPreallocateExtend(t *testing.T, f *os.File) {
+	size := int64(64 * 1000)
+	if err := Preallocate(f, size, true); err != nil {
 		t.Fatal(err)
 	}
 
-	size := 64 * 1000
-	err = Preallocate(f, size)
+	stat, err := f.Stat()
 	if err != nil {
 		t.Fatal(err)
 	}
+	if stat.Size() != size {
+		t.Errorf("size = %d, want %d", stat.Size(), size)
+	}
+}
+
+func TestPreallocateFixed(t *testing.T) { runPreallocTest(t, testPreallocateFixed) }
+func testPreallocateFixed(t *testing.T, f *os.File) {
+	size := int64(64 * 1000)
+	if err := Preallocate(f, size, false); err != nil {
+		t.Fatal(err)
+	}
 
 	stat, err := f.Stat()
 	if err != nil {
@@ -51,3 +51,17 @@ func TestPreallocate(t *testing.T) {
 		t.Errorf("size = %d, want %d", stat.Size(), 0)
 	}
 }
+
+func runPreallocTest(t *testing.T, test func(*testing.T, *os.File)) {
+	p, err := ioutil.TempDir(os.TempDir(), "preallocateTest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(p)
+
+	f, err := ioutil.TempFile(p, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	test(t, f)
+}

pkg/fileutil/preallocate_unix.go

@@ -0,0 +1,48 @@
+// Copyright 2016 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package fileutil
+
+import (
+	"os"
+	"syscall"
+)
+
+func preallocExtend(f *os.File, sizeInBytes int64) error {
+	// use mode = 0 to change size
+	err := syscall.Fallocate(int(f.Fd()), 0, 0, sizeInBytes)
+	if err != nil {
+		errno, ok := err.(syscall.Errno)
+		// treat not support as nil error
+		if ok && errno == syscall.ENOTSUP {
+			return preallocExtendTrunc(f, sizeInBytes)
+		}
+	}
+	return err
+}
+
+func preallocFixed(f *os.File, sizeInBytes int64) error {
+	// use mode = 1 to keep size; see FALLOC_FL_KEEP_SIZE
+	err := syscall.Fallocate(int(f.Fd()), 1, 0, sizeInBytes)
+	if err != nil {
+		errno, ok := err.(syscall.Errno)
+		// treat not supported as nil error
+		if ok && errno == syscall.ENOTSUP {
+			return nil
+		}
+	}
+	return err
+}

pkg/fileutil/preallocate_unsupported.go

@@ -12,17 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build !linux
+// +build !linux,!darwin
 
 package fileutil
 
 import "os"
 
-// Preallocate tries to allocate the space for given
-// file. This operation is only supported on linux by a
-// few filesystems (btrfs, ext4, etc.).
-// If the operation is unsupported, no error will be returned.
-// Otherwise, the error encountered will be returned.
-func Preallocate(f *os.File, sizeInBytes int) error {
-	return nil
+func preallocExtend(f *os.File, sizeInBytes int64) error {
+	return preallocExtendTrunc(f, sizeInBytes)
 }
+
+func preallocFixed(f *os.File, sizeInBytes int64) error { return nil }

wal/wal.go

@@ -192,7 +192,7 @@ func openAtIndex(dirpath string, snap walpb.Snapshot, write bool) (*WAL, error)
 			rc.Close()
 			return nil, err
 		}
-		if err := fileutil.Preallocate(w.tail().File, segmentSizeBytes); err != nil {
+		if err := fileutil.Preallocate(w.tail().File, segmentSizeBytes, false); err != nil {
 			rc.Close()
 			plog.Errorf("failed to allocate space when creating new wal file (%v)", err)
 			return nil, err
@@ -350,7 +350,7 @@ func (w *WAL) cut() error {
 	prevCrc = w.encoder.crc.Sum32()
 	w.encoder = newEncoder(w.tail(), prevCrc)
 
-	if err = fileutil.Preallocate(w.tail().File, segmentSizeBytes); err != nil {
+	if err = fileutil.Preallocate(w.tail().File, segmentSizeBytes, false); err != nil {
 		plog.Errorf("failed to allocate space when creating new wal file (%v)", err)
 		return err
 	}