diff --git a/snap/db.go b/snap/db.go index 01d897ae8..dcbd3bd67 100644 --- a/snap/db.go +++ b/snap/db.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "path/filepath" + "time" "github.com/coreos/etcd/pkg/fileutil" ) @@ -30,6 +31,8 @@ var ErrNoDBSnapshot = errors.New("snap: snapshot file doesn't exist") // SaveDBFrom saves snapshot of the database from the given reader. It // guarantees the save operation is atomic. func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { + start := time.Now() + f, err := ioutil.TempFile(s.dir, "tmp") if err != nil { return 0, err @@ -37,7 +40,9 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { var n int64 n, err = io.Copy(f, r) if err == nil { + fsyncStart := time.Now() err = fileutil.Fsync(f) + snapDBFsyncSec.Observe(time.Since(fsyncStart).Seconds()) } f.Close() if err != nil { @@ -57,6 +62,7 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { plog.Infof("saved database snapshot to disk [total bytes: %d]", n) + snapDBSaveSec.Observe(time.Since(start).Seconds()) return n, nil } diff --git a/snap/metrics.go b/snap/metrics.go index 433ef09d4..0d3b7e63e 100644 --- a/snap/metrics.go +++ b/snap/metrics.go @@ -33,9 +33,33 @@ var ( Help: "The marshalling cost distributions of save called by snapshot.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) + + snapDBSaveSec = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "etcd", + Subsystem: "snap_db", + Name: "save_total_duration_seconds", + Help: "The total latency distributions of v3 snapshot save", + + // lowest bucket start of upper bound 0.1 sec (100 ms) with factor 2 + // highest bucket start of 0.1 sec * 2^9 == 51.2 sec + Buckets: prometheus.ExponentialBuckets(0.1, 2, 10), + }) + + snapDBFsyncSec = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "etcd", + Subsystem: "snap_db", + Name: "fsync_duration_seconds", + Help: "The latency distributions of fsyncing .snap.db file", + + // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 + // highest bucket start of 0.001 sec * 2^13 == 8.192 sec + Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), + }) ) func init() { prometheus.MustRegister(saveDurations) prometheus.MustRegister(marshallingDurations) + prometheus.MustRegister(snapDBSaveSec) + prometheus.MustRegister(snapDBFsyncSec) }