From 5bc8f1650c103cfff86cadc3ac4a63deea593be4 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 12 Aug 2020 10:23:22 -0700 Subject: [PATCH] etcdserver: add OS level FD metrics Similar counts are exposed via Prometheus. This adds the one that are perceived by etcd server. e.g. os_fd_limit 120000 os_fd_used 14 process_cpu_seconds_total 0.31 process_max_fds 120000 process_open_fds 17 Signed-off-by: Gyuho Lee --- etcdserver/metrics.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/etcdserver/metrics.go b/etcdserver/metrics.go index b3311a462..d543c30f4 100644 --- a/etcdserver/metrics.go +++ b/etcdserver/metrics.go @@ -129,6 +129,19 @@ var ( Help: "Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.", }, []string{"server_id"}) + + fdUsed = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "os", + Subsystem: "fd", + Name: "used", + Help: "The number of used file descriptors.", + }) + fdLimit = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "os", + Subsystem: "fd", + Name: "limit", + Help: "The file descriptor limit.", + }) ) func init() { @@ -149,6 +162,8 @@ func init() { prometheus.MustRegister(currentVersion) prometheus.MustRegister(currentGoVersion) prometheus.MustRegister(serverID) + prometheus.MustRegister(fdUsed) + prometheus.MustRegister(fdLimit) currentVersion.With(prometheus.Labels{ "server_version": version.Version, @@ -159,7 +174,6 @@ func init() { } func monitorFileDescriptor(done <-chan struct{}) { - // This ticker will check File Descriptor Requirements ,and count all fds in used. // And recorded some logs when in used >= limit/5*4. Just recorded message. // If fds was more than 10K,It's low performance due to FDUsage() works. @@ -173,11 +187,13 @@ func monitorFileDescriptor(done <-chan struct{}) { plog.Errorf("cannot monitor file descriptor usage (%v)", err) return } + fdUsed.Set(float64(used)) limit, err := runtime.FDLimit() if err != nil { plog.Errorf("cannot monitor file descriptor usage (%v)", err) return } + fdLimit.Set(float64(limit)) if used >= limit/5*4 { plog.Warningf("80%% of the file descriptor limit is used [used = %d, limit = %d]", used, limit) }