Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

etcdserver: add OS level FD metrics

Browse Source 
Similar counts are exposed via Prometheus.
This adds the one that are perceived by etcd server.

e.g.

os_fd_limit 120000
os_fd_used 14
process_cpu_seconds_total 0.31
process_max_fds 120000
process_open_fds 17

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
This commit is contained in:
Gyuho Lee 2020-08-12 10:23:22 -07:00
parent cf558ee8b7
commit 008074187c
1 changed files with 17 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
etcdserver/metrics.go
View File

@ -151,6 +151,19 @@ var (
Help: "Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.", Help: "Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.",
}, },
[]string{"server_id"}) []string{"server_id"})
fdUsed = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "os",
Subsystem: "fd",
Name: "used",
Help: "The number of used file descriptors.",
})
fdLimit = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "os",
Subsystem: "fd",
Name: "limit",
Help: "The file descriptor limit.",
})
) )
func init() { func init() {
@ -174,6 +187,8 @@ func init() {
prometheus.MustRegister(isLearner) prometheus.MustRegister(isLearner)
prometheus.MustRegister(learnerPromoteSucceed) prometheus.MustRegister(learnerPromoteSucceed)
prometheus.MustRegister(learnerPromoteFailed) prometheus.MustRegister(learnerPromoteFailed)
prometheus.MustRegister(fdUsed)
prometheus.MustRegister(fdLimit)
currentVersion.With(prometheus.Labels{ currentVersion.With(prometheus.Labels{
"server_version": version.Version, "server_version": version.Version,
@ -184,7 +199,6 @@ func init() {
} }
func monitorFileDescriptor(lg *zap.Logger, done <-chan struct{}) { func monitorFileDescriptor(lg *zap.Logger, done <-chan struct{}) {
// This ticker will check File Descriptor Requirements ,and count all fds in used. // This ticker will check File Descriptor Requirements ,and count all fds in used.
// And recorded some logs when in used >= limit/5*4. Just recorded message. // And recorded some logs when in used >= limit/5*4. Just recorded message.
// If fds was more than 10K,It's low performance due to FDUsage() works. // If fds was more than 10K,It's low performance due to FDUsage() works.
@ -202,6 +216,7 @@ func monitorFileDescriptor(lg *zap.Logger, done <-chan struct{}) {
} }
return return
} }
fdUsed.Set(float64(used))
limit, err := runtime.FDLimit() limit, err := runtime.FDLimit()
if err != nil { if err != nil {
if lg != nil { if lg != nil {
@ -211,6 +226,7 @@ func monitorFileDescriptor(lg *zap.Logger, done <-chan struct{}) {
} }
return return
} }
fdLimit.Set(float64(limit))
if used >= limit/5*4 { if used >= limit/5*4 {
if lg != nil { if lg != nil {
lg.Warn("80% of file descriptors are used", zap.Uint64("used", used), zap.Uint64("limit", limit)) lg.Warn("80% of file descriptors are used", zap.Uint64("used", used), zap.Uint64("limit", limit))