Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

etcdserver: add OS level FD metrics

Browse Source 
Similar counts are exposed via Prometheus.
This adds the one that are perceived by etcd server.

e.g.

os_fd_limit 120000
os_fd_used 14
process_cpu_seconds_total 0.31
process_max_fds 120000
process_open_fds 17

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
This commit is contained in:
Gyuho Lee 2020-08-12 10:23:22 -07:00
parent 0bed5fffd4
commit 5bc8f1650c
1 changed files with 17 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
etcdserver/metrics.go
View File

@ -129,6 +129,19 @@ var (
Help: "Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.", Help: "Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.",
}, },
[]string{"server_id"}) []string{"server_id"})
fdUsed = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "os",
Subsystem: "fd",
Name: "used",
Help: "The number of used file descriptors.",
})
fdLimit = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "os",
Subsystem: "fd",
Name: "limit",
Help: "The file descriptor limit.",
})
) )
func init() { func init() {
@ -149,6 +162,8 @@ func init() {
prometheus.MustRegister(currentVersion) prometheus.MustRegister(currentVersion)
prometheus.MustRegister(currentGoVersion) prometheus.MustRegister(currentGoVersion)
prometheus.MustRegister(serverID) prometheus.MustRegister(serverID)
prometheus.MustRegister(fdUsed)
prometheus.MustRegister(fdLimit)
currentVersion.With(prometheus.Labels{ currentVersion.With(prometheus.Labels{
"server_version": version.Version, "server_version": version.Version,
@ -159,7 +174,6 @@ func init() {
} }
func monitorFileDescriptor(done <-chan struct{}) { func monitorFileDescriptor(done <-chan struct{}) {
// This ticker will check File Descriptor Requirements ,and count all fds in used. // This ticker will check File Descriptor Requirements ,and count all fds in used.
// And recorded some logs when in used >= limit/5*4. Just recorded message. // And recorded some logs when in used >= limit/5*4. Just recorded message.
// If fds was more than 10K,It's low performance due to FDUsage() works. // If fds was more than 10K,It's low performance due to FDUsage() works.
@ -173,11 +187,13 @@ func monitorFileDescriptor(done <-chan struct{}) {
plog.Errorf("cannot monitor file descriptor usage (%v)", err) plog.Errorf("cannot monitor file descriptor usage (%v)", err)
return return
} }
fdUsed.Set(float64(used))
limit, err := runtime.FDLimit() limit, err := runtime.FDLimit()
if err != nil { if err != nil {
plog.Errorf("cannot monitor file descriptor usage (%v)", err) plog.Errorf("cannot monitor file descriptor usage (%v)", err)
return return
} }
fdLimit.Set(float64(limit))
if used >= limit/5*4 { if used >= limit/5*4 {
plog.Warningf("80%% of the file descriptor limit is used [used = %d, limit = %d]", used, limit) plog.Warningf("80%% of the file descriptor limit is used [used = %d, limit = %d]", used, limit)
} }