diff --git a/app/app.go b/app/app.go index e7b92760c..d1f8f743b 100644 --- a/app/app.go +++ b/app/app.go @@ -87,6 +87,7 @@ func (app *kaspadApp) main(startedChan chan<- struct{}) error { if app.cfg.Profile != "" { profiling.Start(app.cfg.Profile, log) } + profiling.TrackHeap(app.cfg.AppDir, log) // Return now if an interrupt signal was triggered. if signal.InterruptRequested(interrupt) { diff --git a/util/profiling/profiling.go b/util/profiling/profiling.go index 11818d05f..29b6b13ca 100644 --- a/util/profiling/profiling.go +++ b/util/profiling/profiling.go @@ -1,16 +1,26 @@ package profiling import ( + "fmt" "github.com/kaspanet/kaspad/infrastructure/logger" "net" "net/http" + "os" + "path/filepath" + "time" // Required for profiling _ "net/http/pprof" "github.com/kaspanet/kaspad/util/panics" + "runtime" + "runtime/pprof" ) +// heapDumpFileName is the name of the heap dump file. We want every run to have its own +// file, so we append the timestamp of the program launch time to the file name. +var heapDumpFileName = fmt.Sprintf("heap-%s.pprof", time.Now().Format(time.RFC3339)) + // Start starts the profiling server func Start(port string, log *logger.Logger) { spawn := panics.GoroutineWrapperFunc(log) @@ -22,3 +32,45 @@ func Start(port string, log *logger.Logger) { log.Error(http.ListenAndServe(listenAddr, nil)) }) } + +// TrackHeap tracks the size of the heap and dumps a profile if it passes a limit +func TrackHeap(appDir string, log *logger.Logger) { + spawn := panics.GoroutineWrapperFunc(log) + spawn("profiling.TrackHeap", func() { + dumpFolder := filepath.Join(appDir, "dumps") + err := os.MkdirAll(dumpFolder, 0700) + if err != nil { + log.Errorf("Could not create heap dumps folder at %s", dumpFolder) + return + } + const limitInGigabytes = 8 + trackHeapSize(limitInGigabytes*1024*1024*1024, dumpFolder, log) + }) +} + +func trackHeapSize(heapLimit uint64, dumpFolder string, log *logger.Logger) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for range ticker.C { + memStats := &runtime.MemStats{} + runtime.ReadMemStats(memStats) + // If we passed the expected heap limit, dump the heap profile to a file + if memStats.HeapAlloc > heapLimit { + dumpHeapProfile(heapLimit, dumpFolder, memStats, log) + } + } +} + +func dumpHeapProfile(heapLimit uint64, dumpFolder string, memStats *runtime.MemStats, log *logger.Logger) { + heapFile := filepath.Join(dumpFolder, heapDumpFileName) + log.Infof("Saving heap statistics into %s (HeapAlloc=%d > %d=heapLimit)", heapFile, memStats.HeapAlloc, heapLimit) + f, err := os.Create(heapFile) + defer f.Close() + if err != nil { + log.Infof("Could not create heap profile: %s", err) + return + } + if err := pprof.WriteHeapProfile(f); err != nil { + log.Infof("Could not write heap profile: %s", err) + } +}